refactor(core): remove tree in context (#1376)

* refactor(core): remove tree info in uiContext

* chore(core): fix lint

* chore(core): remove dom-based locator

* fix(core): test cases

* chore(core): fix lint

* fix(core): test cases

Author: yuyutaotao

apps/chrome-extension/src/utils/eventOptimizer.ts

@@ -1,5 +1,5 @@
 import Insight from '@midscene/core';
-import type { BaseElement, Rect, UIContext } from '@midscene/core';
+import type { Rect, UIContext } from '@midscene/core';
 import type { RecordedEvent } from '@midscene/recorder';
 import { globalModelConfigManager } from '@midscene/shared/env';
 import { compositeElementInfoImg } from '@midscene/shared/img';
@@ -105,10 +105,9 @@ export const generateAIDescription = async (
 
   const descriptionPromise = (async () => {
     try {
-      const mockContext: UIContext<BaseElement> = {
+      const mockContext: UIContext = {
         screenshotBase64: event.screenshotBefore as string,
         size: { width: event.pageInfo.width, height: event.pageInfo.height },
-        tree: { node: null, children: [] },
       };
 
       const insight = new Insight(mockContext);

packages/core/src/agent/agent.ts

@@ -67,7 +67,6 @@ import {
   parsePrompt,
   printReportMsg,
 } from './utils';
-import { trimContextByViewport } from './utils';
 
 const debug = getDebug('agent');
 
@@ -348,10 +347,8 @@ export class Agent<
   }
 
   appendExecutionDump(execution: ExecutionDump) {
-    // use trimContextByViewport to process execution
-    const trimmedExecution = trimContextByViewport(execution);
     const currentDump = this.dump;
-    currentDump.executions.push(trimmedExecution);
+    currentDump.executions.push(execution);
   }
 
   dumpDataString() {

packages/core/src/agent/index.ts

@@ -11,9 +11,6 @@ export { cacheFileExt } from './task-cache';
 
 export { TaskExecutor } from './tasks';
 
-export {
-  getCurrentExecutionFile,
-  trimContextByViewport,
-} from './utils';
+export { getCurrentExecutionFile } from './utils';
 
 export type { AgentOpt } from '../types';

packages/core/src/agent/task-builder.ts

@@ -19,6 +19,7 @@ import type {
 import { InsightError } from '@/types';
 import { sleep } from '@/utils';
 import type { IModelConfig } from '@midscene/shared/env';
+import { generateElementByPosition } from '@midscene/shared/extractor';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
 import type { TaskCache } from './task-cache';
@@ -351,7 +352,7 @@ export class TaskBuilder {
         let { uiContext } = taskContext;
 
         assert(
-          param?.prompt || param?.id || param?.bbox,
+          param?.prompt || param?.bbox,
           `No prompt or id or position or bbox to locate, param=${JSON.stringify(
             param,
           )}`,
@@ -381,12 +382,18 @@ export class TaskBuilder {
         };
 
         // from xpath
-        let elementFromXpath: Rect | undefined;
+        let rectFromXpath: Rect | undefined;
         if (param.xpath && this.interface.rectMatchesCacheFeature) {
-          elementFromXpath = await this.interface.rectMatchesCacheFeature({
+          rectFromXpath = await this.interface.rectMatchesCacheFeature({
             xpaths: [param.xpath],
           });
         }
+        const elementFromXpath = rectFromXpath
+          ? generateElementByPosition({
+              x: rectFromXpath.left + rectFromXpath.width / 2,
+              y: rectFromXpath.top + rectFromXpath.height / 2,
+            })
+          : undefined;
         const userExpectedPathHitFlag = !!elementFromXpath;
 
         const cachePrompt = param.prompt;
@@ -408,7 +415,7 @@ export class TaskBuilder {
 
         const elementFromPlan =
           !userExpectedPathHitFlag && !cacheHitFlag
-            ? matchElementFromPlan(param, uiContext.tree)
+            ? matchElementFromPlan(param)
             : undefined;
         const planHitFlag = !!elementFromPlan;
 
@@ -485,7 +492,7 @@ export class TaskBuilder {
         if (!element) {
           if (locateDump) {
             throw new InsightError(
-              `Element not found: ${param.prompt}`,
+              `Element not found : ${param.prompt}`,
               locateDump,
             );
           }
@@ -513,8 +520,7 @@ export class TaskBuilder {
           hitBy = {
             from: 'Planning',
             context: {
-              id: elementFromPlan?.id,
-              bbox: elementFromPlan?.bbox,
+              rect: elementFromPlan?.rect,
             },
           };
         }

packages/core/src/agent/tasks.ts

@@ -30,6 +30,7 @@ import { ExecutionSession } from './execution-session';
 import { TaskBuilder } from './task-builder';
 import type { TaskCache } from './task-cache';
 export { locatePlanForLocate } from './task-builder';
+import { descriptionOfTree } from '@midscene/shared/extractor';
 import { taskTitleStr } from './ui-utils';
 import { parsePrompt } from './utils';
 
@@ -422,11 +423,25 @@ export class TaskExecutor {
         }
 
         let extractResult;
+
+        let extraPageDescription = '';
+        if (opt?.domIncluded && this.interface.getElementsNodeTree) {
+          debug('appending tree info for page');
+          const tree = await this.interface.getElementsNodeTree();
+          extraPageDescription = await descriptionOfTree(
+            tree,
+            200,
+            false,
+            opt?.domIncluded === 'visible-only',
+          );
+        }
+
         try {
           extractResult = await this.insight.extract<any>(
             demandInput,
             modelConfig,
             opt,
+            extraPageDescription,
             multimodalPrompt,
           );
         } catch (error) {

packages/core/src/agent/ui-utils.ts

@@ -128,7 +128,6 @@ export function paramStr(task: ExecutionTask) {
   if (task.type === 'Insight') {
     value =
       locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||
-      (task as ExecutionTaskInsightLocate)?.param?.id ||
       (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||
       (task as ExecutionTaskInsightAssertion)?.param?.assertion;
   }

packages/core/src/agent/utils.ts

@@ -1,27 +1,17 @@
-import { elementByPositionWithElementInfo } from '@/ai-model';
 import type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';
 import type { AbstractInterface } from '@/device';
 import type {
-  BaseElement,
   ElementCacheFeature,
-  ElementTreeNode,
-  ExecutionDump,
-  ExecutionTask,
-  ExecutorContext,
   LocateResultElement,
   PlanningLocateParam,
   UIContext,
 } from '@/types';
 import { uploadTestInfoToServer } from '@/utils';
-import { NodeType } from '@midscene/shared/constants';
 import {
   MIDSCENE_REPORT_TAG_NAME,
   globalConfigManager,
 } from '@midscene/shared/env';
-import {
-  generateElementByPosition,
-  getNodeFromCacheList,
-} from '@midscene/shared/extractor';
+import { generateElementByPosition } from '@midscene/shared/extractor';
 import { getDebug } from '@midscene/shared/logger';
 import { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';
 import { assert, logMsg, uuid } from '@midscene/shared/utils';
@@ -55,10 +45,6 @@ export async function commonContextParser(
   debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);
 
   return {
-    tree: {
-      node: null,
-      children: [],
-    },
     size,
     screenshotBase64: screenshotBase64!,
   };
@@ -133,26 +119,18 @@ export function generateCacheId(fileName?: string): string {
 
 export function matchElementFromPlan(
   planLocateParam: PlanningLocateParam,
-  tree: ElementTreeNode<BaseElement>,
-) {
+): LocateResultElement | undefined {
   if (!planLocateParam) {
     return undefined;
   }
-  if (planLocateParam.id) {
-    return getNodeFromCacheList(planLocateParam.id);
-  }
 
   if (planLocateParam.bbox) {
     const centerPosition = {
       x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
       y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),
     };
-    let element = elementByPositionWithElementInfo(tree, centerPosition);
-
-    if (!element) {
-      element = generateElementByPosition(centerPosition) as BaseElement;
-    }
 
+    const element = generateElementByPosition(centerPosition);
     return element;
   }
 
@@ -198,10 +176,6 @@ export async function matchElementFromCache(
         Math.round(rect.top + rect.height / 2),
       ],
       rect,
-      xpaths: [],
-      attributes: {
-        nodeType: NodeType.POSITION,
-      },
     };
 
     cacheDebug('cache hit, prompt: %s', cachePrompt);
@@ -212,59 +186,6 @@ export async function matchElementFromCache(
   }
 }
 
-export function trimContextByViewport(execution: ExecutionDump) {
-  function filterVisibleTree(
-    node: ElementTreeNode<BaseElement>,
-  ): ElementTreeNode<BaseElement> | null {
-    if (!node) return null;
-
-    // recursively process all children
-    const filteredChildren = Array.isArray(node.children)
-      ? (node.children
-          .map(filterVisibleTree)
-          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])
-      : [];
-
-    // if the current node is visible, keep it and the filtered children
-    if (node.node && node.node.isVisible === true) {
-      return {
-        ...node,
-        children: filteredChildren,
-      };
-    }
-
-    // if the current node is invisible, but has visible children, create an empty node to include these children
-    if (filteredChildren.length > 0) {
-      return {
-        node: null,
-        children: filteredChildren,
-      };
-    }
-
-    // if the current node is invisible and has no visible children, return null
-    return null;
-  }
-
-  return {
-    ...execution,
-    tasks: Array.isArray(execution.tasks)
-      ? execution.tasks.map((task: ExecutionTask) => {
-          const newTask = { ...task };
-          if (task.uiContext?.tree) {
-            newTask.uiContext = {
-              ...task.uiContext,
-              tree: filterVisibleTree(task.uiContext.tree) || {
-                node: null,
-                children: [],
-              },
-            };
-          }
-          return newTask;
-        })
-      : execution.tasks,
-  };
-}
-
 declare const __VERSION__: string | undefined;
 
 export const getMidsceneVersion = (): string => {

packages/core/src/ai-model/common.ts

@@ -1,5 +1,4 @@
 import type {
-  AIUsageInfo,
   BaseElement,
   DeviceAction,
   ElementTreeNode,

packages/core/src/ai-model/index.ts

@@ -4,10 +4,7 @@ export {
   callAI,
 } from './service-caller/index';
 export { systemPromptToLocateElement } from './prompt/llm-locator';
-export {
-  describeUserPage,
-  elementByPositionWithElementInfo,
-} from './prompt/util';
+export { describeUserPage } from './prompt/util';
 export {
   generatePlaywrightTest,
   generatePlaywrightTestStream,