Skip to content

Commit

Permalink
fix: fix scroll action in chrome extension
Browse files Browse the repository at this point in the history
  • Loading branch information
@yuyutaotao
yuyutaotao committed Dec 20, 2024
1 parent 48fa8a5 commit 77e6bb6
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 77 deletions.
2 changes: 1 addition & 1 deletion apps/site/docs/zh/index.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Midscene.js 采用了多模态大语言模型（LLM），能够直观地“理

```typescript
// 👀 输入关键字，执行搜索
// 注：尽管这是一个英文页面，你也可以用中文指令控制它
// 尽管这是一个英文页面，你也可以用中文指令控制它
await ai('在搜索框输入 "Headphones" ，敲回车');

// 👀 找到列表里耳机相关的信息
Expand Down
62 changes: 41 additions & 21 deletions packages/web-integration/src/appium/page.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import fs from 'node:fs';
import type { Size } from '@midscene/core/.';
import type { Point, Size } from '@midscene/core/.';
import { getTmpFile } from '@midscene/core/utils';
import { base64Encoded, resizeImg } from '@midscene/shared/img';
import { DOMParser } from '@xmldom/xmldom';
Expand Down Expand Up @@ -104,62 +104,82 @@ export class Page implements AbstractPage {
}

// Scroll to top element
async scrollUntilTop(distance?: number): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;
async scrollUntilTop(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, -scrollDistance, 100);
await this.mouseWheel(0, -9999999, 100);
}

// Scroll to bottom element
async scrollUntilBottom(distance?: number): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;
async scrollUntilBottom(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, scrollDistance, 100);
await this.mouseWheel(0, 9999999, 100);
}

async scrollUntilLeft(distance?: number): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;
async scrollUntilLeft(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(-scrollDistance, 0, 100);
await this.mouseWheel(-9999999, 0, 100);
}

async scrollUntilRight(distance?: number): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;
async scrollUntilRight(startingPoint?: Point): Promise<void> {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(scrollDistance, 0, 100);
await this.mouseWheel(9999999, 0, 100);
}

// Scroll up one screen
async scrollUp(distance?: number): Promise<void> {
async scrollUp(distance?: number, startingPoint?: Point): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, -scrollDistance, 1000);
}

// Scroll down one screen
async scrollDown(distance?: number): Promise<void> {
async scrollDown(distance?: number, startingPoint?: Point): Promise<void> {
const { height } = await this.browser.getWindowSize();
const scrollDistance = distance || height * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(0, scrollDistance, 1000);
}

async scrollLeft(distance?: number): Promise<void> {
async scrollLeft(distance?: number, startingPoint?: Point): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(-scrollDistance, 0, 1000);
}

async scrollRight(distance?: number): Promise<void> {
async scrollRight(distance?: number, startingPoint?: Point): Promise<void> {
const { width } = await this.browser.getWindowSize();
const scrollDistance = distance || width * 0.7;

if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}

await this.mouseWheel(scrollDistance, 0, 1000);
}

Expand Down
69 changes: 53 additions & 16 deletions packages/web-integration/src/chrome-extension/page.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import fs from 'node:fs';
import type { WebKeyInput } from '@/common/page';
import type { ElementInfo } from '@/extractor';
import type { AbstractPage } from '@/page';
import type { Rect, Size } from '@midscene/core/.';
import type { Point, Rect, Size } from '@midscene/core/.';
import { ifInBrowser } from '@midscene/shared/utils';
import type { Protocol as CDPTypes } from 'devtools-protocol';

Expand Down Expand Up @@ -184,44 +184,76 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
return url || '';
}

async scrollUntilTop() {
async scrollUntilTop(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(0, -9999999);
}

async scrollUntilBottom() {
async scrollUntilBottom(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(0, 9999999);
}

async scrollUntilLeft() {
async scrollUntilLeft(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(-9999999, 0);
}

async scrollUntilRight() {
async scrollUntilRight(startingPoint?: Point) {
if (startingPoint) {
await this.mouse.move(startingPoint.left, startingPoint.top);
}
return this.mouse.wheel(9999999, 0);
}

async scrollUp(distance?: number) {
async scrollUp(distance?: number, startingPoint?: Point) {
const { height } = await this.size();
const scrollDistance = distance || height * 0.7;
return this.mouse.wheel(0, -scrollDistance);
return this.mouse.wheel(
0,
-scrollDistance,
startingPoint?.left,
startingPoint?.top,
);
}

async scrollDown(distance?: number) {
async scrollDown(distance?: number, startingPoint?: Point) {
const { height } = await this.size();
const scrollDistance = distance || height * 0.7;
return this.mouse.wheel(0, scrollDistance);
return this.mouse.wheel(
0,
scrollDistance,
startingPoint?.left,
startingPoint?.top,
);
}

async scrollLeft(distance?: number) {
async scrollLeft(distance?: number, startingPoint?: Point) {
const { width } = await this.size();
const scrollDistance = distance || width * 0.7;
return this.mouse.wheel(-scrollDistance, 0);
return this.mouse.wheel(
-scrollDistance,
0,
startingPoint?.left,
startingPoint?.top,
);
}

async scrollRight(distance?: number) {
async scrollRight(distance?: number, startingPoint?: Point) {
const { width } = await this.size();
const scrollDistance = distance || width * 0.7;
return this.mouse.wheel(scrollDistance, 0);
return this.mouse.wheel(
scrollDistance,
0,
startingPoint?.left,
startingPoint?.top,
);
}

async clearInput(element: ElementInfo) {
Expand Down Expand Up @@ -267,11 +299,16 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
clickCount: 1,
});
},
wheel: async (deltaX: number, deltaY: number) => {
wheel: async (
deltaX: number,
deltaY: number,
startX?: number,
startY?: number,
) => {
await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
type: 'mouseWheel',
x: 10,
y: 10,
x: startX || 10,
y: startY || 10,
deltaX,
deltaY,
});
Expand Down
42 changes: 28 additions & 14 deletions packages/web-integration/src/common/tasks.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,35 +314,49 @@ export class PageTaskExecutor {
thought: plan.thought,
locate: plan.locate,
executor: async (taskParam, { element }) => {
if (element) {
await this.page.mouse.move(
element.center[0],
element.center[1],
);
}
const startingPoint = element
? {
left: element.center[0],
top: element.center[1],
}
: undefined;
const scrollToEventName = taskParam?.scrollType;
if (scrollToEventName === 'untilTop') {
await this.page.scrollUntilTop();
await this.page.scrollUntilTop(startingPoint);
} else if (scrollToEventName === 'untilBottom') {
await this.page.scrollUntilBottom();
await this.page.scrollUntilBottom(startingPoint);
} else if (scrollToEventName === 'untilRight') {
await this.page.scrollUntilRight();
await this.page.scrollUntilRight(startingPoint);
} else if (scrollToEventName === 'untilLeft') {
await this.page.scrollUntilLeft();
await this.page.scrollUntilLeft(startingPoint);
} else if (scrollToEventName === 'once') {
if (taskParam.direction === 'down') {
await this.page.scrollDown(taskParam.distance || undefined);
await this.page.scrollDown(
taskParam.distance || undefined,
startingPoint,
);
} else if (taskParam.direction === 'up') {
await this.page.scrollUp(taskParam.distance || undefined);
await this.page.scrollUp(
taskParam.distance || undefined,
startingPoint,
);
} else if (taskParam.direction === 'left') {
await this.page.scrollLeft(taskParam.distance || undefined);
await this.page.scrollLeft(
taskParam.distance || undefined,
startingPoint,
);
} else if (taskParam.direction === 'right') {
await this.page.scrollRight(taskParam.distance || undefined);
await this.page.scrollRight(
taskParam.distance || undefined,
startingPoint,
);
} else {
throw new Error(
`Unknown scroll direction: ${taskParam.direction}`,
);
}
// until mouse event is done
await sleep(500);
} else {
throw new Error(
`Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
Expand Down
16 changes: 8 additions & 8 deletions packages/web-integration/src/page.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { Size } from '@midscene/core/.';
import type { Point, Size } from '@midscene/core/.';
import type { WebKeyInput } from './common/page';
import type { WebUIContext } from './common/utils';
import type { ElementInfo } from './extractor';
Expand Down Expand Up @@ -33,13 +33,13 @@ export abstract class AbstractPage {

async clearInput(element: ElementInfo): Promise<void> {}

abstract scrollUntilTop(): Promise<void>;
abstract scrollUntilBottom(): Promise<void>;
abstract scrollUntilLeft(): Promise<void>;
abstract scrollUntilRight(): Promise<void>;
abstract scrollUp(distance?: number): Promise<void>;
abstract scrollDown(distance?: number): Promise<void>;
abstract scrollLeft(distance?: number): Promise<void>;
abstract scrollUntilTop(startingPoint?: Point): Promise<void>;
abstract scrollUntilBottom(startingPoint?: Point): Promise<void>;
abstract scrollUntilLeft(startingPoint?: Point): Promise<void>;
abstract scrollUntilRight(startingPoint?: Point): Promise<void>;
abstract scrollUp(distance?: number, startingPoint?: Point): Promise<void>;
abstract scrollDown(distance?: number, startingPoint?: Point): Promise<void>;
abstract scrollLeft(distance?: number, startingPoint?: Point): Promise<void>;
abstract scrollRight(distance?: number): Promise<void>;

abstract _forceUsePageContext?(): Promise<WebUIContext>;
Expand Down
17 changes: 9 additions & 8 deletions packages/web-integration/src/playground/static-page.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
type WebUIContext,
} from '@/common/utils';
import type { AbstractPage } from '@/page';
import type { Point } from '@midscene/core/.';

const ThrowNotImplemented: any = (methodName: string) => {
throw new Error(
Expand Down Expand Up @@ -39,35 +40,35 @@ export default class StaticPage implements AbstractPage {
return this.uiContext.url;
}

async scrollUntilTop() {
async scrollUntilTop(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilTop');
}

async scrollUntilBottom() {
async scrollUntilBottom(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilBottom');
}

async scrollUntilLeft() {
async scrollUntilLeft(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilLeft');
}

async scrollUntilRight() {
async scrollUntilRight(startingPoint?: Point) {
return ThrowNotImplemented('scrollUntilRight');
}

async scrollUp(distance?: number) {
async scrollUp(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollUp');
}

async scrollDown(distance?: number) {
async scrollDown(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollDown');
}

async scrollLeft(distance?: number) {
async scrollLeft(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollLeft');
}

async scrollRight(distance?: number) {
async scrollRight(distance?: number, startingPoint?: Point) {
return ThrowNotImplemented('scrollRight');
}

Expand Down
Loading

0 comments on commit 77e6bb6

Please sign in to comment.