feat: implement the 'sleep' action (#41)

packages/midscene/src/automation/planning.ts

@@ -26,11 +26,14 @@ export function systemPromptToTaskPlanning(query: string) {
     * param: { scrollType: 'ScrollUntilBottom', 'ScrollUntilTop', 'ScrollDown', 'ScrollUp' }
   * type: 'Error'
     * param: { message: string }, the error message
+  * type: 'Sleep'
+    * param: { timeMs: number }, wait for timeMs milliseconds 
   
   Here is an example of how to decompose a task.
-  When a user says 'Input "Weather in Shanghai" into the search bar, hit enter', by viewing the page screenshot and description, you my decompose this task into something like this:
+  When a user says 'Input "Weather in Shanghai" into the search bar, wait 1 second, hit enter', by viewing the page screenshot and description, you my decompose this task into something like this:
   * Find: 'The search bar'
   * Input: 'Weather in Shanghai'
+  * Sleep: 1000
   * KeyboardPress: 'Enter'
   
   Remember: 

packages/midscene/src/types.ts

@@ -176,7 +176,8 @@ export interface PlanningAction<ParamType = any> {
     | 'KeyboardPress'
     | 'Scroll'
     | 'Error'
-    | 'Assert';
+    | 'Assert'
+    | 'Sleep';
   param: ParamType;
 }
 
@@ -203,6 +204,10 @@ export interface PlanningActionParamAssert {
   assertion: string;
 }
 
+export interface PlanningActionParamSleep {
+  timeMs: number;
+}
+
 /**
  * misc
  */

packages/midscene/tests/automation/__snapshots__/planning.test.ts.snap

@@ -0,0 +1,13 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`automation - planning > basic run 1`] = `
+{
+  "timeMs": 3500,
+}
+`;
+
+exports[`automation - planning > basic run 2`] = `
+{
+  "value": "Enter",
+}
+`;

packages/midscene/tests/automation/planning.test.ts

@@ -13,16 +13,19 @@ describe('automation - planning', () => {
     const { context } = await getPageDataOfTestName('todo');
 
     const { plans } = await plan(
-      'type "Why is the earth a sphere?", hit Enter',
+      'type "Why is the earth a sphere?", wait 3.5s, hit Enter',
       {
         context,
       },
     );
-    expect(plans.length).toBe(3);
+    expect(plans.length).toBe(4);
     expect(plans[0].thought).toBeTruthy();
     expect(plans[0].type).toBe('Locate');
     expect(plans[1].type).toBe('Input');
-    expect(plans[2].type).toBe('KeyboardPress');
+    expect(plans[2].type).toBe('Sleep');
+    expect(plans[2].param).toMatchSnapshot();
+    expect(plans[3].type).toBe('KeyboardPress');
+    expect(plans[3].param).toMatchSnapshot();
   });
 
   it('should raise an error when prompt is irrelevant with page', async () => {

packages/web-integration/src/common/tasks.ts

@@ -20,6 +20,7 @@ import Insight, {
   type PlanningActionParamHover,
   type PlanningActionParamInputOrKeyPress,
   type PlanningActionParamScroll,
+  type PlanningActionParamSleep,
   type PlanningActionParamTap,
 } from '@midscene/core';
 import { base64Encoded } from '@midscene/core/image';
@@ -191,7 +192,6 @@ export class PageTaskExecutor {
             };
           return taskActionInput;
         }
-
         if (plan.type === 'KeyboardPress') {
           const taskActionKeyboardPress: ExecutionTaskActionApply<PlanningActionParamInputOrKeyPress> =
             {
@@ -271,6 +271,19 @@ export class PageTaskExecutor {
             };
           return taskActionScroll;
         }
+        if (plan.type === 'Sleep') {
+          const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =
+            {
+              type: 'Action',
+              subType: 'Sleep',
+              param: plan.param,
+              executor: async (taskParam) => {
+                assert(taskParam.timeMs, 'No time to sleep');
+                await sleep(taskParam.timeMs);
+              },
+            };
+          return taskActionSleep;
+        }
         if (plan.type === 'Error') {
           throw new Error(`Got a task plan with type Error: ${plan.thought}`);
         }

packages/web-integration/tests/puppeteer/showcase.test.ts

@@ -13,8 +13,7 @@ describe('puppeteer integration', () => {
     const mid = new PuppeteerAgent(page);
 
     // perform a search
-    await mid.aiAction('type "Headphones" in search box, hit Enter');
-    await sleep(5000);
+    await mid.aiAction('type "Headphones" in search box, hit Enter, wait 5s');
 
     // find the items
     const items = await mid.aiQuery(