Merge branch 'main' into feat/allow-custom-page-description

.github/ISSUE_TEMPLATE/llm-connectivity-issue---模型链接错误.md

@@ -0,0 +1,21 @@
+---
+name: LLM Connectivity Issue / 模型链接错误
+about: How to solve the LLM connectivity problem
+title: "[Connectivity]"
+labels: ''
+assignees: ''
+
+---
+
+## Read this before open issue
+
+- Doc: how to choose and config a model: https://midscenejs.com/model-provider.html
+- Use this project to check the connection: https://github.com/web-infra-dev/midscene-example/tree/main/connectivity-test
+
+## If the error persists, tell us these information
+
+- Where are you using Midscene.js: Chrome extension, yaml with cli, Puppeteer,…
+- The version of Midscene.js
+- The model name and endpoint (if could be public）
+- The error message
+- Do NOT include your API key in your issue! Revoke it immediately if it is leaked in your issue.

README.md

@@ -48,6 +48,13 @@ Midscene.js is an AI-powered automation SDK can control the page, perform assert
 * [Integrate with Playwright](https://midscenejs.com/integrate-with-playwright.html)
 * [Customize Model and Provider](https://midscenejs.com/model-provider.html)
 
+## Community
+
+* [Lark Group](https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=291q2b25-e913-411a-8c51-191e59aab14d)
+
+
+  <img src="https://github.com/user-attachments/assets/7c132fbf-37a7-4005-8fb1-59342efdf9b2" alt="lark group link" width="300" />
+
 ## License
 
 Midscene.js is [MIT licensed](https://github.com/web-infra-dev/midscene/blob/main/LICENSE).

README.zh.md

@@ -45,6 +45,13 @@ Midscene.js 是一个由 AI 驱动的自动化 SDK，能够使用自然语言对
 * [集成到 Playwright](https://midscenejs.com/zh/integrate-with-playwright.html)
 * [自定义模型和服务商](https://midscenejs.com/zh/model-provider.html)
 
+## 社区
+
+* [飞书群](https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=291q2b25-e913-411a-8c51-191e59aab14d)
+
+  <img src="https://github.com/user-attachments/assets/7c132fbf-37a7-4005-8fb1-59342efdf9b2" alt="lark group link" width="300" />
+
+
 ## 授权许可
 
 Midscene.js 遵循 [MIT 许可协议](https://github.com/web-infra-dev/midscene/blob/main/LICENSE)。

apps/site/docs/en/integrate-with-puppeteer.mdx

@@ -7,6 +7,8 @@ import { PackageManagerTabs } from '@theme';
 
 :::info Demo Project
 you can check the demo project of Puppeteer here: [https://github.com/web-infra-dev/midscene-example/blob/main/puppeteer-demo](https://github.com/web-infra-dev/midscene-example/blob/main/puppeteer-demo)
+
+There is also a demo of Puppeteer with Vitest: [https://github.com/web-infra-dev/midscene-example/tree/main/puppeteer-with-vitest-demo](https://github.com/web-infra-dev/midscene-example/tree/main/puppeteer-with-vitest-demo)
 :::
 
 ## Preparation

apps/site/docs/en/model-provider.md

@@ -28,6 +28,9 @@ export MIDSCENE_OPENAI_INIT_CONFIG_JSON='{"baseURL":"....","defaultHeaders":{"ke
 
 # if you want to use proxy. Midscene uses `socks-proxy-agent` under the hood.
 export MIDSCENE_OPENAI_SOCKS_PROXY="socks5://127.0.0.1:1080"
+
+# if you want to specify the max tokens for the model
+export OPENAI_MAX_TOKENS=2048
 ```
 
 ## Using Azure OpenAI Service

apps/site/docs/en/quick-experience.mdx

@@ -43,4 +43,10 @@ After experiencing, you may want to write some code to integrate Midscene. There
 
 * Extension fails to run and shows 'Cannot access a chrome-extension:// URL of different extension'
 
-Make sure you are using the Midscene extension on a normal http(s):// page. If the error persists, it's mainly due to conflicts with other extensions injecting `<iframes />` into the page. Try disabling the suspicious plugins and refresh.
+It's mainly due to conflicts with other extensions injecting `<iframe />` or `<script />` into the page. Try disabling the suspicious plugins and refresh. 
+
+To find the suspicious plugins:
+
+1. Open the Devtools of the page, find the `<script>` or `<iframe>` with a url like `chrome-extension://{ID-of-the-suspicious-plugin}/...`.
+2. Copy the ID from the url, open chrome://extensions/, find the plugin with the same ID, disable it.
+3. Refresh the page, try again.

apps/site/docs/zh/integrate-with-puppeteer.mdx

@@ -7,6 +7,8 @@ import { PackageManagerTabs } from '@theme';
 
 :::info 样例项目
 你可以在这里看到向 Puppeteer 集成的样例项目：[https://github.com/web-infra-dev/midscene-example/blob/main/puppeteer-demo](https://github.com/web-infra-dev/midscene-example/blob/main/puppeteer-demo)
+
+这里还有一个 Puppeteer 和 Vitest 结合的样例项目：[https://github.com/web-infra-dev/midscene-example/tree/main/puppeteer-with-vitest-demo](https://github.com/web-infra-dev/midscene-example/tree/main/puppeteer-with-vitest-demo)
 :::
 
 ## 准备工作

apps/site/docs/zh/model-provider.md

@@ -25,6 +25,9 @@ export MIDSCENE_OPENAI_INIT_CONFIG_JSON='{"baseURL":"....","defaultHeaders":{"ke
 
 # 可选, 如果你想使用代理。Midscene 使用 `socks-proxy-agent` 作为底层库。
 export MIDSCENE_OPENAI_SOCKS_PROXY="socks5://127.0.0.1:1080"
+
+# 可选, 如果你想指定模型 max_tokens
+export OPENAI_MAX_TOKENS=2048
 ```
 
 ## 使用 Azure OpenAI 服务时的配置

apps/site/docs/zh/quick-experience.mdx

@@ -41,4 +41,10 @@ OPENAI_API_KEY="sk-replace-by-your-own"
 
 * 插件运行失败，提示 'Cannot access a chrome-extension:// URL of different extension'
 
-请确保你在普通的 http(s):// 页面上使用 Midscene.js 插件。如果依然报错，一般是与其他插件冲突（被注入了 `<iframe />` ）所致，临时禁用可疑插件即可。
+这一般是与其他插件冲突所致，如页面已经被其他插件注入了 `<iframe />` 或 `<script />`。
+
+找到可疑插件：
+
+1. 打开页面的调试器，找到被其他插件注入的 `<iframe />` 或 `<script />`，一般 URL 是 `chrome-extension://{这串就是ID}/...` 格式，复制其 ID。
+2. 打开 chrome://extensions/ ，找到相同 ID 的插件，禁用它。
+3. 刷新页面，再次尝试。

apps/site/rspress.config.ts

@@ -19,6 +19,12 @@ export default defineConfig({
         mode: 'link',
         content: 'https://github.com/web-infra-dev/midscene',
       },
+      {
+        icon: 'lark',
+        mode: 'link',
+        content:
+          'https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=291q2b25-e913-411a-8c51-191e59aab14d',
+      },
     ],
     footer: {
       message: `

package.json

@@ -1,7 +1,7 @@
 {
   "name": "midscene",
   "private": true,
-  "version": "0.8.8",
+  "version": "0.8.12",
   "scripts": {
     "build": "nx run-many --target=build --projects=@midscene/core,@midscene/shared,@midscene/visualizer,@midscene/web,@midscene/cli --verbose",
     "test": "nx run-many --target=test --projects=@midscene/core,--projects=@midscene/shared,@midscene/visualizer,@midscene/web,@midscene/cli --verbose",

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/cli",
   "description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
-  "version": "0.8.8",
+  "version": "0.8.12",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -23,8 +23,8 @@
   "dependencies": {
     "@midscene/core": "workspace:*",
     "@midscene/web": "workspace:*",
-    "http-server": "14.1.1",
-    "puppeteer": "23.0.2"
+    "puppeteer": "23.0.2",
+    "http-server": "14.1.1"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",
@@ -44,7 +44,7 @@
     "yargs": "17.7.2"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "publishConfig": {
     "access": "public",

packages/cli/src/index.ts

@@ -1,6 +1,6 @@
 import 'dotenv/config';
 import { matchYamlFiles, parseProcessArgs } from './cli-utils';
-import { playYamlFiles } from './yaml-player';
+import { playYamlFiles } from './yaml-runner';
 
 Promise.resolve(
   (async () => {

packages/cli/src/printer.ts

@@ -1,16 +1,15 @@
 import { basename, dirname, relative } from 'node:path';
-import chalk from 'chalk';
 import type {
-  MidsceneYamlFileContext,
-  MidsceneYamlFlowItem,
-  MidsceneYamlFlowItemAIAction,
-  MidsceneYamlFlowItemAIAssert,
-  MidsceneYamlFlowItemAIQuery,
-  MidsceneYamlFlowItemAIWaitFor,
-  MidsceneYamlFlowItemSleep,
   ScriptPlayerStatusValue,
   ScriptPlayerTaskStatus,
-} from './types';
+} from '@midscene/core';
+import { type ScriptPlayer, flowItemBrief } from '@midscene/web/yaml';
+import chalk from 'chalk';
+
+export interface MidsceneYamlFileContext {
+  file: string;
+  player: ScriptPlayer;
+}
 
 export const isTTY = process.env.MIDSCENE_CLI_LOG_ON_NON_TTY
   ? false
@@ -24,51 +23,8 @@ export const currentSpinningFrame = () => {
   ];
 };
 
-export const flowItemBrief = (flowItem?: MidsceneYamlFlowItem) => {
-  if (!flowItem) {
-    return '';
-  }
-
-  const sliceText = (text?: string) => {
-    const lengthLimit = 60;
-    if (text && text.length > lengthLimit) {
-      return `${text.slice(0, lengthLimit)}...`;
-    }
-
-    return text || '';
-  };
-
-  if (
-    (flowItem as MidsceneYamlFlowItemAIAction).aiAction ||
-    (flowItem as MidsceneYamlFlowItemAIAction).ai
-  ) {
-    return `aiAction: ${sliceText(
-      (flowItem as MidsceneYamlFlowItemAIAction).aiActionProgressTip ||
-        (flowItem as MidsceneYamlFlowItemAIAction).aiAction ||
-        (flowItem as MidsceneYamlFlowItemAIAction).ai,
-    )}`;
-  }
-  if ((flowItem as MidsceneYamlFlowItemAIAssert).aiAssert) {
-    return `aiAssert: ${sliceText(
-      (flowItem as MidsceneYamlFlowItemAIAssert).aiAssert,
-    )}`;
-  }
-  if ((flowItem as MidsceneYamlFlowItemAIQuery).aiQuery) {
-    return `aiQuery: ${sliceText((flowItem as MidsceneYamlFlowItemAIQuery).aiQuery)}`;
-  }
-  if ((flowItem as MidsceneYamlFlowItemAIWaitFor).aiWaitFor) {
-    return `aiWaitFor: ${sliceText(
-      (flowItem as MidsceneYamlFlowItemAIWaitFor).aiWaitFor,
-    )}`;
-  }
-  if ((flowItem as MidsceneYamlFlowItemSleep).sleep) {
-    return `sleep: ${(flowItem as MidsceneYamlFlowItemSleep).sleep}`;
-  }
-  return '';
-};
-
 // status: init / running / done / error
-function textForStatus(status: ScriptPlayerStatusValue) {
+function indicatorForStatus(status: ScriptPlayerStatusValue) {
   if (status === 'init') {
     return chalk.gray('◌');
   }
@@ -88,7 +44,7 @@ export const contextInfo = (context: MidsceneYamlFileContext) => {
   const fileName = basename(filePath);
   const fileDir = dirname(filePath);
   const fileNameToPrint = `${chalk.gray(`${fileDir}/`)}${fileName}`;
-  const fileStatusText = textForStatus(context.player.status);
+  const fileStatusText = indicatorForStatus(context.player.status);
   const contextActionText =
     typeof context.player.currentTaskIndex === 'undefined' &&
     context.player.status === 'running'
@@ -146,7 +102,7 @@ export const singleTaskInfo = (task: ScriptPlayerTaskStatus) => {
       ? `\n${indent}${chalk.gray('error:')}\n${indent}${indent}${task.error?.message}`
       : '';
 
-  const statusText = textForStatus(task.status);
+  const statusText = indicatorForStatus(task.status);
   const mergedLine = `${statusText} ${task.name} ${stepText}${errorText}`;
   return {
     nameText: task.name,
@@ -164,14 +120,14 @@ function paddingLines(lines: string[]) {
 }
 
 export const contextTaskListSummary = (
-  taskStatus: ScriptPlayerTaskStatus[],
+  taskStatusArray: ScriptPlayerTaskStatus[],
   context: MidsceneYamlFileContext,
 ) => {
   const prefixLines: string[] = [];
   const currentLine: string[] = [];
   const suffixText: string[] = [];
   const { mergedText: fileInfo } = contextInfo(context);
-  for (const task of taskStatus) {
+  for (const task of taskStatusArray) {
     const { mergedLine } = singleTaskInfo(task);
 
     if (context.player.status === 'init') {