From 0311d8e874ade1e3bc71fdee8180001a35c54044 Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Thu, 20 Nov 2025 20:47:03 +0800 Subject: [PATCH 1/9] fix rendering tables --- ms_agent/app/fin_research.py | 106 +++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 12 deletions(-) diff --git a/ms_agent/app/fin_research.py b/ms_agent/app/fin_research.py index e89529284..64da11fe4 100644 --- a/ms_agent/app/fin_research.py +++ b/ms_agent/app/fin_research.py @@ -538,6 +538,93 @@ def replace_image(match): return re.sub(pattern, replace_image, markdown_content) +MARKDOWN_TABLE_STYLE_BLOCK = """ +.markdown-html-content .fin-table-wrapper { + margin: 1.5rem 0; + border: 1px solid rgba(15, 23, 42, 0.12); + border-radius: 18px; + overflow-x: auto; + background: #ffffff; + box-shadow: 0 12px 30px rgba(15, 23, 42, 0.08); +} +.markdown-html-content .fin-table-wrapper table { + width: 100%; + border-collapse: collapse; + min-width: 640px; +} +.markdown-html-content .fin-table-wrapper table th, +.markdown-html-content .fin-table-wrapper table td { + border: 1px solid rgba(15, 23, 42, 0.12); + padding: 12px 16px; + text-align: left; + font-size: 0.95rem; +} +.markdown-html-content .fin-table-wrapper table thead { + background: rgba(59, 130, 246, 0.08); + font-weight: 600; + color: #0f172a; +} +.markdown-html-content .fin-table-wrapper table tbody tr:nth-child(even) { + background: rgba(15, 23, 42, 0.03); +} +.markdown-html-content .fin-table-wrapper table caption { + caption-side: bottom; + padding: 0.75rem 0.5rem 0; + color: #475569; + font-size: 0.9rem; +} +.markdown-html-content .fin-table-wrapper table code { + background: rgba(99, 102, 241, 0.12); + color: #4c1d95; + padding: 2px 6px; + border-radius: 6px; +} +@media (max-width: 640px) { + .markdown-html-content .fin-table-wrapper table { + min-width: 520px; + } +} +.dark .markdown-html-content .fin-table-wrapper { + background: #0f172a; + border-color: #1e293b; + box-shadow: 0 12px 30px rgba(2, 6, 23, 0.45); +} +.dark .markdown-html-content .fin-table-wrapper table th, +.dark .markdown-html-content .fin-table-wrapper table td { + border-color: rgba(148, 163, 184, 0.35); + color: #e2e8f0; +} +.dark .markdown-html-content .fin-table-wrapper table thead { + background: rgba(59, 130, 246, 0.25); + color: #f1f5f9; +} +.dark .markdown-html-content .fin-table-wrapper table tbody tr:nth-child(even) { + background: rgba(15, 23, 42, 0.8); +} +.dark .markdown-html-content .fin-table-wrapper table caption { + color: #cbd5f5; +} +.dark .markdown-html-content .fin-table-wrapper table code { + background: #020617 !important; + color: #fef9c3 !important; +} +""" + +_TABLE_WRAPPER_PATTERN = re.compile(r'()', + flags=re.IGNORECASE | re.DOTALL) + + +def _wrap_tables_with_container(html_content: str) -> str: + """Ensure markdown tables are wrapped for styling/scrolling.""" + def _inject_wrapper(match: re.Match) -> str: + table_html = match.group(1) + if 'fin-table-wrapper' in table_html: + return table_html + return f'
{table_html}
' + + return _TABLE_WRAPPER_PATTERN.sub(_inject_wrapper, html_content) + + def _render_markdown_html_core(markdown_content: str, add_permalink: bool = True) -> Tuple[str, str]: latex_placeholders = {} @@ -580,6 +667,7 @@ def protect_latex(match): html_content = md.convert(protected_content) for placeholder, latex_formula in latex_placeholders.items(): html_content = html_content.replace(placeholder, latex_formula) + html_content = _wrap_tables_with_container(html_content) container_id = f'katex-content-{int(time.time() * 1_000_000)}' return html_content, container_id @@ -591,6 +679,9 @@ def _build_inline_markdown_html(html_content: str, container_id: str) -> str: href="https://cdn.jsdelivr.net/npm/katex@0.16.9/dist/katex.min.css" integrity="sha384-n8MVd4RsNIU0tAv4ct0nTaAbDJwPJzDEaqSD1odI+WdtXRGWt2kTvGFasHpSy3SV" crossorigin="anonymous"> +
{html_content}
@@ -714,18 +805,9 @@ def build_exportable_report_html(markdown_content: str, border-radius: 16px; box-shadow: 0 20px 40px rgba(15, 23, 42, 0.12); } - .markdown-html-content table { - width: 100%; - border-collapse: collapse; - margin: 1.5rem 0; - font-size: 0.95rem; - } - .markdown-html-content table th, - .markdown-html-content table td { - border: 1px solid rgba(15, 23, 42, 0.15); - padding: 12px 16px; - text-align: left; - } + """ + base_css += MARKDOWN_TABLE_STYLE_BLOCK + base_css += """ .markdown-html-content blockquote { border-left: 4px solid #6366f1; padding: 0.5rem 1.5rem; From 6f07584adef6a51073e98c44fd61e3c49a6af291 Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Mon, 24 Nov 2025 17:12:55 +0800 Subject: [PATCH 2/9] reduce redundant tool call and support gpt5 --- projects/fin_research/aggregator.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/projects/fin_research/aggregator.yaml b/projects/fin_research/aggregator.yaml index dffb0ef38..b7624ff84 100644 --- a/projects/fin_research/aggregator.yaml +++ b/projects/fin_research/aggregator.yaml @@ -12,7 +12,7 @@ generation_config: prompt: system: | - You are an intelligent financial analysis agent. + You are an intelligent financial analysis agent. You solve financial analysis tasks through systematic tool usage and step-by-step reasoning. Your task is to generate a comprehensive financial report by integrating insights from two separate sources: - Financial Data Analysis Report — derived from collected financial metrics, datasets, and quantitative analyses. - Online Sentiment Analysis Report — derived from web-based data, including news, social media, \ @@ -21,8 +21,12 @@ prompt: + Follow the multi-phase workflow defined below. + After completing a phase, you MUST NOT summarize, pause, or ask whether to continue without tool calls. + Immediately and automatically start the next phase. + Phase1: Synthesize Findings and Create Report Outline: - You MUST begin each message in this phase with `[ACT=outline]: ` on the first line to indicate intent. + **You MUST begin each message in this phase with `[ACT=outline]: ` on the first line to indicate intent.** - Synthesize important findings: - Carefully read and interpret the user's original plan for the financial analysis task \ (e.g., focus on industry trends, company performance, investment risks, etc.). @@ -38,11 +42,9 @@ prompt: - You are encouraged to retrieve one or more principles as the foundation for constructing \ the report outline, ensuring the final report's professionalism and refinement. - Output the report outline to a file in the default working directory. - - Avoid redundant tool calls for file queries or reads when the relevant information has \ - already been loaded or provided in the current context. Phase2: Generate the Final Report Chapter by Chapter: - You MUST begin each message in this phase with `[ACT=partial_report]: ` on the first line to indicate intent. + **You MUST begin each message in this phase with `[ACT=partial_report]: ` on the first line to indicate intent.** - Generate the report chapter by chapter according to the outline, for each chapter: - Write the full content for that chapter only and persist it to `.md`. - From Chapter 2 onward, append a **“Mismatch with Prior Chapters”** section listing any inconsistencies \ @@ -52,7 +54,7 @@ prompt: - Save the consolidated results to `cross_chapter_mismatches.md` in the default working directory. Phase3: Consolidate and Finalize the Report: - You MUST begin each message in this phase with `[ACT=final_report]: ` on the first line to indicate intent. + **You MUST begin each message in this phase with `[ACT=final_report]: ` on the first line to indicate intent.** - Integrate all chapters into a coherent final report: - Review the report outline, all chapter files, `cross_chapter_mismatches.md`, and user's original plan for the financial analysis task. - Summarize and resolve all mismatch issues, ensuring consistency in data, scope, and terminology. @@ -68,6 +70,8 @@ prompt: - Retrieve principles depending on the principle_skill server. - To reduce the number of conversation turns, multiple tools can be invoked simultaneously within a single turn when necessary. - When using the file_system---write_file tool, pass an empty string ('') to the path argument to use the default working directory. + - Avoid redundant tool calls for file queries or reads when the relevant information has \ + already been loaded or provided in the current context. From 91ed3218f31965866de7431ef76f22c0aa9f9325 Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Tue, 25 Nov 2025 01:26:39 +0800 Subject: [PATCH 3/9] update code tool readme and update readme in fin_research --- docs/en/Components/Tools.md | 66 +++++++++++++++++++ docs/en/Projects/FinResearch.md | 31 +++++++++ .../Components/\345\267\245\345\205\267.md" | 64 +++++++++++++----- ...61\345\272\246\347\240\224\347\251\266.md" | 29 ++++++++ projects/fin_research/README.md | 33 +++++++++- projects/fin_research/README_zh.md | 31 ++++++++- 6 files changed, 235 insertions(+), 19 deletions(-) diff --git a/docs/en/Components/Tools.md b/docs/en/Components/Tools.md index c97464d4e..df753bb93 100644 --- a/docs/en/Components/Tools.md +++ b/docs/en/Components/Tools.md @@ -53,6 +53,72 @@ Parameters: - path: `str`, relative directory based on the `output` in yaml configuration. If empty, lists all files in the root directory. +### code_executor + +Code execution tool that can run Python code either in a sandboxed environment or directly in the local Python environment. The behavior is controlled by the `tools.code_executor.implementation` field. + +- When omitted or set to `sandbox`: + - Uses an `ms-enclave` based sandbox. The sandbox can be created locally with Docker or via a remote HTTP service. + - Currently supports two sandbox types: `docker` and `docker_notebook`. The former is suitable for non-interactive/stateless execution; the latter maintains notebook-style state across calls. + - The configured `output_dir` on the host is mounted into the sandbox at `/data` so code can read and write persistent artifacts there. + +- When set to `python_env`: + - Runs code in the local Python environment. The tool API is aligned with the sandbox version and supports both Jupyter-kernel based execution and plain Python interpreter execution. + - Required dependencies should be installed locally; on the first run, common data-analysis and execution dependencies (such as `numpy`, `pandas`, etc.) will be installed automatically when missing. + +#### notebook_executor + +- **Sandbox mode**: Executes code inside a `docker_notebook` sandbox, preserving state (variables, imports, dataframes, etc.) across calls. Files under the mounted data directory are available at `/data/...`, and you can also run simple shell commands from code cells using the standard `!` prefix. +- **Local mode**: Executes code in a local Jupyter kernel, with environment isolation and state persistence across calls. In the notebook environment you can also use simple shell commands via the standard `!` syntax. + +**Parameters**: + +- **code**: `string` – Python code to execute. +- **description**: `string` – Short description of what the code is doing. +- **timeout**: `integer` – Optional execution timeout in seconds; if omitted, the tool-level default is used. + +#### python_executor + +- **Sandbox mode**: Executes Python code in a `docker`-type sandbox using the sandbox’s Python interpreter, typically used when you do not need full notebook-style interaction. +- **Local mode**: Executes code with the local Python interpreter in a stateless fashion; each call has its own execution context and does not share variables with previous calls. + +**Parameters**: + +- **code**: `string` – Python code to execute. +- **description**: `string` – Short description of what the code is doing. +- **timeout**: `integer` – Optional execution timeout in seconds; if omitted, the tool-level default is used. + +#### shell_executor + +- **Sandbox mode**: Dedicated to `docker`-type sandboxes and executes shell commands inside the sandbox using `bash`, supporting basic operations like `ls`, `cd`, `mkdir`, `rm`, etc., and access to files under `/data`. +- **Local mode**: Executes shell commands using the local `bash` interpreter with the working directory set to `output_dir`; this is convenient for development but generally not recommended for production. + +**Parameters**: + +- **command**: `string` – Shell command to execute. +- **timeout**: `integer` – Optional execution timeout in seconds; if omitted, the tool-level default is used. + +#### file_operation + +- **Sandbox mode**: Dedicated to `docker`-type sandboxes and performs basic file operations inside the sandbox (create, read, write, delete, list, exists). Paths are interpreted as sandbox-internal paths; in most cases you should work under `/data/...`. +- **Local mode**: Performs the same basic file operations on the local filesystem but always constrained under `output_dir` to prevent accessing arbitrary locations. + +**Parameters**: + +- **operation**: `string` – Type of file operation to perform; one of `'create'`, `'read'`, `'write'`, `'delete'`, `'list'`, `'exists'`. +- **file_path**: `string` – File or directory path (sandbox-internal in sandbox mode; relative to or under `output_dir` in local mode). +- **content**: `string` – Optional, content to write when `operation` is `'write'`. +- **encoding**: `string` – Optional file encoding, default `utf-8`. + +#### reset_executor + +- **Sandbox mode**: Recreates the sandbox (or restarts the notebook kernel) to clear all variables and session state when the environment becomes unstable. +- **Local mode**: Restarts the local Jupyter kernel used by `notebook_executor`, dropping all in-memory state. + +#### get_executor_info + +- **Sandbox mode**: Returns the current sandbox status and configuration summary (such as memory/CPU limits, available tools, etc.). +- **Local mode**: Returns basic information about the local execution environment (working directory, whether it is initialized, current execution count, uptime, etc.). ### MCP Tools diff --git a/docs/en/Projects/FinResearch.md b/docs/en/Projects/FinResearch.md index 86bbe8502..96f968721 100644 --- a/docs/en/Projects/FinResearch.md +++ b/docs/en/Projects/FinResearch.md @@ -73,11 +73,42 @@ pip install akshare baostock ### Sandbox Environment +By default, the Collector and Analyst agents use a Docker-based sandbox to safely execute code: + ```bash pip install ms-enclave docker websocket-client # https://github.com/modelscope/ms-enclave bash projects/fin_research/tools/build_jupyter_image.sh ``` +If you prefer not to install Docker and related dependencies, you can configure a local code execution tool instead. In both `analyst.yaml` and `collector.yaml`, change the default `tools` configuration to: + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - python_executor + - shell_executor + - file_operation +``` + +With this configuration, code is executed via a Jupyter kernel–based notebook executor that isolates environment variables and supports shell command execution; the necessary dependencies (for data analysis and code execution) will be installed automatically on the first run. + +If you only need a lighter-weight Python execution environment and do not want to introduce notebook-related dependencies, you can instead use: + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - notebook_executor + - file_operation +``` + +This configuration uses an independent Python executor together with a shell command executor and is suitable for lightweight code execution scenarios. + ### Environment Variables Configure API keys in your system environment or in YAML. diff --git "a/docs/zh/Components/\345\267\245\345\205\267.md" "b/docs/zh/Components/\345\267\245\345\205\267.md" index e86be0307..032c3faf8 100644 --- "a/docs/zh/Components/\345\267\245\345\205\267.md" +++ "b/docs/zh/Components/\345\267\245\345\205\267.md" @@ -53,44 +53,78 @@ MS-Agent支持很多内部工具： - path: `str`, 基于yaml配置中的`output`的相对目录。如果为空，则列出根目录下的所有文件。 -### code_execution +### code_executor -代码执行工具，基于沙箱环境运行代码，支持基于HTTP或本地建立沙箱运行环境，主要支持docker和docker-notebook两种环境类型，分别适合于无状态的代码运行和需要在对话内保持上下文状态的代码运行。 -工具基于ms-enclave实现，依赖于本地的docker环境，如果代码执行需要的依赖较多，需要预先构建包含所需依赖的镜像。准备好基础镜像后，需要完善本次启动容器的基础配置，如配置所选择的执行环境类型、可用的工具、容器需要挂载的目录等等，默认的挂载目录基于yaml配置中的`output`字段，在沙箱内挂载为`/data`。 +代码执行工具，支持基于本地 Python 环境执行代码或基于沙箱环境执行代码。可以在 `tools.code_executor` 下通过配置 `implementation` 字段选择执行环境。 + +- 默认或 `implementation: sandbox` 启动沙箱模式： + - 基于沙箱环境运行代码，支持通过本地 Docker 或远程 HTTP 服务建立沙箱运行环境，主要支持 `docker` 和 `docker_notebook` 两种环境类型，分别适合于无状态的代码运行和需要在对话内保持上下文状态的代码运行。 + - 工具基于 `ms-enclave` 实现，依赖于本地可用的 Docker 环境。如果代码执行需要的依赖较多，建议预先构建包含所需依赖的镜像。准备好基础镜像后，需要完善本次启动容器的基础配置，如配置所选择的执行环境类型、可用的工具、容器需要挂载的目录等。 + - 默认会将配置中的 `output_dir` 目录挂载到沙箱内的 `/data`，用于读写持久化文件。 + +- `implementation: python_env` 时启动本地模式： + - 基于本地 Python 环境执行代码，在工具设计上与沙箱环境保持一致，支持 Jupyter Notebook 和 Python 解释器两种执行方式，分别适合于需要在对话内保持上下文状态的代码运行和无状态的代码运行。 + - 所需的依赖需要在本地进行配置，第一次运行时会自动安装常用的数据分析和代码执行基础依赖（例如 `numpy`、`pandas` 等）。 #### notebook_executor -该方法专用于docker-notebook类型沙箱环境，可以在notebook内执行代码并保持对话内的上下文，同时支持使用shell命令完成沙箱环境下的各种操作。 +沙箱模式下，该方法对应 `docker_notebook` 类型沙箱环境，可以在 Notebook 内执行代码并保持对话内的上下文；代码可以访问挂载在 `/data` 下的文件，支持在代码中通过`!`前缀执行简单的shell命令。 + +本地模式下，该方法基于本地 Jupyter Kernel 执行代码，提供对环境变量的隔离，支持对话内的上下文保持，并支持在代码中通过 `!` 前缀执行简单的 shell 命令。相应依赖会在首次运行时自动安装（包括数据分析和代码执行需要的依赖）。 + +参数： - code: `str`, 需要执行的代码。 -- description: `str`, 代码工作内容的描述。 +- description: `str`, 代码工作内容的简要描述。 +- timeout: `int`, 可选，执行超时时间（秒），不配置时使用工具默认值。 #### python_executor -该方法专用于docker类型沙箱环境，可以使用沙箱内的本地代码解释器运行代码。 +沙箱模式下，该方法专用于 `docker` 类型沙箱环境，可以使用沙箱内的 Python 解释器运行代码，适合不需要完整 Notebook 交互能力的执行场景。 + +本地模式下，该方法基于本地 Python 解释器执行代码，每次调用互相独立，不保留上下文。 + +参数： - code: `str`, 需要执行的代码。 -- description: `str`, 代码工作内容的描述。 +- description: `str`, 代码工作内容的简要描述。 +- timeout: `int`, 可选，执行超时时间（秒），不配置时使用工具默认值。 #### shell_executor -该方法专用于docker类型沙箱环境，可以使用bash在沙箱内执行shell命令，支持基本的shell操作例如ls、cd、mkdir、rm等等。 +沙箱模式下，该方法专用于 `docker` 类型沙箱环境，该方法使用 bash 在沙箱内执行 shell 命令，支持基本的 shell 操作例如 `ls`、`cd`、`mkdir`、`rm` 等，并可以访问 `/data` 目录下的文件。 -- command: `str`, 需要执行的shell命令。 +本地模式下，该方法使用本地的 bash 解释器执行 shell 命令，工作目录为配置中的 `output_dir`，支持基本的 shell 操作，但不建议在生产环境中使用。 + +参数： + +- command: `str`, 需要执行的 shell 命令。 +- timeout: `int`, 可选，执行超时时间（秒），不配置时使用工具默认值。 #### file_operation -该方法专用于docker类型沙箱环境，可以在沙箱内执行基本的文件操作，包括创建、读、写、删除、列出、判断是否存在等。 +沙箱模式下，该方法专用于 `docker` 类型沙箱环境，用于在沙箱内执行基本的文件操作，包括创建、读、写、删除、列出、判断是否存在等。文件路径基于沙箱内部路径，通常建议在挂载目录 `/data` 下进行读写。 + +本地模式下，该方法用于直接对本地文件系统进行基础操作，但所有路径都会被约束在 `output_dir` 目录内部，以避免越权访问。 + +参数： + +- operation: `str`, 要执行的文件操作类型，可选值为 `'create'`、`'read'`、`'write'`、`'delete'`、`'list'`、`'exists'`。 +- file_path: `str`, 要操作的文件或目录路径（沙箱模式下为容器内路径，本地模式下通常为相对于 `output_dir` 的路径，也可以传入受限的绝对路径）。 +- content: `str`, 可选，仅在 `write` 操作时需要，写入文件的内容。 +- encoding: `str`, 可选，文件编码，默认为 `utf-8`。 + +#### reset_executor -- operation: `str`, 要执行的文件操作类型，可选值为'create'、'read'、'write'、'delete'、'list'、'exists'。 +沙箱模式下，用于在沙箱环境崩溃或 Notebook 内变量状态混乱时重启沙箱环境或重建内核，清空所有状态。 -#### reset_sandbox +本地模式下，专用于在 Notebook 执行出现问题时对本地 Jupyter Kernel 进行重启，清空当前会话的所有状态。 -用于在沙箱环境崩溃时重启沙箱环境，例如notebook内变量状态混乱时重置所有状态。 +#### get_executor_info -#### get_sandbox_info +沙箱模式下，获取当前沙箱状态与环境的基础信息，例如沙箱 ID、运行状态、资源限制配置和可用工具列表等。 -获取当前沙箱状态与环境信息。 +本地模式下，用于获取当前本地 Python 执行环境的基本信息，例如工作目录、是否已初始化、当前执行次数以及运行时长等。 ### MCP工具 diff --git "a/docs/zh/Projects/\351\207\221\350\236\215\346\267\261\345\272\246\347\240\224\347\251\266.md" "b/docs/zh/Projects/\351\207\221\350\236\215\346\267\261\345\272\246\347\240\224\347\251\266.md" index 6555fa242..ce5433392 100644 --- "a/docs/zh/Projects/\351\207\221\350\236\215\346\267\261\345\272\246\347\240\224\347\251\266.md" +++ "b/docs/zh/Projects/\351\207\221\350\236\215\346\267\261\345\272\246\347\240\224\347\251\266.md" @@ -73,6 +73,8 @@ pip install akshare baostock ### 沙箱环境 +Collector 与 Analyst 默认使用 Docker 沙箱以安全执行代码（可选）： + ```bash # 安装 ms-enclave（https://github.com/modelscope/ms-enclave） pip install ms-enclave docker websocket-client @@ -81,6 +83,33 @@ pip install ms-enclave docker websocket-client bash projects/fin_research/tools/build_jupyter_image.sh ``` +如果不希望安装 Docker 等依赖，也可以选择配置本地代码执行工具，推荐将 `analyst.yaml` 和 `collector.yaml` 中默认的 `tools` 配置修改为： + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - python_executor + - shell_executor + - file_operation +``` + +该配置下默认依赖 Jupyter Kernel 执行代码，提供对环境变量的隔离，并支持 shell 命令执行，相应的依赖将在第一次运行代码时自动安装（包括数据分析和代码执行需要的依赖）。如果希望只使用更轻量的 Python 执行环境而不引入其他依赖，可以修改为： + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - notebook_executor + - file_operation +``` + +该配置使用独立的 Python 执行器和 Shell 命令执行器，适合轻量级代码执行场景。 + ### 环境变量 在系统环境或 YAML 中配置 API Key： diff --git a/projects/fin_research/README.md b/projects/fin_research/README.md index a8fbc2856..4af074ffd 100644 --- a/projects/fin_research/README.md +++ b/projects/fin_research/README.md @@ -84,16 +84,45 @@ pip install akshare baostock ### Sandbox Setup -The Collector and Analyst agents require Docker for sandboxed execution: +The Collector and Analyst agents default use Docker for sandboxed code execution (optional): ```bash # install ms-enclave (https://github.com/modelscope/ms-enclave) pip install ms-enclave docker websocket-client -# build the required Docker image, make sure you have installed Docker on your device +# build the required Docker image, make sure you have installed Docker on your system bash projects/fin_research/tools/build_jupyter_image.sh ``` +If you prefer not to install Docker and related dependencies, you can instead configure the local code execution tool by modifying the default `tools` section in both `analyst.yaml` and `collector.yaml`: + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - python_executor + - shell_executor + - file_operation +``` + +With this configuration, code is executed through a Jupyter kernel–based notebook executor that isolates environment variables and supports running shell commands. The required dependencies (including those for data analysis and code execution) will be installed automatically on the first run. + +If you want a lighter-weight Python-only execution environment without introducing additional notebook dependencies, you can use: + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - notebook_executor + - file_operation +``` + +This configuration uses an independent Python executor together with a shell command executor and is suitable for lightweight code execution scenarios. + ## 🚀 Quickstart ### Environment Configuration diff --git a/projects/fin_research/README_zh.md b/projects/fin_research/README_zh.md index 644d3d606..cfbc6c93c 100644 --- a/projects/fin_research/README_zh.md +++ b/projects/fin_research/README_zh.md @@ -84,16 +84,43 @@ pip install akshare baostock ### 沙箱环境 -Collector 与 Analyst 需要 Docker 沙箱以安全执行代码： +Collector 与 Analyst 默认使用 Docker 沙箱以安全执行代码（可选）： ```bash # 安装 ms-enclave（https://github.com/modelscope/ms-enclave） pip install ms-enclave docker websocket-client -# 构建所需 Docker 镜像（确保设备已安装并运行 Docker） +# 构建所需 Docker 镜像（确保系统已安装并运行 Docker） bash projects/fin_research/tools/build_jupyter_image.sh ``` +如果不希望安装Docker等依赖，也可以选择配置本地代码执行工具，推荐将`analyst.yaml`和`collector.yaml`中默认的`tools`配置修改为： + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - python_executor + - shell_executor + - file_operation +``` + +该配置下默认依赖Jupyter Kernel执行代码，提供对环境变量的隔离，并支持shell命令执行，相应的依赖将在第一次运行代码时自动安装（包括数据分析和代码执行需要的依赖）。如果希望只使用更轻量的Python执行环境而不引入其他依赖，可以修改为： + +```yaml +tools: + code_executor: + mcp: false + implementation: python_env + exclude: + - notebook_executor + - file_operation +``` + +该配置使用独立的Python执行器和Shell命令执行器，适合轻量级代码执行场景。 + ## 🚀 快速开始 ### 环境变量配置 From 88173f2fea05c9793dad4da6e45fc2f1f5a0ea4d Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Tue, 25 Nov 2025 21:34:51 +0800 Subject: [PATCH 4/9] support for spec_loader and enhance the stability --- ms_agent/app/fin_research.py | 56 ++- ms_agent/tools/docling/doc_loader.py | 5 +- projects/fin_research/aggregator.yaml | 12 +- projects/fin_research/analyst.yaml | 13 +- projects/fin_research/collector.yaml | 2 + projects/fin_research/orchestrator.yaml | 2 + .../fin_research/tools/principle_skill.py | 331 -------------- projects/fin_research/tools/spec_constant.py | 106 +++++ projects/fin_research/tools/spec_loader.py | 406 ++++++++++++++++++ .../principle_specs}/Boston_Matrix.md | 0 .../principle_specs}/MECE.md | 0 .../principle_specs}/Minto_Pyramid.md | 0 .../principle_specs}/Pareto_80-20.md | 0 .../principle_specs}/SWOT.md | 0 .../principle_specs}/Value_Chain.md | 0 .../writing_specs/Bullets_Paragraph_Rhythm.md | 24 ++ .../writing_specs/Density_Length_Control.md | 27 ++ .../writing_specs/Methodology_Exposure.md | 25 ++ .../specs/writing_specs/Structure_Layering.md | 28 ++ .../writing_specs/Task_Focus_Relevance.md | 27 ++ .../specs/writing_specs/Tone_Analyst_Voice.md | 26 ++ 21 files changed, 746 insertions(+), 344 deletions(-) delete mode 100644 projects/fin_research/tools/principle_skill.py create mode 100644 projects/fin_research/tools/spec_constant.py create mode 100644 projects/fin_research/tools/spec_loader.py rename projects/fin_research/tools/{principles => specs/principle_specs}/Boston_Matrix.md (100%) rename projects/fin_research/tools/{principles => specs/principle_specs}/MECE.md (100%) rename projects/fin_research/tools/{principles => specs/principle_specs}/Minto_Pyramid.md (100%) rename projects/fin_research/tools/{principles => specs/principle_specs}/Pareto_80-20.md (100%) rename projects/fin_research/tools/{principles => specs/principle_specs}/SWOT.md (100%) rename projects/fin_research/tools/{principles => specs/principle_specs}/Value_Chain.md (100%) create mode 100644 projects/fin_research/tools/specs/writing_specs/Bullets_Paragraph_Rhythm.md create mode 100644 projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md create mode 100644 projects/fin_research/tools/specs/writing_specs/Methodology_Exposure.md create mode 100644 projects/fin_research/tools/specs/writing_specs/Structure_Layering.md create mode 100644 projects/fin_research/tools/specs/writing_specs/Task_Focus_Relevance.md create mode 100644 projects/fin_research/tools/specs/writing_specs/Tone_Analyst_Voice.md diff --git a/ms_agent/app/fin_research.py b/ms_agent/app/fin_research.py index 64da11fe4..1767d3e70 100644 --- a/ms_agent/app/fin_research.py +++ b/ms_agent/app/fin_research.py @@ -2209,6 +2209,36 @@ def create_interface(): opacity: 0.95; } + .main-header .main-intro { + max-width: 1024px; + margin: 1rem auto 0.75rem; + padding: 0.85rem 1.25rem; + background: rgba(15, 23, 42, 0.22); + border-radius: 0.85rem; + border: 1px solid rgba(255, 255, 255, 0.28); + font-size: clamp(0.95rem, 1.4vw, 1.1rem); + line-height: 1.65; + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.08), + 0 10px 30px rgba(15, 23, 42, 0.18); + backdrop-filter: blur(2px); + } + + .main-header .main-intro span { + display: block; + } + + .main-header .main-intro .cn { + font-weight: 600; + letter-spacing: 0.01em; + } + + .main-header .main-intro .en { + margin-top: 0.35rem; + font-size: clamp(0.9rem, 1.25vw, 1.05rem); + color: rgba(226, 232, 240, 0.95); + letter-spacing: 0.01em; + } + .main-header .powered-by { margin-top: 0.35rem; font-size: clamp(0.85rem, 1.2vw, 1rem); @@ -2826,6 +2856,14 @@ def create_interface(): background: linear-gradient(135deg, #1e40af 0%, #1e3a8a 100%); } + .dark .main-header .main-intro { + background: rgba(15, 23, 42, 0.6); + border-color: rgba(148, 163, 184, 0.35); + box-shadow: inset 0 0 0 1px rgba(59, 130, 246, 0.25), + 0 10px 30px rgba(2, 6, 23, 0.55); + color: rgba(248, 250, 252, 0.95); + } + .dark .status-container { background: #1e293b; border-color: #334155; @@ -2943,6 +2981,10 @@ def create_interface():

📊 FinResearch 金融深度研究

Multi-Agent Financial Research Workflow

+
+ 面向金融研究的多智能体分析引擎，实现从原始市场信号到专业级研究的自动化、端到端金融报告生成。 + A multi-agent analysis engine for financial research that automates the journey from raw market signals to professional-grade insights and end-to-end financial report generation. +

Powered by Readme + | + + Examples +

""") @@ -3046,7 +3094,7 @@ def create_interface(): search_api_key = gr.Textbox( label='搜索引擎 API Key (可选 | Optional)', - placeholder='支持 exa: / serpapi: ', + placeholder='输入格式：exa:xxx 或 serpapi:xxx', type='password' ) @@ -3082,7 +3130,7 @@ def create_interface():
-
⏳ 等待启动... | Waiting to start...
+
⏳ 准备就绪... | Ready for Execution....
@@ -3199,10 +3247,10 @@ def create_interface(): 💡 提示 | Tip

- 研究任务通常需要十几分钟时间完成。您可以实时查看右侧的执行状态，了解当前是哪个 Agent 在工作。建议在研究目标中明确指定股票代码、时间范围和关注的分析维度，以获得更精准的结果。 + 研究任务通常需要十几分钟时间完成。您可以实时查看右侧的执行状态，了解当前是哪个 Agent 在工作。建议在研究目标中明确指定股票代码、时间范围和关注的分析维度，以获得更精准的结果。如果希望获得速度更快、更稳定的体验，建议在本地进行部署。 - Research tasks typically take several minutes to complete. You can monitor the execution status on the right to see which agent is working. Specify stock tickers, time ranges, and analysis dimensions for more accurate results. + Research tasks typically take several minutes to complete. You can monitor the execution status on the right to see which agent is working. Specify stock tickers, time ranges, and analysis dimensions for more accurate results. For a faster and more stable experience, we recommend deploying it locally.

diff --git a/ms_agent/tools/docling/doc_loader.py b/ms_agent/tools/docling/doc_loader.py index 7b84c7fcf..18c4bdfec 100644 --- a/ms_agent/tools/docling/doc_loader.py +++ b/ms_agent/tools/docling/doc_loader.py @@ -256,9 +256,10 @@ def check_file_valid(file: tuple[int, str]) -> tuple[int, str] | None: file_paths = [(i, file) for i, file in enumerate(url_or_files) if file and not file.startswith('http')] preprocessed = [] + max_workers = min(8, (os.cpu_count() or 4)) # Step1: Remove urls that cannot be processed - with ThreadPoolExecutor() as executor: + with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [ executor.submit(check_url_valid, url) for url in http_urls ] @@ -268,7 +269,7 @@ def check_file_valid(file: tuple[int, str]) -> tuple[int, str] | None: preprocessed.append(result) # Step2: Add file paths that are valid - with ThreadPoolExecutor() as executor: + with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [ executor.submit(check_file_valid, file) for file in file_paths ] diff --git a/projects/fin_research/aggregator.yaml b/projects/fin_research/aggregator.yaml index b7624ff84..319566d02 100644 --- a/projects/fin_research/aggregator.yaml +++ b/projects/fin_research/aggregator.yaml @@ -7,6 +7,8 @@ llm: generation_config: stream: true + stream_options: + include_usage: true prompt: @@ -59,6 +61,8 @@ prompt: - Review the report outline, all chapter files, `cross_chapter_mismatches.md`, and user's original plan for the financial analysis task. - Summarize and resolve all mismatch issues, ensuring consistency in data, scope, and terminology. - Search for and retain valuable visual elements from the historical context and available resources in the working directory. + - You are STRONGLY ENCOURAGED to retrieve relevant writing specs before drafting the final report, \ + using the spec tools that are currently visible via the spec_loader server. - Perform self-check and refinement for logical flow, clarity, and professional tone, \ and optionally append a brief **Final_Checklist** section summarizing remaining issues and validations. - Output the final report in markdown formated text without tool calls. @@ -67,7 +71,8 @@ prompt: - **Every turn MUST include at least one tool call — without exception — unless you are producing the final report. \ Failure to call a tool in any non-final-report turn is considered a violation of the protocol.** - - Retrieve principles depending on the principle_skill server. + - Retrieve principle specs, writing specs, or other available specs through the spec_loader server, \ + based on the spec tools currently exposed to you. - To reduce the number of conversation turns, multiple tools can be invoked simultaneously within a single turn when necessary. - When using the file_system---write_file tool, pass an empty string ('') to the path argument to use the default working directory. - Avoid redundant tool calls for file queries or reads when the relevant information has \ @@ -92,6 +97,7 @@ prompt: - Maintains a professional tone and report-style formatting. - Keep figure and table numbering independent, each following its own consistent sequence. - Please generate the final report in the same language used in the plan (Chinese, English, etc.). + - You are encouraged to follow the writing specs retrieved from the spec_loader server if available. @@ -125,8 +131,10 @@ tools: mcp: false exclude: - create_directory + spec_loader: + mcp: false plugins: - - tools/principle_skill + - tools/spec_loader handler: time_handler diff --git a/projects/fin_research/analyst.yaml b/projects/fin_research/analyst.yaml index 13eb6e873..992750142 100644 --- a/projects/fin_research/analyst.yaml +++ b/projects/fin_research/analyst.yaml @@ -7,6 +7,8 @@ llm: generation_config: stream: true + stream_options: + include_usage: true prompt: @@ -74,17 +76,16 @@ prompt: assumptions (frequency, annualization, risk-free rate, etc.), conclusions, limitations, and next steps. - You must output all file paths as relative paths to the / directory in this phase, \ for example: "./sessions/session_b5e8d412/profitability_trends.png". + - You MUST output the final report in natural language before you stop. - - Use standard OpenAI function calling to invoke tools. \ - Do NOT output code in assistant's natural language output. - - Every turn MUST include at least one tool call, unless you're providing the FINAL summary. + - Use standard OpenAI function calling to invoke tools. Do NOT output code in assistant's natural language output. + - If you use [ACT=code], [ACT=collect], or [ACT=fix], you MUST include at least one tool call in that turn. + - If you use [ACT=report], you MUST output the comprehensive summary in markdown format and MUST NOT call any tools in that turn. - After each tool call, carefully review the output. - State explicitly what you learned and what comes next. - Continue calling tools until you have sufficient evidence to conclude. - - When analysis is complete and you need to provide a comprehensive summary, \ - you can use [ACT=report] without tools and stop. @@ -123,6 +124,7 @@ prompt: - If a read fails: rotate encodings; if a column is missing: print df.columns and adjust; if time parsing fails: print sample bad rows and fix. - Keep outputs reproducible and auditable (print what changed and why). - Use shell_executor only for non-destructive tasks (e.g., mkdir -p /sessions/..., ls -l) if shell_executor is AVAILABLE. + - NEVER invent data or results. Only use values from retrieved data or executed code. If data is missing or non-computable, say so explicitly. @@ -153,6 +155,7 @@ prompt: [ACT=report] Purpose: Delivering the final synthesis: data & cleaning summary, key figures (absolute paths), \ metrics & assumptions, conclusions, limitations, next steps. + // Note: This final report MUST appear exactly once in natural language output before the conversation ends. // Note: This is the ONLY case where no tool call follows - final summary only. Example 4 - Error handling: diff --git a/projects/fin_research/collector.yaml b/projects/fin_research/collector.yaml index e33b7d868..5eb27f228 100644 --- a/projects/fin_research/collector.yaml +++ b/projects/fin_research/collector.yaml @@ -7,6 +7,8 @@ llm: generation_config: stream: true + stream_options: + include_usage: true prompt: diff --git a/projects/fin_research/orchestrator.yaml b/projects/fin_research/orchestrator.yaml index 4a69544d1..3a5977ec0 100644 --- a/projects/fin_research/orchestrator.yaml +++ b/projects/fin_research/orchestrator.yaml @@ -7,6 +7,8 @@ llm: generation_config: stream: true + stream_options: + include_usage: true prompt: diff --git a/projects/fin_research/tools/principle_skill.py b/projects/fin_research/tools/principle_skill.py deleted file mode 100644 index f72c94ef9..000000000 --- a/projects/fin_research/tools/principle_skill.py +++ /dev/null @@ -1,331 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. -# flake8: noqa -import os -from typing import Any, Dict, List, Optional, Tuple - -import json -from ms_agent.llm.utils import Tool -from ms_agent.tools.base import ToolBase -from ms_agent.utils import get_logger - -logger = get_logger() - -PRINCIPLE_GUIDE = """ - -- **MECE (Mutually Exclusive, Collectively Exhaustive) — non-overlapping, no-omission framing** - - **Use for:** Building problem & metric trees, defining scopes and boundaries, avoiding gaps/duplication. - - **Best for:** Kick-off structuring of any report (industry/company/portfolio/risk). - - **Deliverable:** 3-5 first-level dimensions; second-level factors with measurement definitions; a “Problem → Scope → Metrics” blueprint. - -- **Value Chain (Porter) — sources of cost/value** - - **Use for:** Explaining fundamentals and levers behind Gross Margin / ROIC (primary + support activities). - - **Best for:** Company & supply-chain research; cost curve and pass-through analysis. - - **Deliverable:** Stage → Drivers → Bottlenecks → Improvements → Financial impact (quantified to GM/Cash Flow/KPIs). - -- **BCG Growth-Share Matrix (Boston Matrix) — growth x share portfolio positioning** - - **Use for:** Placing multi-business/multi-track items into Star/Cash Cow/Question Mark/Dog to guide resource/weighting decisions. - - **Best for:** Comparing industry sub-segments; managing company business portfolios. - - **Deliverable:** Quadrant mapping; capital/attention flow plan (e.g., from Cows → Stars/Questions); target weights and migration triggers. - -- **80/20 (Pareto) — focus on the vital few** - - **Use for:** Selecting the top ~20% drivers that explain most outcomes across metrics/assets/factors; compressing workload. - - **Best for:** Return/risk attribution; metric prioritization; evidence triage. - - **Deliverable:** Top-K key drivers + quantified contributions + tracking KPIs; fold the remainder into “long-tail management.” - -- **SWOT → TOWS — from inventory to action pairing** - - **Use for:** Pairing internal (S/W) and external (O/T) to form SO/WO/ST/WT **actionable strategies** with KPIs. - - **Best for:** Strategy setting, post-investment management, risk hedging and adjustment thresholds. - - **Deliverable:** Action list with owners/KPIs/thresholds and financial mapping (revenue/GM/cash-flow impact). - -- **Pyramid / Minto — conclusion-first presentation wrapper** - - **Use for:** Packaging analysis as “Answer → 3 parallel supports → key evidence/risk hedges” for fast executive reading. - - **Best for:** Executive summaries, IC materials, report front pages. - - **Deliverable:** One-sentence conclusion (direction + range + time frame), three parallel key points, strongest evidence charts. - -""" -ROUTING_GUIDE = """ - -Here are some Heuristic hints for selecting the appropriate principles for the task: -- Need to “frame & define scope”? → Start with **MECE**; if explaining costs/moats, add **Value Chain**. -- Multi-business/multi-track “allocation decisions”? → Use **BCG** for positioning & weights, then **80/20** to focus key drivers. -- Want to turn inventory into **executable actions**? → **SWOT→TOWS** for strategy+KPI and threshold design. -- Delivering to management? → Present the whole piece with **Pyramid**; other principles provide evidence and structural core. - -""" - - -class PrincipleSkill(ToolBase): - """Aggregate access to multiple analysis principles. - - Server name: `principle_skill` - - This tool exposes a single function `load_principles` that loads one or more - principle knowledge files and returns their content to the model. Each - principle provides concise concept definitions and guidance on how to apply - the principle to financial analysis and report writing. The underlying - knowledge is stored as Markdown files and can be configured via - `tools.principle_skill.principle_dir` in the agent config. When not provided, - the tool falls back to `projects/fin_research/tools/principles` - under the current working directory. - - Supported principle identifiers (case-insensitive, synonyms allowed): - - MECE → MECE.md - - Pyramid / Minto / Minto Pyramid → Minto_Pyramid.md - - SWOT → SWOT.md - - Value Chain → Value_Chain.md - - Pareto / 80-20 / 80/20 → Pareto_80-20.md - - Boston Matrix / BCG / Boston Consulting Group → Boston_Matrix.md - """ - - PRINCIPLE_DIR = 'projects/fin_research/tools/principles' - - def __init__(self, config): - super().__init__(config) - tools_cfg = getattr(config, 'tools', - None) if config is not None else None - self.exclude_func(getattr(tools_cfg, 'principle_skill', None)) - - configured_dir = None - if tools_cfg is not None: - configured_dir = getattr(tools_cfg, 'principle_dir', None) - - default_root = os.getcwd() - default_dir = os.path.join(default_root, self.PRINCIPLE_DIR) - - # If a config-specified directory exists, prefer it; else use default. - self.principle_dir = configured_dir or default_dir - - # Build a mapping from normalized user inputs to on-disk filenames and display names - self._name_to_file: Dict[str, - Tuple[str, - str]] = self._build_principle_index() - - async def connect(self): - # Warn once if the directory cannot be found; still operate to allow deferred config - if not os.path.isdir(self.principle_dir): - logger.warning_once( - f'[principle_skill] Principle directory not found: {self.principle_dir}. ' - f'Configure tools.principle_skill.principle_dir or ensure default exists.' - ) - - async def get_tools(self) -> Dict[str, Any]: - tools: List[Tool] = { - 'principle_skill': [ - Tool( - tool_name='load_principles', - server_name='principle_skill', - description= - (f'Load one or more analysis principles (concept + how to apply to ' - f'financial analysis) and return their curated Markdown content.\n\n' - f'This is a single-aggregator tool designed to fetch multiple principles ' - f'in one call. Provide a list of requested principles via the "principles" ' - f'parameter. The tool supports common synonyms and is case-insensitive.\n\n' - f'Examples of valid principle identifiers: "MECE", "Pyramid", "Minto", ' - f'"SWOT", "Value Chain", "Pareto", "80-20", "80/20", "Boston Matrix", "BCG".\n\n' - f'When format is "markdown" (default), the tool returns a single combined ' - f'Markdown string (optionally including section titles). When format is ' - f'"json", the tool returns a JSON object mapping principle to content.\n' - f'{PRINCIPLE_GUIDE}\n' - f'{ROUTING_GUIDE}\n'), - parameters={ - 'type': 'object', - 'properties': { - 'principles': { - 'type': - 'array', - 'items': { - 'type': 'string' - }, - 'description': - ('List of principles to load. Case-insensitive; supports synonyms.\n' - 'Allowed identifiers include (non-exhaustive):\n' - '- MECE\n- Pyramid\n- Minto\n- SWOT\n- Value Chain\n' - '- Pareto\n- 80-20\n- 80/20\n- Boston Matrix\n- BCG\n' - ), - }, - 'format': { - 'type': - 'string', - 'enum': ['markdown', 'json'], - 'description': - ('Output format: "markdown" (combined Markdown string) or "json" ' - '(JSON object mapping principle to content). Default: "markdown".' - ), - }, - 'include_titles': { - 'type': - 'boolean', - 'description': - ('When format="markdown", if true, each section is prefixed with a ' - 'Markdown heading of the canonical principle title. Default: true.' - ), - }, - 'join_with': { - 'type': - 'string', - 'description': - ('When format="markdown", the delimiter used to join multiple ' - 'sections. Default: "\n\n---\n\n".'), - }, - 'strict': { - 'type': - 'boolean', - 'description': - ('If true, unknown principles cause an error. If false, unknown ' - 'items are ignored with a note in the output. Default: false.' - ), - }, - }, - 'required': ['principles'], - 'additionalProperties': False, - }, - ) - ] - } - - if hasattr(self, 'exclude_functions') and self.exclude_functions: - tools['principle_skill'] = [ - t for t in tools['principle_skill'] - if t.tool_name not in self.exclude_functions - ] - - return tools - - async def call_tool(self, server_name: str, *, tool_name: str, - tool_args: dict) -> str: - return await getattr(self, tool_name)(**tool_args) - - async def load_principles( - self, - principles: List[str], - format: str = 'markdown', - include_titles: bool = False, - join_with: str = '\n\n---\n\n', - strict: bool = False, - ) -> str: - """Load requested principle documents and return their content. - - Returns: - str: Markdown string (default) or JSON string mapping principle → content. - """ - - if not principles: - return json.dumps( - { - 'success': False, - 'error': 'No principles provided.' - }, - ensure_ascii=False, - indent=2, - ) - - resolved: Dict[str, Tuple[str, str]] = {} - unknown: List[str] = [] - for name in principles: - key = self._normalize_name(name) - if key in self._name_to_file: - resolved[name] = self._name_to_file[key] - else: - unknown.append(name) - - if unknown and strict: - return json.dumps( - { - 'success': - False, - 'error': - 'Unknown principles (strict mode): ' + ', '.join(unknown) - }, - ensure_ascii=False, - indent=2, - ) - - loaded: Dict[str, str] = {} - for original_name, (filename, canonical_title) in resolved.items(): - path = os.path.join(self.principle_dir, filename) - try: - with open(path, 'r') as f: - content = f.read().strip() - loaded[canonical_title] = content - except Exception as e: # noqa - loaded[ - canonical_title] = f'Failed to load {filename}: {str(e)}' - - if not loaded: - return json.dumps( - { - 'success': False, - 'error': 'Failed to load any principles.' - }, - ensure_ascii=False, - indent=2, - ) - - if format == 'json': - payload = { - 'success': True, - 'principles': loaded, - 'unknown': unknown, - 'source_dir': self.principle_dir, - } - return json.dumps(payload, ensure_ascii=False) - - # Default: markdown - sections: List[str] = [] - for title, content in loaded.items(): - if include_titles: - sections.append(f'# {title}\n\n{content}') - else: - sections.append(content) - - if unknown and not strict: - sections.append( - f'> Note: Unknown principles ignored: {", ".join(unknown)}') - - return json.dumps( - { - 'success': True, - 'sections': sections - }, - ensure_ascii=False, - indent=2, - ) - - def _build_principle_index(self) -> Dict[str, Tuple[str, str]]: - """Return mapping from normalized query → (filename, canonical title).""" - entries: List[Tuple[List[str], str, str]] = [ - # synonyms, filename, canonical title - (['mece', 'mutually exclusive and collectively exhaustive'], - 'MECE.md', 'MECE'), - ([ - 'pyramid', 'minto', 'minto pyramid', 'pyramid principle', - 'minto_pyramid' - ], 'Minto_Pyramid.md', 'Pyramid (Minto Pyramid)'), - (['swot', 'swot analysis'], 'SWOT.md', 'SWOT'), - (['value chain', 'value-chain', - 'value_chain'], 'Value_Chain.md', 'Value Chain'), - ([ - 'pareto', '80-20', '80/20', 'pareto 80-20', 'pareto_80-20', - '8020' - ], 'Pareto_80-20.md', 'Pareto (80/20 Rule)'), - ([ - 'boston matrix', 'bcg', 'boston consulting group', - 'boston_matrix', 'boston' - ], 'Boston_Matrix.md', 'Boston Matrix (BCG)'), - ] - - index: Dict[str, Tuple[str, str]] = {} - for synonyms, filename, title in entries: - for s in synonyms: - index[self._normalize_name(s)] = (filename, title) - return index - - @staticmethod - def _normalize_name(name: str) -> str: - s = (name or '').strip().lower() - s = s.replace('_', ' ').replace('-', ' ') - s = ' '.join(s.split()) # collapse whitespace - # normalize 80/20 variants - s = s.replace('80/20', '80-20').replace('80 20', '80-20') - s = s.replace('8020', '80-20') - return s diff --git a/projects/fin_research/tools/spec_constant.py b/projects/fin_research/tools/spec_constant.py new file mode 100644 index 000000000..904f2b9e4 --- /dev/null +++ b/projects/fin_research/tools/spec_constant.py @@ -0,0 +1,106 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# flake8: noqa +# isort: skip_file +# yapf: disable + +WRITING_SPEC_GUIDE = """ + + +- **Structure & Layering — control section depth and hierarchy** + - **Use for:** Deciding how many levels of headings/sections to use in a report. + - **Best for:** Long-form financial/company/industry reports where the model might create 4+ nested levels. + - **Key constraints:** Default to 2–3 heading levels; avoid creating sub-sub-subsections with only 1–2 short paragraphs. + +- **Methodology Exposure — how much to talk about frameworks** + - **Use for:** When you are tempted to write long “Research Methodology” sections or repeatedly mention MECE/SWOT/80-20 in the main text. + - **Best for:** Analyst-style reports where frameworks should be *used* implicitly instead of being “lectured”. + - **Key constraints:** Briefly mention the approach once if needed; do NOT devote full chapters to methods; avoid repeating framework names as slogans. + +- **Bullets & Paragraph Rhythm — bullets vs narrative** + - **Use for:** Deciding when to use bullet points vs continuous paragraphs. + - **Best for:** Sections with many drivers/risks/factors where you may over-bullet every sentence. + - **Key constraints:** Bullets for lists (drivers, risks, recommendations); keep explanatory reasoning in paragraphs; avoid “one sentence per bullet” patterns. + +- **Task Focus & Relevance — stay anchored to the user's question** + - **Use for:** Ensuring that each chapter directly serves the original question instead of drifting into generic industry essays. + - **Best for:** Prompts that ask for specific company/period/comparison/forecast. + - **Key constraints:** Minimize repeated background; tie each section back to the key metrics and drivers required by the task (e.g., profitability, cash-flow quality, competition, forecasts). + +- **Tone & Analyst Voice — sound like a human analyst, not a textbook** + - **Use for:** Choosing phrasing style and presentation voice. + - **Best for:** Sell-side / buy-side style reports, IC memos, investment notes. + - **Key constraints:** Conclusion-first; 2–4 key supporting points; professional but readable; avoid academic jargon and over-formal “methodology lectures”. + +- **Density & Length Control — right amount of detail** + - **Use for:** Controlling report length and pruning low-value content. + - **Best for:** Long multi-chapter outputs where token budget and human attention are limited. + - **Key constraints:** Prioritize conclusions, drivers, and critical numbers; compress or omit peripheral background; avoid repeating the same facts in multiple chapters. + +""" +WRITING_ROUTING_GUIDE = """ + +Heuristics for selecting writing style specs: + +- Report feels too much like an academic paper or you want a clear report skeleton? + → Load **Structure & Layering** + **Tone & Analyst Voice**. + +- You're about to write a long “Research Methodology” chapter or heavily talk about MECE/SWOT/etc.? + → Load **Methodology Exposure** (and follow its constraints strictly). + +- You're using many bullet points and the text starts looking like a checklist? + → Load **Bullets & Paragraph Rhythm** to rebalance bullets vs narrative flow. + +- The user's question is narrow (e.g., “past 4 quarters + next 2 quarters”), but you're expanding a lot on generic industry background? + → Load **Task Focus & Relevance** to keep all chapters anchored to the core task. + +- The answer tends to be very long and repetitive, and you need to compress while preserving value? + → Load **Density & Length Control**; it tells you what to prune and what to keep. + +You can combine multiple specs in one call, e.g.: +- For an analyst-style profitability & forecast report: + → [ "structure", "tone", "methods", "bullets", "focus" ] + +""" + +PRINCIPLE_SPEC_GUIDE = """ + +- **MECE (Mutually Exclusive, Collectively Exhaustive) — non-overlapping, no-omission framing** + - **Use for:** Building problem & metric trees, defining scopes and boundaries, avoiding gaps/duplication. + - **Best for:** Kick-off structuring of any report (industry/company/portfolio/risk). + - **Deliverable:** 3-5 first-level dimensions; second-level factors with measurement definitions; a “Problem → Scope → Metrics” blueprint. + +- **Value Chain (Porter) — sources of cost/value** + - **Use for:** Explaining fundamentals and levers behind Gross Margin / ROIC (primary + support activities). + - **Best for:** Company & supply-chain research; cost curve and pass-through analysis. + - **Deliverable:** Stage → Drivers → Bottlenecks → Improvements → Financial impact (quantified to GM/Cash Flow/KPIs). + +- **BCG Growth-Share Matrix (Boston Matrix) — growth x share portfolio positioning** + - **Use for:** Placing multi-business/multi-track items into Star/Cash Cow/Question Mark/Dog to guide resource/weighting decisions. + - **Best for:** Comparing industry sub-segments; managing company business portfolios. + - **Deliverable:** Quadrant mapping; capital/attention flow plan (e.g., from Cows → Stars/Questions); target weights and migration triggers. + +- **80/20 (Pareto) — focus on the vital few** + - **Use for:** Selecting the top ~20% drivers that explain most outcomes across metrics/assets/factors; compressing workload. + - **Best for:** Return/risk attribution; metric prioritization; evidence triage. + - **Deliverable:** Top-K key drivers + quantified contributions + tracking KPIs; fold the remainder into “long-tail management.” + +- **SWOT → TOWS — from inventory to action pairing** + - **Use for:** Pairing internal (S/W) and external (O/T) to form SO/WO/ST/WT **actionable strategies** with KPIs. + - **Best for:** Strategy setting, post-investment management, risk hedging and adjustment thresholds. + - **Deliverable:** Action list with owners/KPIs/thresholds and financial mapping (revenue/GM/cash-flow impact). + +- **Pyramid / Minto — conclusion-first presentation wrapper** + - **Use for:** Packaging analysis as “Answer → 3 parallel supports → key evidence/risk hedges” for fast executive reading. + - **Best for:** Executive summaries, IC materials, report front pages. + - **Deliverable:** One-sentence conclusion (direction + range + time frame), three parallel key points, strongest evidence charts. + +""" +PRINCIPLE_ROUTING_GUIDE = """ + +Here are some Heuristic hints for selecting the appropriate principles for the task: +- Need to “frame & define scope”? → Start with **MECE**; if explaining costs/moats, add **Value Chain**. +- Multi-business/multi-track “allocation decisions”? → Use **BCG** for positioning & weights, then **80/20** to focus key drivers. +- Want to turn inventory into **executable actions**? → **SWOT→TOWS** for strategy+KPI and threshold design. +- Delivering to management? → Present the whole piece with **Pyramid**; other principles provide evidence and structural core. + +""" diff --git a/projects/fin_research/tools/spec_loader.py b/projects/fin_research/tools/spec_loader.py new file mode 100644 index 000000000..6ed8e1e21 --- /dev/null +++ b/projects/fin_research/tools/spec_loader.py @@ -0,0 +1,406 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +# flake8: noqa +import os +from typing import Any, Dict, List, Tuple + +import json +from ms_agent.llm.utils import Tool +from ms_agent.tools.base import ToolBase +from ms_agent.utils import get_logger +from spec_constant import (PRINCIPLE_ROUTING_GUIDE, PRINCIPLE_SPEC_GUIDE, + WRITING_ROUTING_GUIDE, WRITING_SPEC_GUIDE) + +logger = get_logger() + + +class SpecLoader(ToolBase): + """Aggregate access to multiple specs for financial reports. + + Server name: `spec_loader` + + This tool exposes functions `load__specs` that loads one or more + spec files and returns their content to the model. Each + spec provides concise rules and examples on how to structure and phrase + financial analysis reports in an analyst-like style. + The underlying knowledge is stored as Markdown files and can be configured via + `tools.spec_loader.spec_dir` in the agent config. When not provided, + the tool falls back to `projects/fin_research/tools/specs` + under the current working directory. + + Supported spec tools (case-insensitive, synonyms allowed): + - writing_specs → writing_specs/xxxx.md + - principle_specs → principle_specs/xxxx.md + """ + + SPEC_DIR = 'projects/fin_research/tools/specs' + + def __init__(self, config): + super().__init__(config) + tools_cfg = getattr(config, 'tools', + None) if config is not None else None + spec_cfg = getattr(tools_cfg, 'spec_loader', + None) if tools_cfg is not None else None + self.exclude_func(spec_cfg) + + configured_dir = getattr(spec_cfg, 'spec_dir', + None) if spec_cfg is not None else None + default_dir = os.path.join(os.getcwd(), self.SPEC_DIR) + self.spec_dir = configured_dir or default_dir + + async def connect(self): + # Warn once if the directory cannot be found; still operate to allow deferred config + if not os.path.isdir(self.spec_dir): + logger.warning_once( + f'Spec directory not found: {self.spec_dir}. ' + f'Configure tools.spec_loader.spec_dir or ensure default exists.' + ) + + async def get_tools(self) -> Dict[str, Any]: + tools: Dict[str, List[Tool]] = { + 'spec_loader': [ + Tool( + tool_name='load_writing_specs', + server_name='spec_loader', + description= + ('Load one or more writing-style specs (rules + examples) and return ' + 'their curated Markdown content. Use this when you are unsure about ' + 'how to structure or phrase a financial report in an analyst-like style.\n\n' + 'Supported spec identifiers (case-insensitive, synonyms allowed):\n' + '- structure → section depth / headings\n' + '- methods → how much to expose MECE/SWOT/etc.\n' + '- bullets → bullets vs paragraphs\n' + '- focus → task focus and relevance\n' + '- tone → analyst-style voice\n' + '- density → length and information density control\n\n' + 'Provide a list of requested writing specs via the "writing_specs" parameter.\n\n' + f'{WRITING_SPEC_GUIDE}\n' + f'{WRITING_ROUTING_GUIDE}\n'), + parameters={ + 'type': 'object', + 'properties': { + 'writing_specs': { + 'type': + 'array', + 'items': { + 'type': 'string' + }, + 'description': + ('List of writing specs to load. Case-insensitive; supports synonyms.\n' + 'Allowed identifiers include (non-exhaustive):\n' + '- structure\n- methods\n- bullets\n- focus\n- tone\n- density\n' + ), + }, + 'format': { + 'type': + 'string', + 'enum': ['markdown', 'json'], + 'description': + ('Output format: "markdown" (combined Markdown string) or "json" ' + '(JSON object mapping spec to content). Default: "markdown".' + ), + }, + 'include_titles': { + 'type': + 'boolean', + 'description': + ('When format="markdown", if true, each section is prefixed with a ' + 'Markdown heading of the canonical spec title. Default: false.' + ), + }, + 'join_with': { + 'type': + 'string', + 'description': + ('When format="markdown", the delimiter used to join multiple ' + 'sections. Default: "\n\n---\n\n".'), + }, + 'strict': { + 'type': + 'boolean', + 'description': + ('If true, unknown specs cause an error. If false, unknown items are ' + 'ignored with a note in the output. Default: false.' + ), + }, + }, + 'required': ['writing_specs'], + 'additionalProperties': False, + }, + ), + Tool( + tool_name='load_principle_specs', + server_name='spec_loader', + description= + (f'Load one or more analysis principles (concept + how to apply to ' + f'financial analysis) and return their curated Markdown content.\n\n' + f'This is a single-aggregator tool designed to fetch multiple principles ' + f'in one call. Provide a list of requested principles via the "principles" ' + f'parameter. The tool supports common synonyms and is case-insensitive.\n\n' + f'Examples of valid principle identifiers: "MECE", "Pyramid", "Minto", ' + f'"SWOT", "Value Chain", "Pareto", "80-20", "80/20", "Boston Matrix", "BCG".\n\n' + f'When format is "markdown" (default), the tool returns a single combined ' + f'Markdown string (optionally including section titles). When format is ' + f'"json", the tool returns a JSON object mapping principle to content.\n' + f'{PRINCIPLE_SPEC_GUIDE}\n' + f'{PRINCIPLE_ROUTING_GUIDE}\n'), + parameters={ + 'type': 'object', + 'properties': { + 'principles': { + 'type': + 'array', + 'items': { + 'type': 'string' + }, + 'description': + ('List of principles to load. Case-insensitive; supports synonyms.\n' + 'Allowed identifiers include (non-exhaustive):\n' + '- MECE\n- Pyramid\n- Minto\n- SWOT\n- Value Chain\n' + '- Pareto\n- 80-20\n- 80/20\n- Boston Matrix\n- BCG\n' + ), + }, + 'format': { + 'type': + 'string', + 'enum': ['markdown', 'json'], + 'description': + ('Output format: "markdown" (combined Markdown string) or "json" ' + '(JSON object mapping principle to content). Default: "markdown".' + ), + }, + 'include_titles': { + 'type': + 'boolean', + 'description': + ('When format="markdown", if true, each section is prefixed with a ' + 'Markdown heading of the canonical principle title. Default: false.' + ), + }, + 'join_with': { + 'type': + 'string', + 'description': + ('When format="markdown", the delimiter used to join multiple ' + 'sections. Default: "\n\n---\n\n".'), + }, + 'strict': { + 'type': + 'boolean', + 'description': + ('If true, unknown principles cause an error. If false, unknown ' + 'items are ignored with a note in the output. Default: false.' + ), + }, + }, + 'required': ['principles'], + 'additionalProperties': False, + }, + ) + ] + } + + if hasattr(self, 'exclude_functions') and self.exclude_functions: + tools['spec_loader'] = [ + t for t in tools['spec_loader'] + if t['tool_name'] not in self.exclude_functions + ] + + return tools + + async def call_tool(self, server_name: str, *, tool_name: str, + tool_args: dict) -> str: + return await getattr(self, tool_name)(**tool_args) + + async def load_writing_specs(self, writing_specs: List[str], + **kwargs) -> str: + writing_spec_map = self._build_writing_spec_index() + return await self.load_specs(writing_spec_map, writing_specs, **kwargs) + + async def load_principle_specs(self, principles: List[str], + **kwargs) -> str: + principle_map = self._build_principle_spec_index() + return await self.load_specs(principle_map, principles, **kwargs) + + async def load_specs( + self, + spec_map: Dict[str, Tuple[str, str]], + specs: List[str], + format: str = 'markdown', + include_titles: bool = False, + join_with: str = '\n\n---\n\n', + strict: bool = False, + ) -> str: + """Load requested specs documents and return their content. + + Returns: + str: Markdown string (default) or JSON string mapping spec to content. + """ + + if not specs: + return json.dumps( + { + 'success': False, + 'error': 'No specs provided.' + }, + ensure_ascii=False, + indent=2, + ) + + resolved: Dict[str, Tuple[str, str]] = {} + unknown: List[str] = [] + for name in specs: + key = self._normalize_name(name) + if key in spec_map: + resolved[name] = spec_map[key] + else: + unknown.append(name) + + if unknown and strict: + return json.dumps( + { + 'success': False, + 'error': + 'Unknown specs (strict mode): ' + ', '.join(unknown), + }, + ensure_ascii=False, + indent=2, + ) + + loaded: Dict[str, str] = {} + for _, (filename, canonical_title) in resolved.items(): + path = os.path.join(self.spec_dir, filename) + try: + with open(path, 'r') as f: + content = f.read().strip() + loaded[canonical_title] = content + except Exception as e: # noqa + loaded[ + canonical_title] = f'Failed to load {filename}: {str(e)}' + + if not loaded: + return json.dumps( + { + 'success': False, + 'error': 'Failed to load any specs.' + }, + ensure_ascii=False, + indent=2, + ) + + if format == 'json': + payload = { + 'success': True, + 'specs': loaded, + 'unknown': unknown, + 'source_dir': self.spec_dir, + } + return json.dumps(payload, ensure_ascii=False) + + # Default: markdown + sections: List[str] = [] + for title, content in loaded.items(): + if include_titles: + sections.append(f'# {title}\n\n{content}') + else: + sections.append(content) + + if unknown and not strict: + sections.append( + f'> Note: Unknown specs ignored: {", ".join(unknown)}') + + return json.dumps( + { + 'success': True, + 'sections': join_with.join(sections) + }, + ensure_ascii=False, + indent=2) + + def _build_writing_spec_index(self) -> Dict[str, Tuple[str, str]]: + """Return writing spec mapping from normalized query → (filename, canonical title).""" + entries = [ + # synonyms, filename, canonical title + ( + ['structure', 'structure & layering', 'layering', 'sections'], + 'writing_specs/Structure_Layering.md', + 'Structure & Layering', + ), + ( + [ + 'methods', 'methodology', 'framework exposure', + 'methodology exposure' + ], + 'writing_specs/Methodology_Exposure.md', + 'Methodology Exposure', + ), + ( + [ + 'bullets', 'bullet', 'bullets & paragraphs', + 'paragraph rhythm' + ], + 'writing_specs/Bullets_Paragraph_Rhythm.md', + 'Bullets & Paragraph Rhythm', + ), + ( + ['focus', 'relevance', 'task focus', 'task focus & relevance'], + 'writing_specs/Task_Focus_Relevance.md', + 'Task Focus & Relevance', + ), + ( + ['tone', 'voice', 'analyst voice', 'tone & analyst voice'], + 'writing_specs/Tone_Analyst_Voice.md', + 'Tone & Analyst Voice', + ), + ( + ['density', 'length', 'density & length', 'length control'], + 'writing_specs/Density_Length_Control.md', + 'Density & Length Control', + ), + ] + + index: Dict[str, Tuple[str, str]] = {} + for synonyms, filename, title in entries: + for s in synonyms: + index[self._normalize_name(s)] = (filename, title) + return index + + def _build_principle_spec_index(self) -> Dict[str, Tuple[str, str]]: + """Return principle spec mapping from normalized query → (filename, canonical title).""" + entries: List[Tuple[List[str], str, str]] = [ + # synonyms, filename, canonical title + (['mece', 'mutually exclusive and collectively exhaustive'], + 'principle_specs/MECE.md', 'MECE'), + ([ + 'pyramid', 'minto', 'minto pyramid', 'pyramid principle', + 'minto_pyramid' + ], 'principle_specs/Minto_Pyramid.md', 'Pyramid (Minto Pyramid)'), + (['swot', 'swot analysis'], 'principle_specs/SWOT.md', 'SWOT'), + (['value chain', 'value-chain', + 'value_chain'], 'principle_specs/Value_Chain.md', 'Value Chain'), + ([ + 'pareto', '80-20', '80/20', 'pareto 80-20', 'pareto_80-20', + '8020' + ], 'principle_specs/Pareto_80-20.md', 'Pareto (80/20 Rule)'), + ([ + 'boston matrix', 'bcg', 'boston consulting group', + 'boston_matrix', 'boston' + ], 'principle_specs/Boston_Matrix.md', 'Boston Matrix (BCG)'), + ] + + index: Dict[str, Tuple[str, str]] = {} + for synonyms, filename, title in entries: + for s in synonyms: + index[self._normalize_name(s)] = (filename, title) + return index + + @staticmethod + def _normalize_name(name: str) -> str: + s = (name or '').strip().lower() + s = s.replace('_', ' ').replace('-', ' ') + s = ' '.join(s.split()) # collapse whitespace + + # normalize 80/20 variants, specially for principle specs + s = s.replace('80/20', '80-20').replace('80 20', '80-20') + s = s.replace('8020', '80-20') + + return s diff --git a/projects/fin_research/tools/principles/Boston_Matrix.md b/projects/fin_research/tools/specs/principle_specs/Boston_Matrix.md similarity index 100% rename from projects/fin_research/tools/principles/Boston_Matrix.md rename to projects/fin_research/tools/specs/principle_specs/Boston_Matrix.md diff --git a/projects/fin_research/tools/principles/MECE.md b/projects/fin_research/tools/specs/principle_specs/MECE.md similarity index 100% rename from projects/fin_research/tools/principles/MECE.md rename to projects/fin_research/tools/specs/principle_specs/MECE.md diff --git a/projects/fin_research/tools/principles/Minto_Pyramid.md b/projects/fin_research/tools/specs/principle_specs/Minto_Pyramid.md similarity index 100% rename from projects/fin_research/tools/principles/Minto_Pyramid.md rename to projects/fin_research/tools/specs/principle_specs/Minto_Pyramid.md diff --git a/projects/fin_research/tools/principles/Pareto_80-20.md b/projects/fin_research/tools/specs/principle_specs/Pareto_80-20.md similarity index 100% rename from projects/fin_research/tools/principles/Pareto_80-20.md rename to projects/fin_research/tools/specs/principle_specs/Pareto_80-20.md diff --git a/projects/fin_research/tools/principles/SWOT.md b/projects/fin_research/tools/specs/principle_specs/SWOT.md similarity index 100% rename from projects/fin_research/tools/principles/SWOT.md rename to projects/fin_research/tools/specs/principle_specs/SWOT.md diff --git a/projects/fin_research/tools/principles/Value_Chain.md b/projects/fin_research/tools/specs/principle_specs/Value_Chain.md similarity index 100% rename from projects/fin_research/tools/principles/Value_Chain.md rename to projects/fin_research/tools/specs/principle_specs/Value_Chain.md diff --git a/projects/fin_research/tools/specs/writing_specs/Bullets_Paragraph_Rhythm.md b/projects/fin_research/tools/specs/writing_specs/Bullets_Paragraph_Rhythm.md new file mode 100644 index 000000000..46a796f7e --- /dev/null +++ b/projects/fin_research/tools/specs/writing_specs/Bullets_Paragraph_Rhythm.md @@ -0,0 +1,24 @@ +# Bullets & Paragraph Rhythm +Balance bullet lists and narrative flow. + +## Overview +Bullets structure lists; paragraphs carry reasoning. +Over-bulleted text reads robotic; insufficient bullets reduces clarity. + +## Core Rules +- Use bullets for **drivers, risks, assumptions, KPIs**. +- Keep explanations and transitions in paragraphs. +- Avoid long sections made only of bullets. + +## Do +- Precede/follow bullet lists with short paragraphs (2–4 sentences). +- Use bullets only when enumerating items. + +## Don’t +- Don’t place every sentence in a bullet. +- Don’t create sections that contain only a single short bullet point without explanation. + +## Checklist +- [ ] Bullets only where listing is appropriate +- [ ] Reasoning remains in prose +- [ ] Bullet sections are not excessively long diff --git a/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md b/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md new file mode 100644 index 000000000..fcdd5534a --- /dev/null +++ b/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md @@ -0,0 +1,27 @@ +# Density & Length Control +Maintain high information value with minimal redundancy. + +## Overview +Long reports risk redundancy. +This spec ensures conciseness without losing analytical depth. + +## Core Rules +- Prioritize high-signal content directly related to profitability, cash flows, competition, and forecasts + (e.g., margins, key drivers, valuation, competitive moves). +- Compress low-value background while keeping essential macro/policy factors that affect conclusions. +- One main idea per paragraph; supporting evidence can stay in the same paragraph. + +## Do +- Summarize background when necessary instead of narrating it in full. . +- Consolidate similar facts into a single, high-density paragraph or table. +- Prune paragraphs that don’t change conclusions, risk assessment, or forecasts. + +## Don’t +- Don’t repeat identical information across chapters unless it is a deliberate recap in the executive summary or conclusion. +- Don’t include irrelevant company history or generic industry description. +- Don’t let sections balloon in length without new insights or decision-relevant facts. + +## Checklist +- [ ] No unnecessary repetition of facts across sections (brief recaps in summary/conclusion are allowed) +- [ ] Each paragraph adds analytical or decision-relevant value +- [ ] Section length is proportional to its importance for the investment view diff --git a/projects/fin_research/tools/specs/writing_specs/Methodology_Exposure.md b/projects/fin_research/tools/specs/writing_specs/Methodology_Exposure.md new file mode 100644 index 000000000..37338b486 --- /dev/null +++ b/projects/fin_research/tools/specs/writing_specs/Methodology_Exposure.md @@ -0,0 +1,25 @@ +# Methodology Exposure +Control how frameworks appear in reports. + +## Overview +Analyst reports should **use** frameworks implicitly, not **discuss** them like a textbook. + +## Core Rules +- As a rule of thumb, mention methodology at most once in the main text, and keep it brief. +- Only add extra clarification in footnotes or appendices when strictly necessary (e.g., for data sources or backtests). +- No standalone “Methodology” chapters. +- Avoid repeated naming of MECE / SWOT / 80-20 / Value Chain. +- Emphasize **conclusions and drivers**, not analytical process. + +## Do +- Use frameworks silently to organize logic. +- Introduce the approach in 1 sentence if needed. + +## Don’t +- Don't describe frameworks in detail. +- Don’t repeatedly cite them or make them focal content. + +## Checklist +- [ ] Frameworks named ≤1 time in the main text +- [ ] No method-focused chapters +- [ ] Focus stays on results, not process diff --git a/projects/fin_research/tools/specs/writing_specs/Structure_Layering.md b/projects/fin_research/tools/specs/writing_specs/Structure_Layering.md new file mode 100644 index 000000000..b0586167b --- /dev/null +++ b/projects/fin_research/tools/specs/writing_specs/Structure_Layering.md @@ -0,0 +1,28 @@ +# Structure & Layering +Guidelines for clean, shallow section hierarchy. + +## Overview +Control outline depth so the report reads like an analyst note, not an academic paper. +Keep structures simple, readable, and coherent. +Use this shallow hierarchy as the default template; only exceed it in rare, task-specific cases + +## Core Rules +- **Default to 2–3 heading levels** (Chapter → Section → Subsection). +- Avoid trivial micro-sections that don’t add structural value. Short, necessary sections (e.g., risk disclaimer) are acceptable. +- Use **natural paragraphs** for flow; don’t fragment text excessively. +- Keep a typical pattern: Summary → Core Analysis → Forecast → Risks → Conclusion. + +## Do +- Use headings only for meaningful topic shifts. +- Merge closely related content. +- Maintain internal narrative continuity. + +## Don’t +- Don’t use deep hierarchies (1.2.3.4.5). +- Don’t repeat identical background across sections. +- Don’t make bullet-only sections. + +## Checklist +- [ ] ≤3 levels used +- [ ] No redundant micro-sections +- [ ] Narrative flows smoothly diff --git a/projects/fin_research/tools/specs/writing_specs/Task_Focus_Relevance.md b/projects/fin_research/tools/specs/writing_specs/Task_Focus_Relevance.md new file mode 100644 index 000000000..e6d1ad6e6 --- /dev/null +++ b/projects/fin_research/tools/specs/writing_specs/Task_Focus_Relevance.md @@ -0,0 +1,27 @@ +# Task Focus & Relevance +Keep the report anchored to the user’s specific question. + +## Overview +Avoid drifting into long industry essays. +Ensure all content serves the **company-scope-time window** defined by the task. + +## Core Rules +- Tie every major section to one or more core analytical dimensions required by the task (e.g., profitability, growth, cash flow, competition, risk, or forecast). +- Minimize repeated or irrelevant background. +- Avoid policy histories or macro digressions unless directly relevant. +- When macro/policy factors are mentioned, immediately link them to specific company metrics or trend changes in the defined time window. + +## Do +- Restate the task in the introduction. +- Explicitly link each section to the target companies/time frame. +- Consolidate industry/macro background into **one short background subsection** (e.g., 1–2 paragraphs in the introduction). +- When later sections need background, **reuse it in one sentence and reference the earlier subsection** (e.g., “As noted in Section 1.1, …”) instead of rewriting the full context. + +## Don’t +- Don’t include unrelated macro/policy background. +- Don’t duplicate the same context across chapters. + +## Checklist +- [ ] Each section can be explained in one sentence how it serves the task. +- [ ] Background concise and non-repetitive +- [ ] No scope creep diff --git a/projects/fin_research/tools/specs/writing_specs/Tone_Analyst_Voice.md b/projects/fin_research/tools/specs/writing_specs/Tone_Analyst_Voice.md new file mode 100644 index 000000000..169a8e742 --- /dev/null +++ b/projects/fin_research/tools/specs/writing_specs/Tone_Analyst_Voice.md @@ -0,0 +1,26 @@ +# Tone & Analyst Voice +Write like a professional financial analyst. + +## Overview +Aim for a concise, conclusion-first, evidence-backed voice. +Avoid academic-style writing or methodology-focused exposition. + +## Core Rules +- Start with clear, top-line conclusions. +- Use professional but readable language. +- Calibrate confidence in forecasts (avoid absolute claims). +- Avoid academic framing and jargon-heavy methodology talk. + +## Do +- Use causal reasoning (“due to…”, “driven by…”). +- Use domain-specific vocabulary naturally. +- Keep sentences tight and information-dense. + +## Don’t +- Don’t slip into academic or thesis-like style. +- Don’t overstate certainty (“will definitely”, “must”). + +## Checklist +- [ ] Conclusion-first structure +- [ ] Tone = analyst, not textbook +- [ ] Language concise, factual, and readable From e4a76178ecbb722e1ee3a88def76d41921c45a03 Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Tue, 25 Nov 2025 21:35:55 +0800 Subject: [PATCH 5/9] fix type in Density_Length_Control.md --- .../tools/specs/writing_specs/Density_Length_Control.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md b/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md index fcdd5534a..efe5a3da8 100644 --- a/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md +++ b/projects/fin_research/tools/specs/writing_specs/Density_Length_Control.md @@ -12,7 +12,7 @@ This spec ensures conciseness without losing analytical depth. - One main idea per paragraph; supporting evidence can stay in the same paragraph. ## Do -- Summarize background when necessary instead of narrating it in full. . +- Summarize background when necessary instead of narrating it in full. - Consolidate similar facts into a single, high-density paragraph or table. - Prune paragraphs that don’t change conclusions, risk assessment, or forecasts. From 227f3ef9a0560c1c49af81109bc99310cde02a88 Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Wed, 26 Nov 2025 17:05:23 +0800 Subject: [PATCH 6/9] support for AgentTool --- ms_agent/tools/__init__.py | 1 + ms_agent/tools/agent_tool.py | 330 +++++++++++++++++++++++++++++++++ ms_agent/tools/tool_manager.py | 7 + 3 files changed, 338 insertions(+) create mode 100644 ms_agent/tools/agent_tool.py diff --git a/ms_agent/tools/__init__.py b/ms_agent/tools/__init__.py index 9d4517403..0f2e2cdf3 100644 --- a/ms_agent/tools/__init__.py +++ b/ms_agent/tools/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) Alibaba, Inc. and its affiliates. +from .agent_tool import AgentTool from .code import CodeExecutionTool, SandboxManagerFactory from .filesystem_tool import FileSystemTool from .mcp_client import MCPClient diff --git a/ms_agent/tools/agent_tool.py b/ms_agent/tools/agent_tool.py new file mode 100644 index 000000000..a3a2ec319 --- /dev/null +++ b/ms_agent/tools/agent_tool.py @@ -0,0 +1,330 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +import os +import uuid +from collections import defaultdict +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Union + +import json +from ms_agent.agent.loader import AgentLoader +from ms_agent.llm.utils import Message, Tool +from ms_agent.tools.base import ToolBase +from ms_agent.utils import get_logger +from omegaconf import DictConfig, ListConfig, OmegaConf + +logger = get_logger() + + +def _to_container(value: Any) -> Any: + if isinstance(value, DictConfig): + return OmegaConf.to_container(value, resolve=True) + if isinstance(value, ListConfig): + return OmegaConf.to_container(value, resolve=True) + return value + + +@dataclass +class _AgentToolSpec: + tool_name: str + description: str + parameters: Dict[str, Any] + config_path: Optional[str] + inline_config: Optional[Dict[str, Any]] + server_name: str + tag_prefix: str + input_mode: str + request_field: Optional[str] + input_template: Optional[str] + output_mode: str + max_output_chars: int + trust_remote_code: Optional[bool] + env: Optional[Dict[str, str]] + + +class AgentTool(ToolBase): + """Expose existing ms-agent agents as callable tools.""" + + DEFAULT_SERVER = 'agent_tools' + + def __init__(self, config: DictConfig, **kwargs): + super().__init__(config) + self._trust_remote_code = kwargs.get('trust_remote_code', True) + self._specs: Dict[str, _AgentToolSpec] = {} + self._server_tools: Dict[str, List[Tool]] = {} + self._load_specs() + + @property + def enabled(self) -> bool: + return bool(self._specs) + + def _load_specs(self): + tools_cfg = getattr(self.config, 'tools', DictConfig({})) + agent_tools_cfg = getattr(tools_cfg, 'agent_tools', None) + if agent_tools_cfg is None: + return + + if isinstance(agent_tools_cfg, DictConfig) and hasattr( + agent_tools_cfg, 'definitions'): + definitions = agent_tools_cfg.definitions + server_name = getattr(agent_tools_cfg, 'server_name', + self.DEFAULT_SERVER) + else: + definitions = agent_tools_cfg + server_name = self.DEFAULT_SERVER + + definitions_list: List[Any] + if isinstance(definitions, DictConfig): + definitions_list = [definitions] + elif isinstance(definitions, ListConfig): + definitions_list = list(definitions) + elif isinstance(definitions, list): + definitions_list = definitions + else: + logger.warning('agent_tools configuration is not iterable; skip.') + return + + for idx, spec_cfg in enumerate(definitions_list): + spec = self._build_spec(spec_cfg, server_name, idx) + if spec is None: + continue + if spec.tool_name in self._specs: + logger.warning( + 'Duplicate agent tool name detected: %s, overriding previous definition.', + spec.tool_name) + self._specs[spec.tool_name] = spec + + self._build_server_index() + + def _build_spec(self, cfg: Union[DictConfig, Dict[str, Any]], + default_server, idx: int) -> Optional[_AgentToolSpec]: + cfg = cfg or {} + cfg = cfg if isinstance(cfg, DictConfig) else DictConfig(cfg) + tool_name = getattr(cfg, 'tool_name', None) or getattr( + cfg, 'name', None) + if not tool_name: + logger.warning( + 'agent_tools[%s] missing tool_name/name field, skip.', idx) + return None + + agent_cfg = getattr(cfg, 'agent', None) + config_path = getattr(cfg, 'config_path', None) + inline_cfg = getattr(cfg, 'config', None) + if agent_cfg is not None: + config_path = getattr(agent_cfg, 'config_path', config_path) + inline_cfg = getattr(agent_cfg, 'config', inline_cfg) + inline_cfg = _to_container( + inline_cfg) if inline_cfg is not None else None + + if not config_path and inline_cfg is None: + logger.warning( + 'agent_tools[%s] (%s) missing config_path/config definition.', + idx, tool_name) + return None + + description = getattr(cfg, 'description', + f'Invoke agent "{tool_name}" as a tool.') + parameters = getattr(cfg, 'parameters', None) + if parameters is None: + parameters = { + 'type': 'object', + 'properties': { + 'request': { + 'type': + 'string', + 'description': + f'Task description forwarded to the sub-agent {tool_name}.' + }, + }, + 'required': ['request'], + 'additionalProperties': True, + } + else: + parameters = _to_container(parameters) + + tag_prefix = getattr( + cfg, 'tag_prefix', + f'{getattr(self.config, "tag", "agent")}-{tool_name}-') + + request_field = getattr(cfg, 'request_field', 'request') + input_template = getattr(cfg, 'input_template', None) + input_mode = getattr(cfg, 'input_mode', 'text') + output_mode = getattr(cfg, 'output_mode', 'final_message') + max_chars = int(getattr(cfg, 'max_output_chars', 5000)) + server_name = getattr(cfg, 'server_name', default_server) + trust_remote_code = getattr(cfg, 'trust_remote_code', None) + + env_cfg = getattr(cfg, 'env', None) + env_cfg = _to_container(env_cfg) if env_cfg is not None else None + + if config_path and not os.path.isabs(config_path): + base_dir = getattr(self.config, 'local_dir', None) + if base_dir: + config_path = os.path.normpath( + os.path.join(base_dir, config_path)) + + return _AgentToolSpec( + tool_name=tool_name, + description=description, + parameters=parameters, + config_path=config_path, + inline_config=inline_cfg, + server_name=server_name, + tag_prefix=tag_prefix, + input_mode=input_mode, + request_field=request_field, + input_template=input_template, + output_mode=output_mode, + max_output_chars=max_chars, + trust_remote_code=trust_remote_code, + env=env_cfg, + ) + + def _build_server_index(self): + server_map: Dict[str, List[Tool]] = {} + for spec in self._specs.values(): + server_map.setdefault(spec.server_name, []).append( + Tool( + tool_name=spec.tool_name, + server_name=spec.server_name, + description=spec.description, + parameters=spec.parameters, + )) + self._server_tools = server_map + + async def connect(self): + return None + + async def cleanup(self): + return None + + async def get_tools(self) -> Dict[str, Any]: + return self._server_tools + + async def call_tool(self, server_name: str, *, tool_name: str, + tool_args: dict) -> str: + if tool_name not in self._specs: + raise ValueError(f'Agent tool "{tool_name}" not registered.') + spec = self._specs[tool_name] + if spec.server_name != server_name: + raise ValueError( + f'Agent tool "{tool_name}" is not part of server "{server_name}".' + ) + + payload = self._build_payload(tool_args, spec) + agent = self._build_agent(spec) + messages = await self._run_agent(agent, payload) + return self._format_output(messages, spec) + + def _build_agent(self, spec: _AgentToolSpec): + if spec.inline_config is not None: + config_override = OmegaConf.create(spec.inline_config) + else: + config_override = None + + trust_remote_code = spec.trust_remote_code + if trust_remote_code is None: + trust_remote_code = self._trust_remote_code + + tag = f'{spec.tag_prefix}{uuid.uuid4().hex[:8]}' + agent = AgentLoader.build( + config_dir_or_id=spec.config_path, + config=config_override, + env=spec.env, + tag=tag, + trust_remote_code=trust_remote_code, + ) + + generation_cfg = getattr(agent.config, 'generation_config', + DictConfig({})) + # OmegaConf.update( + # generation_cfg, + # 'stream', + # False, + # merge=True, + # ) + agent.config.generation_config = generation_cfg + return agent + + async def _run_agent(self, agent, payload): + result = await agent.run(payload) + if hasattr(result, '__aiter__'): + history = None + async for chunk in result: + history = chunk + result = history + return result + + def _build_payload(self, tool_args: dict, spec: _AgentToolSpec): + if spec.input_mode == 'messages': + field = spec.request_field or 'messages' + raw_messages = tool_args.get(field) + if not isinstance(raw_messages, list): + raise ValueError( + f'Agent tool "{spec.tool_name}" expects "{field}" to be a list of messages.' + ) + return [ + Message( + role=msg.get('role', 'user'), + content=msg.get('content', ''), + tool_calls=msg.get('tool_calls', []), + tool_call_id=msg.get('tool_call_id'), + name=msg.get('name'), + reasoning_content=msg.get('reasoning_content', ''), + ) for msg in raw_messages # TODO: Change role to user or not + ] + + if spec.input_template: + template_args = defaultdict(lambda: '', tool_args) + try: + return spec.input_template.format_map(template_args) + except Exception as exc: + logger.warning( + 'Failed to render input template for tool %s: %s. Falling back to JSON payload.', + spec.tool_name, exc) + + field = spec.request_field or 'request' + if field in tool_args and isinstance(tool_args[field], str): + return tool_args[field] + + return json.dumps(tool_args, ensure_ascii=False, indent=2) + + def _format_output(self, messages: Any, spec: _AgentToolSpec) -> str: + if not isinstance(messages, list): + return self._truncate(str(messages), spec.max_output_chars) + + if spec.output_mode == 'history': + serialized = [self._serialize_message(msg) for msg in messages] + return self._truncate( + json.dumps(serialized, ensure_ascii=False, indent=2), + spec.max_output_chars) + + if spec.output_mode == 'raw_json': + serialized = [msg.to_dict() for msg in messages] # type: ignore + return self._truncate( + json.dumps(serialized, ensure_ascii=False), + spec.max_output_chars) + + # Default: return final assistant message text + for msg in reversed(messages): + if getattr(msg, 'role', '') == 'assistant': + return self._truncate(msg.content or '', spec.max_output_chars) + + return self._truncate(messages[-1].content or '', + spec.max_output_chars) + + def _serialize_message(self, message: Message) -> Dict[str, Any]: + data = message.to_dict() + if data.get('tool_calls'): + for call in data['tool_calls']: + if isinstance(call.get('arguments'), dict): + call['arguments'] = json.dumps( + call['arguments'], ensure_ascii=False) + return data + + @staticmethod + def _truncate(text: str, limit: int) -> str: + if limit <= 0: + return text + if len(text) <= limit: + return text + return text[:limit] + '\n\n[AgentTool truncated output]' diff --git a/ms_agent/tools/tool_manager.py b/ms_agent/tools/tool_manager.py index 5dae9d331..9cc84df63 100644 --- a/ms_agent/tools/tool_manager.py +++ b/ms_agent/tools/tool_manager.py @@ -10,6 +10,7 @@ import json from ms_agent.llm.utils import Tool, ToolCall +from ms_agent.tools.agent_tool import AgentTool from ms_agent.tools.base import ToolBase from ms_agent.tools.code import CodeExecutionTool, LocalCodeExecutionTool from ms_agent.tools.filesystem_tool import FileSystemTool @@ -66,6 +67,12 @@ def __init__(self, if hasattr(config, 'tools') and hasattr(config.tools, 'financial_data_fetcher'): self.extra_tools.append(FinancialDataFetcher(config)) + if hasattr(config, 'tools') and getattr(config.tools, 'agent_tools', + None): + agent_tool = AgentTool( + config, trust_remote_code=self.trust_remote_code) + if agent_tool.enabled: + self.extra_tools.append(agent_tool) self.tool_call_timeout = getattr(config, 'tool_call_timeout', TOOL_CALL_TIMEOUT) local_dir = self.config.local_dir if hasattr(self.config, From b54576b5a59791f4f1e058bbbab872ec07aeed0e Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Wed, 26 Nov 2025 18:58:11 +0800 Subject: [PATCH 7/9] support timeout interruption for request handling in the docling processing pipeline --- ms_agent/tools/docling/doc_loader.py | 12 +++++++++--- ms_agent/tools/docling/patches.py | 16 ++++++++++++++++ ms_agent/utils/utils.py | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ms_agent/tools/docling/doc_loader.py b/ms_agent/tools/docling/doc_loader.py index 18c4bdfec..5daf1652b 100644 --- a/ms_agent/tools/docling/doc_loader.py +++ b/ms_agent/tools/docling/doc_loader.py @@ -1,7 +1,9 @@ # flake8: noqa +# yapf: disable import ast import os from typing import Dict, Iterator, List, Optional, Tuple, Union +from unittest.mock import patch as mock_patch from docling.backend.html_backend import HTMLDocumentBackend from docling.datamodel.accelerator_options import AcceleratorOptions @@ -21,7 +23,8 @@ download_models_pic_classifier_ms, html_handle_figure, html_handle_image, - patch_easyocr_models) + patch_easyocr_models, + requests_get_with_timeout) from ms_agent.utils.logger import get_logger from ms_agent.utils.patcher import patch from ms_agent.utils.utils import normalize_url_or_file, txt_to_html @@ -196,9 +199,10 @@ def check_url_valid(url: tuple[int, str]) -> tuple[int, str] | None: return None # Try to send a HEAD request to check if the URL is accessible - response = requests.head(_url, timeout=10) + response = requests.head(_url, timeout=(10, 25)) if response.status_code >= 400: - response = requests.get(_url, stream=True, timeout=10) + response = requests.get( + _url, stream=True, timeout=(10, 25)) if response.status_code >= 400: logger.warning( f'URL returned error status {response.status_code}: {_url}' @@ -304,6 +308,8 @@ def _postprocess(doc: DoclingDocument) -> Union[DoclingDocument, None]: download_models_pic_classifier_ms) @patch(HTMLDocumentBackend, 'handle_image', html_handle_image) @patch(HTMLDocumentBackend, 'handle_figure', html_handle_figure) + @mock_patch('docling_core.utils.file.requests.get', + requests_get_with_timeout) def load( self, urls_or_files: list[str], diff --git a/ms_agent/tools/docling/patches.py b/ms_agent/tools/docling/patches.py index ac4f45521..b2ec77afd 100644 --- a/ms_agent/tools/docling/patches.py +++ b/ms_agent/tools/docling/patches.py @@ -1,4 +1,5 @@ # flake8: noqa +import sys from pathlib import Path from bs4 import Tag @@ -170,3 +171,18 @@ def patch_easyocr_models(): 'url'] = 'https://modelscope.cn/models/ms-agent/kannada_g2/resolve/master/kannada_g2.zip' recognition_models['gen2']['cyrillic_g2'][ 'url'] = 'https://modelscope.cn/models/ms-agent/cyrillic_g2/resolve/master/cyrillic_g2.zip' + + +def requests_get_with_timeout( + *args, + _original_requests_get=sys.modules['requests'].get, + **kwargs +): # yapf: disable + """ + Wrapper for requests.get that enforces a default timeout if none is provided. + This is used to patch docling_core.utils.file.requests.get only. + """ + if 'timeout' not in kwargs or kwargs['timeout'] is None: + kwargs['timeout'] = (10, 30) + + return _original_requests_get(*args, **kwargs) diff --git a/ms_agent/utils/utils.py b/ms_agent/utils/utils.py index a2a7da3ee..4c8cd45ae 100644 --- a/ms_agent/utils/utils.py +++ b/ms_agent/utils/utils.py @@ -375,7 +375,7 @@ def load_image_from_url_to_pil(url: str) -> 'Image.Image': """ from PIL import Image try: - response = requests.get(url) + response = requests.get(url, timeout=(10, 25)) # Raise an HTTPError for bad responses (4xx or 5xx) response.raise_for_status() image_bytes = BytesIO(response.content) From 9c1df9cc39b68fc6298280d60256c7b2a8f42f6f Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Tue, 9 Dec 2025 21:53:25 +0800 Subject: [PATCH 8/9] support jina_reader --- ms_agent/tools/jina_reader.py | 138 ++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 ms_agent/tools/jina_reader.py diff --git a/ms_agent/tools/jina_reader.py b/ms_agent/tools/jina_reader.py new file mode 100644 index 000000000..d8962f617 --- /dev/null +++ b/ms_agent/tools/jina_reader.py @@ -0,0 +1,138 @@ +import asyncio +import random +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Dict, List, Optional +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +DEFAULT_HEADERS: Dict[str, str] = { + 'User-Agent': + 'Mozilla/5.0 (compatible; ms-agent/1.0; +https://example.com)', + 'Accept': 'text/plain; charset=utf-8', + 'Accept-Language': 'en-US,en;q=0.9', +} + + +@dataclass +class JinaReaderConfig: + base_endpoint: str = 'https://r.jina.ai/' + timeout: float = 30.0 + retries: int = 3 + backoff_base: float = 0.8 + backoff_max: float = 8.0 + headers: Dict[str, + str] = field(default_factory=lambda: DEFAULT_HEADERS.copy()) + + +def _build_reader_url(target_url: str, base_endpoint: str) -> str: + encoded_target = quote(target_url, safe=":/?&=%#@!$'*+,;[]()") + base = base_endpoint if base_endpoint.endswith( + '/') else f'{base_endpoint}/' + return f'{base}{encoded_target}' + + +def _postprocess_text(raw_text: str) -> str: + """ + Lightweight cleanup suitable for LLM consumption. + - Normalize line breaks + - Collapse excessive blank lines + - Trim leading/trailing whitespace + """ + if not raw_text: + return '' + text = raw_text.replace('\r\n', '\n').replace('\r', '\n') + # Collapse 3+ consecutive blank lines down to 2 + while '\n\n\n' in text: + text = text.replace('\n\n\n', '\n\n') + return text.strip() + + +def fetch_single_text(url: str, config: JinaReaderConfig) -> str: + """ + Synchronous fetch of a single URL via Jina Reader with retry/backoff and postprocessing. + """ + request_url = _build_reader_url(url, config.base_endpoint) + attempt = 0 + while True: + attempt += 1 + try: + req = Request(request_url, headers=config.headers) + with urlopen(req, timeout=config.timeout) as resp: + data = resp.read() + return _postprocess_text( + data.decode('utf-8', errors='replace')) + except HTTPError as e: + # Retry on 429 and 5xx, otherwise fail fast + status = getattr(e, 'code', None) + if status in (429, 500, 502, 503, + 504) and attempt <= config.retries: + sleep_s = min(config.backoff_max, + config.backoff_base * (2**(attempt - 1))) + sleep_s *= random.uniform(0.7, 1.4) + time.sleep(sleep_s) + continue + return '' + except URLError: + if attempt <= config.retries: + sleep_s = min(config.backoff_max, + config.backoff_base * (2**(attempt - 1))) + sleep_s *= random.uniform(0.7, 1.4) + time.sleep(sleep_s) + continue + return '' + except Exception: + # Unknown error; do not loop excessively + if attempt <= config.retries: + sleep_s = min(config.backoff_max, + config.backoff_base * (2**(attempt - 1))) + sleep_s *= random.uniform(0.7, 1.4) + time.sleep(sleep_s) + continue + return '' + + +async def fetch_texts_via_jina( + urls: List[str], + config: Optional[JinaReaderConfig] = None, + semaphore: Optional[asyncio.Semaphore] = None, + executor: Optional[ThreadPoolExecutor] = None) -> List[str]: + """ + Asynchronously fetch a list of URLs via Jina Reader. + Allows caller-provided concurrency controls (semaphore/executor) to integrate with pipeline resource management. + """ + if not urls: + return [] + cfg = config or JinaReaderConfig() + loop = asyncio.get_event_loop() + + local_sem = semaphore or asyncio.Semaphore(8) + + async def _bound(u: str) -> str: + async with local_sem: + return await loop.run_in_executor(executor, fetch_single_text, u, + cfg) + + tasks = [_bound(u) for u in urls] + results = await asyncio.gather(*tasks, return_exceptions=True) + texts: List[str] = [] + for r in results: + if isinstance(r, Exception): + continue + if isinstance(r, str) and r.strip(): + texts.append(r) + return texts + + +if __name__ == '__main__': + urls = [ + 'https://arxiv.org/pdf/2408.09869', + 'https://github.com/modelscope/evalscope', + 'https://www.news.cn/talking/20250530/691e47a5d1a24c82bfa2371d1af40630/c.html', + ] + texts = asyncio.run(fetch_texts_via_jina(urls)) + for text in texts: + print(text) + print('-' * 100) From 19c876e89407f2d63b057e541fbf14e0b6bd5ba4 Mon Sep 17 00:00:00 2001 From: alcholiclg Date: Tue, 13 Jan 2026 23:42:11 +0800 Subject: [PATCH 9/9] fix tools config --- projects/fin_research/aggregator.yaml | 7 +++++-- projects/fin_research/collector.yaml | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/projects/fin_research/aggregator.yaml b/projects/fin_research/aggregator.yaml index 319566d02..2efcabfe8 100644 --- a/projects/fin_research/aggregator.yaml +++ b/projects/fin_research/aggregator.yaml @@ -129,8 +129,11 @@ prompt: tools: file_system: mcp: false - exclude: - - create_directory + include: + - read_file + - write_file + - list_files + - delete_file_or_dir spec_loader: mcp: false plugins: diff --git a/projects/fin_research/collector.yaml b/projects/fin_research/collector.yaml index 5eb27f228..fde543744 100644 --- a/projects/fin_research/collector.yaml +++ b/projects/fin_research/collector.yaml @@ -147,10 +147,9 @@ tools: max_concurrent: 1 # Maximum concurrent requests (reduced to work with thread semaphore) file_system: mcp: false - exclude: - - create_directory - - read_file - - list_files + include: + - write_file + - delete_file_or_dir handler: time_handler