Merge branch 'master' of github.com:Mintplex-Labs/anything-llm into r…

…ender

collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js

@@ -45,30 +45,20 @@ class GitLabRepoLoader {
   }
 
   #validGitlabUrl() {
-    const UrlPattern = require("url-pattern");
     const validPatterns = [
-      new UrlPattern("https\\://gitlab.com/(:author*)/(:project(*))", {
-        segmentValueCharset: "a-zA-Z0-9-._~%+",
-      }),
+      /https:\/\/gitlab\.com\/(?<author>[^\/]+)\/(?<project>.*)/,
       // This should even match the regular hosted URL, but we may want to know
       // if this was a hosted GitLab (above) or a self-hosted (below) instance
       // since the API interface could be different.
-      new UrlPattern(
-        "(:protocol(http|https))\\://(:hostname*)/(:author*)/(:project(*))",
-        {
-          segmentValueCharset: "a-zA-Z0-9-._~%+",
-        }
-      ),
+      /(http|https):\/\/[^\/]+\/(?<author>[^\/]+)\/(?<project>.*)/,
     ];
 
-    let match = null;
-    for (const pattern of validPatterns) {
-      if (match !== null) continue;
-      match = pattern.match(this.repo);
-    }
-    if (!match) return false;
-    const { author, project } = match;
+    const match = validPatterns
+      .find((pattern) => this.repo.match(pattern)?.groups)
+      ?.exec(this.repo);
+    if (!match?.groups) return false;
 
+    const { author, project } = match.groups;
     this.projectId = encodeURIComponent(`${author}/${project}`);
     this.apiBase = new URL(this.repo).origin;
     this.author = author;

locales/README.zh-CN.md

@@ -47,7 +47,7 @@
 
 AnythingLLM是一个全栈应用程序，您可以使用现成的商业大语言模型或流行的开源大语言模型，再结合向量数据库解决方案构建一个私有ChatGPT，不再受制于人：您可以本地运行，也可以远程托管，并能够与您提供的任何文档智能聊天。
 
-AnythingLLM将您的文档划分为称为`workspaces` (工作区)的对象。工作区的功能类似于线程，同时增加了文档的容器化，。工作区可以共享文档，但工作区之间的内容不会互相干扰或污染，因此您可以保持每个工作区的上下文清晰。
+AnythingLLM将您的文档划分为称为`workspaces` (工作区)的对象。工作区的功能类似于线程，同时增加了文档的容器化。工作区可以共享文档，但工作区之间的内容不会互相干扰或污染，因此您可以保持每个工作区的上下文清晰。
 
 AnythingLLM的一些酷炫特性
 
@@ -56,11 +56,11 @@ AnythingLLM的一些酷炫特性
 - [为您的网站定制的可嵌入聊天窗口](./embed/README.md)
 - 支持多种文档类型（PDF、TXT、DOCX等）
 - 通过简单的用户界面管理向量数据库中的文档
-- 两种对话模式：`聊天`和`查询`。聊天模式保留先前的对话记录。查询模式则是是针对您的文档做简单问答
+- 两种对话模式：`聊天`和`查询`。聊天模式保留先前的对话记录。查询模式则是针对您的文档做简单问答
 - 聊天中会提供所引用的相应文档内容
 - 100%云部署就绪。
 - “部署你自己的LLM模型”。
-- 管理超大文档时高效、低耗。只需要一次就可以嵌入（Embedding)一个庞大的文档或文字记录。比其他文档聊天机器人解决方案节省90%的成本。
+- 管理超大文档时高效、低耗。只需要一次就可以嵌入（Embedding）一个庞大的文档或文字记录。比其他文档聊天机器人解决方案节省90%的成本。
 - 全套的开发人员API，用于自定义集成！
 
 ### 支持的LLM、嵌入模型、转录模型和向量数据库
@@ -102,6 +102,18 @@ AnythingLLM的一些酷炫特性
 - [AnythingLLM内置](https://github.com/Mintplex-Labs/anything-llm/tree/master/server/storage/models#audiovideo-transcription) （默认）
 - [OpenAI](https://openai.com/)
 
+**TTS (文本转语音) 支持：**
+
+- 浏览器内置（默认）
+- [PiperTTSLocal - 在浏览器中运行](https://github.com/rhasspy/piper)
+- [OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech/voice-options)
+- [ElevenLabs](https://elevenlabs.io/)
+- 任何与 OpenAI 兼容的 TTS 服务
+
+**STT (语音转文本) 支持：**
+
+- 浏览器内置（默认）
+
 **支持的向量数据库：**
 
 - [LanceDB](https://github.com/lancedb/lancedb) （默认）
@@ -169,7 +181,7 @@ Mintplex Labs和社区维护了许多部署方法、脚本和模板，您可以
 
 ### 怎样关闭
 
-通过在服务器或 docker 的 `.env` 设置中将 `DISABLE_TELEMETRY` 设置为 “true” 来选择退出 Telemetry 远程信息收集功能。您也可以进入 AnythingLLM 应用 >>> 侧边栏最下方 >>> `隐私和数据` （Privacy&Data) >>> 找到最下方的 Anonymous Telemetry Enabled，点击绿色按钮让它变灰色，从而禁用信息收集功能。
+通过在服务器或 docker 的 `.env` 设置中将 `DISABLE_TELEMETRY` 设置为 “true” 来选择退出 Telemetry 远程信息收集功能。您也可以进入 AnythingLLM 应用 >>> 侧边栏最下方 >>> `隐私和数据` （Privacy&Data） >>> 找到最下方的 Anonymous Telemetry Enabled，点击绿色按钮让它变灰色，从而禁用信息收集功能。
 
 ### 你们跟踪收集哪些信息？
 
@@ -200,7 +212,7 @@ Mintplex Labs和社区维护了许多部署方法、脚本和模板，您可以
 
 ---
 
-版权所有 © 2024 [Mintplex Labs][profile-link]。<br />
+版权所有 © 2025 [Mintplex Labs][profile-link]。<br />
 本项目采用[MIT](./LICENSE)许可证。
 
 <!-- LINK GROUP -->

server/utils/agentFlows/executor.js

@@ -10,12 +10,14 @@ const { Telemetry } = require("../../models/telemetry");
 class FlowExecutor {
   constructor() {
     this.variables = {};
-    this.introspect = () => {}; // Default no-op introspect
-    this.logger = console.info; // Default console.info
+    this.introspect = (...args) => console.log("[introspect] ", ...args);
+    this.logger = console.info;
+    this.aibitat = null;
   }
 
-  attachLogging(introspectFn, loggerFn) {
-    this.introspect = introspectFn || (() => {});
+  attachLogging(introspectFn = null, loggerFn = null) {
+    this.introspect =
+      introspectFn || ((...args) => console.log("[introspect] ", ...args));
     this.logger = loggerFn || console.info;
   }
 
@@ -54,8 +56,7 @@ class FlowExecutor {
       introspect: this.introspect,
       variables: this.variables,
       logger: this.logger,
-      model: process.env.LLM_PROVIDER_MODEL || "gpt-4",
-      provider: process.env.LLM_PROVIDER || "openai",
+      aibitat: this.aibitat,
     };
 
     switch (step.type) {
@@ -101,13 +102,13 @@ class FlowExecutor {
     return result;
   }
 
-  // Execute entire flow
-  async executeFlow(
-    flow,
-    initialVariables = {},
-    introspectFn = null,
-    loggerFn = null
-  ) {
+  /**
+   * Execute entire flow
+   * @param {Object} flow - The flow to execute
+   * @param {Object} initialVariables - Initial variables for the flow
+   * @param {Object} aibitat - The aibitat instance from the agent handler
+   */
+  async executeFlow(flow, initialVariables = {}, aibitat) {
     await Telemetry.sendTelemetry("agent_flow_execution_started");
 
     // Initialize variables with both initial values and any passed-in values
@@ -119,7 +120,8 @@ class FlowExecutor {
       ...initialVariables, // This will override any default values with passed-in values
     };
 
-    this.attachLogging(introspectFn, loggerFn);
+    this.aibitat = aibitat;
+    this.attachLogging(aibitat?.introspect, aibitat?.handlerProps?.log);
     const results = [];
 
     for (const step of flow.config.steps) {

server/utils/agentFlows/executors/api-call.js

@@ -8,8 +8,8 @@ const { safeJsonParse } = require("../../http");
  */
 async function executeApiCall(config, context) {
   const { url, method, headers = [], body, bodyType, formData } = config;
-  const { introspect } = context;
-
+  const { introspect, logger } = context;
+  logger(`\x1b[43m[AgentFlowToolExecutor]\x1b[0m - executing API Call block`);
   introspect(`Making ${method} request to external API...`);
 
   const requestConfig = {

server/utils/agentFlows/executors/llm-instruction.js

@@ -8,23 +8,28 @@ const AIbitat = require("../../agents/aibitat");
  */
 async function executeLLMInstruction(config, context) {
   const { instruction, inputVariable, resultVariable } = config;
-  const { introspect, variables, logger } = context;
-
+  const { introspect, variables, logger, aibitat } = context;
+  logger(
+    `\x1b[43m[AgentFlowToolExecutor]\x1b[0m - executing LLM Instruction block`
+  );
   introspect(`Processing data with LLM instruction...`);
+
   if (!variables[inputVariable]) {
     logger(`Input variable ${inputVariable} not found`);
     throw new Error(`Input variable ${inputVariable} not found`);
   }
 
   try {
+    logger(
+      `Sending request to LLM (${aibitat.defaultProvider.provider}::${aibitat.defaultProvider.model})`
+    );
     introspect(`Sending request to LLM...`);
 
     // Ensure the input is a string since we are sending it to the LLM direct as a message
     let input = variables[inputVariable];
     if (typeof input === "object") input = JSON.stringify(input);
     if (typeof input !== "string") input = String(input);
 
-    const aibitat = new AIbitat();
     const provider = aibitat.getProviderForConfig(aibitat.defaultProvider);
     const completion = await provider.complete([
       {

server/utils/agentFlows/executors/web-scraping.js

@@ -11,13 +11,15 @@ const { summarizeContent } = require("../../agents/aibitat/utils/summarize");
  */
 async function executeWebScraping(config, context) {
   const { url, captureAs = "text" } = config;
-  const { introspect, model, provider } = context;
+  const { introspect, logger, aibitat } = context;
+  logger(
+    `\x1b[43m[AgentFlowToolExecutor]\x1b[0m - executing Web Scraping block`
+  );
 
   if (!url) {
     throw new Error("URL is required for web scraping");
   }
 
-  // Remap the captureAs to the correct mode for the CollectorApi
   const captureMode = captureAs === "querySelector" ? "html" : captureAs;
   introspect(`Scraping the content of ${url} as ${captureAs}`);
   const { success, content } = await new CollectorApi()
@@ -33,29 +35,30 @@ async function executeWebScraping(config, context) {
   }
 
   introspect(`Successfully scraped content from ${url}`);
-
   if (!content || content?.length === 0) {
+    introspect("There was no content to be collected or read.");
     throw new Error("There was no content to be collected or read.");
   }
 
-  const tokenCount = new TokenManager(model).countFromString(content);
-  const contextLimit = Provider.contextLimit(provider, model);
-
-  if (tokenCount < contextLimit) {
-    return content;
-  }
+  const tokenCount = new TokenManager(
+    aibitat.defaultProvider.model
+  ).countFromString(content);
+  const contextLimit = Provider.contextLimit(
+    aibitat.defaultProvider.provider,
+    aibitat.defaultProvider.model
+  );
+  if (tokenCount < contextLimit) return content;
 
   introspect(
-    `This page's content is way too long. I will summarize it right now.`
+    `This page's content is way too long (${tokenCount} tokens). I will summarize it right now.`
   );
   const summary = await summarizeContent({
-    provider,
-    model,
+    provider: aibitat.defaultProvider.provider,
+    model: aibitat.defaultProvider.model,
     content,
   });
 
   introspect(`Successfully summarized content`);
-
   return summary;
 }
 

server/utils/agentFlows/index.js

@@ -141,25 +141,14 @@ class AgentFlows {
    * Execute a flow by UUID
    * @param {string} uuid - The UUID of the flow to execute
    * @param {Object} variables - Initial variables for the flow
-   * @param {Function} introspectFn - Function to introspect the flow
-   * @param {Function} loggerFn - Function to log the flow
+   * @param {Object} aibitat - The aibitat instance from the agent handler
    * @returns {Promise<Object>} Result of flow execution
    */
-  static async executeFlow(
-    uuid,
-    variables = {},
-    introspectFn = null,
-    loggerFn = null
-  ) {
+  static async executeFlow(uuid, variables = {}, aibitat = null) {
     const flow = AgentFlows.loadFlow(uuid);
     if (!flow) throw new Error(`Flow ${uuid} not found`);
     const flowExecutor = new FlowExecutor();
-    return await flowExecutor.executeFlow(
-      flow,
-      variables,
-      introspectFn,
-      loggerFn
-    );
+    return await flowExecutor.executeFlow(flow, variables, aibitat);
   }
 
   /**
@@ -210,12 +199,7 @@ class AgentFlows {
             },
             handler: async (args) => {
               aibitat.introspect(`Executing flow: ${flow.name}`);
-              const result = await AgentFlows.executeFlow(
-                uuid,
-                args,
-                aibitat.introspect,
-                aibitat.handlerProps.log
-              );
+              const result = await AgentFlows.executeFlow(uuid, args, aibitat);
               if (!result.success) {
                 aibitat.introspect(
                   `Flow failed: ${result.results[0]?.error || "Unknown error"}`

server/utils/agents/aibitat/index.js

@@ -798,7 +798,7 @@ ${this.getHistory({ to: route.to })
 
       default:
         throw new Error(
-          `Unknown provider: ${config.provider}. Please use "openai"`
+          `Unknown provider: ${config.provider}. Please use a valid provider.`
         );
     }
   }

server/utils/agents/index.js

@@ -390,7 +390,7 @@ class AgentHandler {
       // Load flow plugin. This is marked by `@@flow_` in the array of functions to load.
       if (name.startsWith("@@flow_")) {
         const uuid = name.replace("@@flow_", "");
-        const plugin = AgentFlows.loadFlowPlugin(uuid);
+        const plugin = AgentFlows.loadFlowPlugin(uuid, this.aibitat);
         if (!plugin) {
           this.log(
             `Flow ${uuid} not found in flows directory. Skipping inclusion to agent cluster.`