plugin-hubtype-analytics & plugin-flow-builder: create and send ai agent event #BLT-1804

packages/botonic-core/src/models/ai-agents.ts

@@ -47,8 +47,9 @@ export interface RunResult {
   messages: AgenticOutputMessage[]
   toolsExecuted: string[]
   exit: boolean
+  error: boolean
   inputGuardrailTriggered: string[]
   outputGuardrailTriggered: string[]
 }
 
-export type InferenceResponse = RunResult | undefined
+export type InferenceResponse = RunResult

packages/botonic-core/src/models/hubtype-analytics.ts

@@ -1,4 +1,5 @@
 export enum EventAction {
+  AiAgent = 'ai_agent',
   FeedbackCase = 'feedback_case',
   FeedbackMessage = 'feedback_message',
   FeedbackConversation = 'feedback_conversation',
@@ -112,6 +113,22 @@ export interface EventKnowledgeBase extends HtBaseEventProps {
   userInput: string
 }
 
+export interface EventAiAgent extends HtBaseEventProps {
+  action: EventAction.AiAgent
+  flowThreadId: string
+  flowId: string
+  flowName: string
+  flowNodeId: string
+  flowNodeContentId: string
+  flowNodeIsMeaningful: boolean
+  toolsExecuted: string[]
+  inputGuardrailTriggered: string[]
+  outputGuardrailTriggered: string[]
+  exit: boolean
+  error: boolean
+  messageId: string
+}
+
 export enum KnowledgebaseFailReason {
   NoKnowledge = 'no_knowledge',
   Hallucination = 'hallucination',

packages/botonic-plugin-ai-agents/src/index.ts

@@ -60,9 +60,14 @@ export default class BotonicPluginAiAgents implements Plugin {
       return await runner.run(messages, context)
     } catch (error) {
       console.error('error plugin returns undefined', error)
-      // Here we can return a InferenceResponse as a exit
-      // but indicate that the inference failed
-      return undefined
+      return {
+        messages: [],
+        toolsExecuted: [],
+        exit: true,
+        error: true,
+        inputGuardrailTriggered: [],
+        outputGuardrailTriggered: [],
+      }
     }
   }
 

packages/botonic-plugin-ai-agents/src/runner.ts

@@ -53,6 +53,7 @@ export class AIAgentRunner {
             ) as AgenticOutputMessage[]),
         toolsExecuted,
         exit: hasExit,
+        error: false,
         inputGuardrailTriggered: [],
         outputGuardrailTriggered: [],
       }
@@ -62,6 +63,7 @@ export class AIAgentRunner {
           messages: [],
           toolsExecuted: [],
           exit: true,
+          error: false,
           inputGuardrailTriggered: error.result.output.outputInfo,
           outputGuardrailTriggered: [],
         }

packages/botonic-plugin-flow-builder/src/action/ai-agent.ts

@@ -1,5 +1,14 @@
-import { FlowContent } from '../content-fields'
-import { FlowAiAgent } from '../content-fields/flow-ai-agent'
+import {
+  BotContext,
+  EventAction,
+  EventAiAgent,
+  InferenceResponse,
+} from '@botonic/core'
+
+import { FlowAiAgent, FlowContent } from '../content-fields'
+import { HtNodeWithContent } from '../content-fields/hubtype-fields'
+import { getFlowBuilderPlugin } from '../helpers'
+import { trackEvent } from '../tracking'
 import { GuardrailRule } from '../types'
 import { FlowBuilderContext } from './index'
 
@@ -46,6 +55,7 @@ export async function getContentsByAiAgent({
   if (!aiAgentResponse) {
     return []
   }
+  trackAiAgentResponse(aiAgentResponse, request, aiAgentContent)
 
   if (aiAgentResponse.exit) {
     return []
@@ -55,3 +65,34 @@ export async function getContentsByAiAgent({
 
   return contents
 }
+
+async function trackAiAgentResponse(
+  aiAgentResponse: InferenceResponse,
+  request: BotContext,
+  aiAgentContent: FlowAiAgent
+) {
+  const flowBuilderPlugin = getFlowBuilderPlugin(request.plugins)
+  const flowId = flowBuilderPlugin.cmsApi.getNodeById<HtNodeWithContent>(
+    aiAgentContent.id
+  ).flow_id
+  const flowName = flowBuilderPlugin.getFlowName(flowId)
+
+  const event: EventAiAgent = {
+    action: EventAction.AiAgent,
+    flowThreadId: request.session.flow_thread_id!,
+    flowId: flowId,
+    flowName: flowName,
+    flowNodeId: aiAgentContent.id,
+    flowNodeContentId: aiAgentContent.code,
+    flowNodeIsMeaningful: true,
+    toolsExecuted: aiAgentResponse?.toolsExecuted ?? [],
+    exit: aiAgentResponse?.exit ?? true,
+    inputGuardrailTriggered: aiAgentResponse?.inputGuardrailTriggered ?? [],
+    outputGuardrailTriggered: [], //aiAgentResponse.outputGuardrailTriggered,
+    error: aiAgentResponse.error,
+    messageId: request.input.message_id!,
+  }
+  const { action, ...eventArgs } = event
+
+  await trackEvent(request, action, eventArgs)
+}

packages/botonic-plugin-flow-builder/src/tracking.ts

@@ -29,7 +29,10 @@ export async function trackFlowContent(
   const cmsApi = flowBuilderPlugin.cmsApi
   for (const content of contents) {
     const nodeContent = cmsApi.getNodeById<HtNodeWithContent>(content.id)
-    if (nodeContent.type !== HtNodeWithContentType.KNOWLEDGE_BASE) {
+    if (
+      nodeContent.type !== HtNodeWithContentType.KNOWLEDGE_BASE &&
+      nodeContent.type !== HtNodeWithContentType.AI_AGENT
+    ) {
       const event = getContentEventArgs(request, nodeContent)
       const { action, ...eventArgs } = event
       await trackEvent(request, action, eventArgs)

packages/botonic-plugin-flow-builder/src/types.ts

@@ -1,13 +1,9 @@
 import {
-  AgenticOutputMessage,
   BotContext,
-  CarouselMessage,
   InferenceResponse,
   KnowledgeBasesResponse,
   PluginPreRequest,
   ResolvedPlugins,
-  TextMessage,
-  TextWithButtonsMessage,
 } from '@botonic/core'
 
 import { FlowContent } from './content-fields'

packages/botonic-plugin-flow-builder/tests/__mocks__/ai-agent.ts

@@ -1,14 +1,23 @@
-import { AgenticOutputMessage, InferenceResponse } from '@botonic/core'
+import { InferenceResponse } from '@botonic/core'
 
-export function mockAiAgentResponse(messages: AgenticOutputMessage[]) {
+export function mockAiAgentResponse({
+  messages = [],
+  toolsExecuted = [],
+  inputGuardrailTriggered = [],
+  outputGuardrailTriggered = [],
+  exit = false,
+  error = false,
+}: Partial<InferenceResponse>) {
   return jest.fn(() => {
     const response: InferenceResponse = {
       messages,
-      toolsExecuted: [],
-      exit: false,
-      inputGuardrailTriggered: [],
-      outputGuardrailTriggered: [],
+      toolsExecuted,
+      inputGuardrailTriggered,
+      outputGuardrailTriggered,
+      exit,
+      error,
     }
+
     return Promise.resolve(response)
   })
 }

packages/botonic-plugin-flow-builder/tests/ai-agent.test.ts

@@ -1,4 +1,4 @@
-import { INPUT } from '@botonic/core'
+import { InferenceResponse, INPUT } from '@botonic/core'
 import { describe, test } from '@jest/globals'
 
 import { FlowAiAgent, FlowText } from '../src'
@@ -12,17 +12,21 @@ describe('Check the contents returned by the plugin when it use an ai agent', ()
   process.env.NODE_ENV = ProcessEnvNodeEnvs.PRODUCTION
 
   test('When input match a keyword, the ai agent not respond', async () => {
+    const mockResponse: Partial<InferenceResponse> = {
+      messages: [
+        {
+          type: 'text',
+          content: {
+            text: 'Ai agent response',
+          },
+        },
+      ],
+    }
+
     const { contents, request } = await createFlowBuilderPluginAndGetContents({
       flowBuilderOptions: {
         flow: aiAgentTestFlow,
-        getAiAgentResponse: mockAiAgentResponse([
-          {
-            type: 'text',
-            content: {
-              text: 'Ai agent response',
-            },
-          },
-        ]),
+        getAiAgentResponse: mockAiAgentResponse(mockResponse),
       },
       requestArgs: {
         input: {
@@ -38,17 +42,21 @@ describe('Check the contents returned by the plugin when it use an ai agent', ()
   })
 
   test('When input not match a keyword or smart intent, the ai agent respond with a text message', async () => {
+    const mockResponse: Partial<InferenceResponse> = {
+      messages: [
+        {
+          type: 'text',
+          content: {
+            text: 'I can provide you with information about current temperatures, forecasts, and the probability of rain for your location. Just let me know where you are or where you’re interested in, and I’ll give you the details!',
+          },
+        },
+      ],
+    }
+
     const { contents } = await createFlowBuilderPluginAndGetContents({
       flowBuilderOptions: {
         flow: aiAgentTestFlow,
-        getAiAgentResponse: mockAiAgentResponse([
-          {
-            type: 'text',
-            content: {
-              text: 'I can provide you with information about current temperatures, forecasts, and the probability of rain for your location. Just let me know where you are or where you’re interested in, and I’ll give you the details!',
-            },
-          },
-        ]),
+        getAiAgentResponse: mockAiAgentResponse(mockResponse),
       },
       requestArgs: {
         input: {
@@ -69,24 +77,28 @@ describe('Check the contents returned by the plugin when it use an ai agent', ()
   })
 
   test('When input not match a keyword or smart intent, the ai agent respond with two messages, a text followed by a text with buttons', async () => {
+    const mockResponse: Partial<InferenceResponse> = {
+      messages: [
+        {
+          type: 'text',
+          content: {
+            text: 'First text message generated by the ai agent',
+          },
+        },
+        {
+          type: 'textWithButtons',
+          content: {
+            text: 'Second text message with buttons generated by the ai agent',
+            buttons: ['Button 1', 'Button 2'],
+          },
+        },
+      ],
+    }
+
     const { contents } = await createFlowBuilderPluginAndGetContents({
       flowBuilderOptions: {
         flow: aiAgentTestFlow,
-        getAiAgentResponse: mockAiAgentResponse([
-          {
-            type: 'text',
-            content: {
-              text: 'First text message generated by the ai agent',
-            },
-          },
-          {
-            type: 'textWithButtons',
-            content: {
-              text: 'Second text message with buttons generated by the ai agent',
-              buttons: ['Button 1', 'Button 2'],
-            },
-          },
-        ]),
+        getAiAgentResponse: mockAiAgentResponse(mockResponse),
       },
       requestArgs: {
         input: {
@@ -115,30 +127,34 @@ describe('Check the contents returned by the plugin when it use an ai agent', ()
   })
 
   test('When input not match a keyword or smart intent, the ai agent respond with two messages, a text followed by a carousel', async () => {
+    const mockResponse: Partial<InferenceResponse> = {
+      messages: [
+        {
+          type: 'text',
+          content: {
+            text: 'First text message generated by the ai agent',
+          },
+        },
+        {
+          type: 'carousel',
+          content: {
+            elements: [
+              {
+                title: 'Carousel element 1',
+                subtitle: 'Carousel element 1 subtitle',
+                image: 'https://via.placeholder.com/150',
+                button: { text: 'Button 1', url: 'https://www.google.com' },
+              },
+            ],
+          },
+        },
+      ],
+    }
+
     const { contents } = await createFlowBuilderPluginAndGetContents({
       flowBuilderOptions: {
         flow: aiAgentTestFlow,
-        getAiAgentResponse: mockAiAgentResponse([
-          {
-            type: 'text',
-            content: {
-              text: 'First text message generated by the ai agent',
-            },
-          },
-          {
-            type: 'carousel',
-            content: {
-              elements: [
-                {
-                  title: 'Carousel element 1',
-                  subtitle: 'Carousel element 1 subtitle',
-                  image: 'https://via.placeholder.com/150',
-                  button: { text: 'Button 1', url: 'https://www.google.com' },
-                },
-              ],
-            },
-          },
-        ]),
+        getAiAgentResponse: mockAiAgentResponse(mockResponse),
       },
       requestArgs: {
         input: {