feat: feedback and add chat to langfuse dataset (#163)

* feat: feedback

* support add chat to langfuse

* update

ddl/2-feedback.sql

@@ -1,16 +1,20 @@
-DROP TABLE knowledge_graph_feedback;
+DROP TABLE feedback;
 
-CREATE TABLE knowledge_graph_feedback
+CREATE TABLE feedback
 (
-    id           INTEGER      NOT NULL AUTO_INCREMENT,
-    trace_id     BINARY(16)   NOT NULL COMMENT 'Langfuse trace ID (UUID)',
-    detail       JSON         NOT NULL COMMENT 'Map, key is source URL, value is `like` or `dislike`.',
-    comment      TEXT         NOT NULL COMMENT 'Comments from user',
-    created_by   VARCHAR(32)  NOT NULL COMMENT 'User id',
-    created_at   DATETIME     NOT NULL COMMENT 'User submit feedback at',
-    reported_at  DATETIME     NULL COMMENT 'Reported to graph.tidb.ai',
-    report_error VARCHAR(512) NULL COMMENT 'Report failure reason if reporting failed.',
+    id                           INTEGER                  NOT NULL AUTO_INCREMENT,
+    chat_id                      INTEGER                  NOT NULL,
+    message_id                   INTEGER                  NOT NULL,
+    trace_id                     BINARY(16)               NOT NULL COMMENT 'Langfuse trace ID (UUID)',
+    action                       ENUM ('like', 'dislike') NOT NULL,
+    comment                      TEXT                     NOT NULL COMMENT 'Comments from user',
+    created_by                   VARCHAR(32)              NOT NULL COMMENT 'User id',
+    created_at                   DATETIME                 NOT NULL COMMENT 'User submit feedback at',
+    knowledge_graph_detail       JSON                     NOT NULL COMMENT 'Map, key is source URL, value is `like` or `dislike`.',
+    knowledge_graph_reported_at  DATETIME                 NULL COMMENT 'Reported to graph.tidb.ai',
+    knowledge_graph_report_error TEXT                     NULL COMMENT 'Report failure reason if reporting failed.',
     PRIMARY KEY (id),
     UNIQUE INDEX (trace_id, created_by),
-    INDEX (created_at, reported_at)
+    INDEX (created_at, knowledge_graph_reported_at),
+    INDEX (chat_id, message_id)
 );

package.json

@@ -61,7 +61,7 @@
     "hast-util-select": "^6.0.2",
     "hast-util-to-text": "^4.0.0",
     "kysely": "^0.27.2",
-    "langfuse": "^3.10.0",
+    "langfuse": "^3.11.2",
     "liquidjs": "^10.10.0",
     "llamaindex": "^0.3.10",
     "luxon": "^3.4.4",

src/app/(main)/(admin)/feedbacks/page.client.tsx

@@ -0,0 +1,53 @@
+'use client';
+
+import { AdminPageHeading } from '@/components/admin-page-heading';
+import { DataTableRemote } from '@/components/data-table-remote';
+import type { Feedback } from '@/core/repositories/feedback';
+import type { ColumnDef } from '@tanstack/react-table';
+import { createColumnHelper } from '@tanstack/table-core';
+import { format } from 'date-fns';
+import { LinkIcon, ThumbsDownIcon, ThumbsUpIcon } from 'lucide-react';
+import Link from 'next/link';
+
+export default function FeedbackPage () {
+  return (
+    <>
+      <AdminPageHeading title="Feedbacks" />
+      <DataTableRemote<Feedback & { chat_key: string, chat_title: string }, any>
+        idColumn="id"
+        api="/api/v1/feedbacks"
+        columns={columns}
+      />
+    </>
+  );
+}
+
+const helper = createColumnHelper<Feedback & { chat_key: string, chat_title: string }>();
+const columns: ColumnDef<Feedback & { chat_key: string, chat_title: string }, any>[] = [
+  helper.accessor('id', {}),
+  helper.accessor('chat_key', {
+    id: 'go_to_chat',
+    header: 'chat',
+    cell: (cell) => (
+      <Link href={`/c/${encodeURIComponent(cell.getValue())}`}>
+        <LinkIcon className="w-3 h-3 mr-1 inline-block" />
+        {cell.row.original.chat_title}
+      </Link>
+    ),
+  }),
+  helper.accessor('action', {
+    cell: (cell) => {
+      switch (cell.getValue()) {
+        case 'like':
+          return <span className="text-green-500 flex items-center gap-1"><ThumbsUpIcon className="w-3 h-3 inline-block" /> Like</span>;
+        case 'dislike':
+          return <span className="text-red-500 flex items-center gap-1"><ThumbsDownIcon className="w-3 h-3 inline-block" /> Dislike</span>;
+      }
+    },
+  }),
+  helper.accessor('comment', {}),
+  helper.accessor('created_by', {}),
+  helper.accessor('created_at', {
+    cell: cell => <time>{format(cell.getValue(), 'yyyy-MM-dd HH:mm:ss')}</time>,
+  }),
+];

src/app/(main)/(admin)/feedbacks/page.tsx

@@ -0,0 +1,12 @@
+import { authGuard } from '@/lib/auth-server';
+import FeedbackPage from './page.client';
+
+export default async function ServerDocumentsPage () {
+  await authGuard('admin');
+
+  return (
+    <FeedbackPage />
+  );
+}
+
+export const dynamic = 'force-dynamic';

src/app/(main)/nav.tsx

@@ -16,7 +16,7 @@ import type { Page } from '@/lib/database';
 import { fetcher } from '@/lib/fetch';
 import { cn } from '@/lib/utils';
 import * as DialogPrimitive from '@radix-ui/react-dialog';
-import { ActivitySquareIcon, BinaryIcon, BotMessageSquareIcon, CogIcon, CommandIcon, FilesIcon, GlobeIcon, HomeIcon, ImportIcon, MenuIcon, MessagesSquareIcon, PlusIcon } from 'lucide-react';
+import { ActivitySquareIcon, BinaryIcon, BotMessageSquareIcon, CogIcon, CommandIcon, FilesIcon, GlobeIcon, HomeIcon, ImportIcon, MenuIcon, MessageCircleQuestionIcon, MessagesSquareIcon, PlusIcon } from 'lucide-react';
 
 import { useSession } from 'next-auth/react';
 import Link from 'next/link';
@@ -94,6 +94,7 @@ export function Nav () {
         title: 'Admin',
         items: [
           { href: '/dashboard', title: 'Overview', icon: ActivitySquareIcon },
+          { href: '/feedbacks', title: 'Feedbacks', icon: MessageCircleQuestionIcon },
           { href: '/documents', title: 'Documents', icon: FilesIcon },
           { href: '/indexes', title: 'Indexes', icon: BinaryIcon },
           { href: '/chat-engines', title: 'Chat Engines', icon: BotMessageSquareIcon },

src/app/api/v1/chats/[id]/messages/[messageId]/feedback/route.ts

@@ -1,5 +1,5 @@
 import { getChat, getChatMessage } from '@/core/repositories/chat';
-import { createKnowledgeGraphFeedback, findKnowledgeGraphFeedback } from '@/core/repositories/knowledge_graph_feedback';
+import { createFeedback, findFeedback } from '@/core/repositories/feedback';
 import { getTraceId } from '@/core/services/feedback/utils';
 import { defineHandler } from '@/lib/next/handler';
 import { notFound } from 'next/navigation';
@@ -33,7 +33,7 @@ export const GET = defineHandler(({
   }
 
   const userId = auth.user.id!;
-  return await findKnowledgeGraphFeedback(getTraceId(message.trace_url), userId);
+  return await findFeedback(getTraceId(message.trace_url), userId);
 });
 
 export const POST = defineHandler(({
@@ -42,7 +42,8 @@ export const POST = defineHandler(({
     messageId: z.coerce.number(),
   }),
   body: z.object({
-    detail: z.record(z.enum(['like', 'dislike'])),
+    action: z.enum(['like', 'dislike']),
+    knowledge_graph_detail: z.record(z.enum(['like', 'dislike'])),
     comment: z.string(),
   }),
   auth: 'anonymous',
@@ -69,7 +70,7 @@ export const POST = defineHandler(({
 
   const traceId = getTraceId(message.trace_url);
   const userId = auth.user.id!;
-  const feedback = await findKnowledgeGraphFeedback(traceId, userId);
+  const feedback = await findFeedback(traceId, userId);
 
   // like whole answer
 
@@ -79,8 +80,11 @@ export const POST = defineHandler(({
     }, { status: 400 });
   }
 
-  await createKnowledgeGraphFeedback({
-    detail: body.detail,
+  await createFeedback({
+    action: body.action,
+    chat_id: params.id,
+    message_id: params.messageId,
+    knowledge_graph_detail: body.knowledge_graph_detail,
     created_by: userId,
     trace_id: getTraceId(message.trace_url),
     created_at: new Date(),

src/app/api/v1/chats/[id]/messages/[messageId]/trace/datasets/route.ts

@@ -0,0 +1,51 @@
+import { getChat, getChatMessage } from '@/core/repositories/chat';
+import { getTraceId } from '@/core/services/feedback/utils';
+import { handleErrors } from '@/lib/fetch';
+import type { LangfuseDatasetsResponse } from '@/lib/langfuse/types';
+import { defineHandler } from '@/lib/next/handler';
+import { Langfuse } from 'langfuse';
+import { notFound } from 'next/navigation';
+import z from 'zod';
+
+export const GET = defineHandler({
+  params: z.object({
+    id: z.coerce.number(),
+    messageId: z.coerce.number(),
+  }),
+}, async ({ params }) => {
+  const chat = await getChat(params.id);
+  const message = await getChatMessage(params.messageId);
+
+  if (!chat || !message) {
+    notFound();
+  }
+
+  if (!message.trace_url) {
+    notFound();
+  }
+
+  const traceId = getTraceId(message.trace_url);
+
+  const datasets: LangfuseDatasetsResponse = await fetch(`https://us.cloud.langfuse.com/api/public/datasets`, {
+    method: 'GET',
+    headers: {
+      Authorization: `Basic ${btoa(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`)}`,
+    },
+    cache: 'no-cache',
+  }).then(handleErrors).then(res => res.json());
+
+  const includedDatasets: string[] = [];
+  const client = new Langfuse();
+
+  await Promise.all(datasets.data.map(async dataset => {
+    const datasetDetails = await client.getDataset(dataset.name);
+    for (let item of datasetDetails.items) {
+      if ((item as any).sourceTraceId === traceId) {
+        includedDatasets.push(datasetDetails.name);
+        break;
+      }
+    }
+  }));
+
+  return includedDatasets;
+});

src/app/api/v1/feedbacks/route.ts

@@ -0,0 +1,22 @@
+import { listFeedbacks } from '@/core/repositories/feedback';
+import { toPageRequest } from '@/lib/database';
+import { defineHandler } from '@/lib/next/handler';
+import z from 'zod';
+
+export const GET = defineHandler({
+  searchParams: z.object({
+    chat_id: z.coerce.number().optional(),
+    message_id: z.coerce.number().optional(),
+  }),
+}, async ({ request, searchParams }) => {
+  const { page, pageSize, sorting } = toPageRequest(request);
+
+  return await listFeedbacks({
+    page,
+    pageSize,
+    sorting,
+    ...searchParams,
+  });
+});
+
+export const dynamic = 'force-dynamic';

src/app/api/v1/langfuse/datasets/[name]/items/route.ts

@@ -0,0 +1,30 @@
+import { handleErrors } from '@/lib/fetch';
+import { defineHandler } from '@/lib/next/handler';
+import { Langfuse } from 'langfuse';
+import z from 'zod';
+
+export const POST = defineHandler({
+  auth: 'admin',
+  params: z.object({
+    name: z.string(),
+  }),
+  body: z.object({
+    traceId: z.string(),
+  }),
+}, async ({ params, body }) => {
+  const trace = await fetch(`https://us.cloud.langfuse.com/api/public/traces/${body.traceId}`, {
+    method: 'GET',
+    headers: {
+      Authorization: `Basic ${btoa(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`)}`,
+    },
+    cache: 'no-cache',
+  }).then(handleErrors).then(res => res.json());
+
+  return await new Langfuse().createDatasetItem({
+    datasetName: params.name,
+    input: trace.input,
+    metadata: trace.metadata,
+    expectedOutput: trace.output,
+    sourceTraceId: body.traceId,
+  });
+});

src/app/api/v1/langfuse/datasets/route.ts

@@ -0,0 +1,17 @@
+import { handleErrors } from '@/lib/fetch';
+import { defineHandler } from '@/lib/next/handler';
+
+export const GET = defineHandler({
+  auth: 'admin',
+}, async ({}) => {
+  const response = await fetch(`https://us.cloud.langfuse.com/api/public/datasets`, {
+    method: 'GET',
+    headers: {
+      Authorization: `Basic ${btoa(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`)}`,
+    },
+  }).then(handleErrors);
+
+  return await response.json()
+});
+
+export const dynamic = 'force-dynamic';

src/components/chat/debug-info.tsx

@@ -1,9 +1,10 @@
 import { useChatEngineOptions } from '@/components/chat/context';
 import { KnowledgeGraphDebugInfo } from '@/components/chat/knowledge-graph-debug-info';
+import { MessageLangfuse } from '@/components/chat/message-langfuse';
 import type { ConversationMessageGroupProps } from '@/components/chat/use-grouped-conversation-messages';
 import { Dialog, DialogContent, DialogHeader, DialogPortal, DialogTrigger } from '@/components/ui/dialog';
-import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area';
-import { WaypointsIcon, WorkflowIcon } from 'lucide-react';
+import { ScrollArea } from '@/components/ui/scroll-area';
+import { WorkflowIcon } from 'lucide-react';
 import 'react-json-view-lite/dist/index.css';
 
 export interface DebugInfoProps {
@@ -23,6 +24,7 @@ export function DebugInfo ({ group }: DebugInfoProps) {
           Langfuse Tracing
         </a>
       </div>}
+      <MessageLangfuse group={group} />
       {graph_retriever?.enable && <KnowledgeGraphDebugInfo group={group} />}
       {graph_retriever?.top_k && <div className="mt-2"><b>Knowledge Graph Top K</b>: {graph_retriever.top_k}</div>}
       {graph_retriever?.reranker && (<div className="mt-2"><b>Knowledge Graph Reranker</b>: {graph_retriever.reranker.provider} {graph_retriever.reranker.options?.model}</div>)}