From 0f3e6c08ad17caf9649c6c59f38e90ca18fe9895 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 12 Jun 2024 16:45:12 +0800 Subject: [PATCH] feat: feedback and add chat to langfuse dataset (#163) * feat: feedback * support add chat to langfuse * update --- ddl/2-feedback.sql | 26 +++-- package.json | 2 +- pnpm-lock.yaml | 14 +-- .../(main)/(admin)/feedbacks/page.client.tsx | 53 +++++++++ src/app/(main)/(admin)/feedbacks/page.tsx | 12 ++ src/app/(main)/nav.tsx | 3 +- .../messages/[messageId]/feedback/route.ts | 16 ++- .../[messageId]/trace/datasets/route.ts | 51 +++++++++ src/app/api/v1/feedbacks/route.ts | 22 ++++ .../langfuse/datasets/[name]/items/route.ts | 30 +++++ src/app/api/v1/langfuse/datasets/route.ts | 17 +++ src/components/chat/debug-info.tsx | 6 +- src/components/chat/message-feedback.tsx | 105 +++++++++++------- src/components/chat/message-langfuse.tsx | 97 ++++++++++++++++ src/components/chat/message-operations.tsx | 2 +- src/components/chat/use-message-feedback.ts | 12 +- src/components/chat/use-message-langfuse.ts | 59 ++++++++++ src/components/use-langfuse-datasets.ts | 15 +++ src/core/db/schema.d.ts | 15 +++ src/core/repositories/feedback.ts | 76 +++++++++++++ src/lib/langfuse/types.d.ts | 12 ++ 21 files changed, 568 insertions(+), 77 deletions(-) create mode 100644 src/app/(main)/(admin)/feedbacks/page.client.tsx create mode 100644 src/app/(main)/(admin)/feedbacks/page.tsx create mode 100644 src/app/api/v1/chats/[id]/messages/[messageId]/trace/datasets/route.ts create mode 100644 src/app/api/v1/feedbacks/route.ts create mode 100644 src/app/api/v1/langfuse/datasets/[name]/items/route.ts create mode 100644 src/app/api/v1/langfuse/datasets/route.ts create mode 100644 src/components/chat/message-langfuse.tsx create mode 100644 src/components/chat/use-message-langfuse.ts create mode 100644 src/components/use-langfuse-datasets.ts create mode 100644 src/core/repositories/feedback.ts create mode 100644 src/lib/langfuse/types.d.ts diff --git a/ddl/2-feedback.sql b/ddl/2-feedback.sql index c6905850..f67c2b19 100644 --- a/ddl/2-feedback.sql +++ b/ddl/2-feedback.sql @@ -1,16 +1,20 @@ -DROP TABLE knowledge_graph_feedback; +DROP TABLE feedback; -CREATE TABLE knowledge_graph_feedback +CREATE TABLE feedback ( - id INTEGER NOT NULL AUTO_INCREMENT, - trace_id BINARY(16) NOT NULL COMMENT 'Langfuse trace ID (UUID)', - detail JSON NOT NULL COMMENT 'Map, key is source URL, value is `like` or `dislike`.', - comment TEXT NOT NULL COMMENT 'Comments from user', - created_by VARCHAR(32) NOT NULL COMMENT 'User id', - created_at DATETIME NOT NULL COMMENT 'User submit feedback at', - reported_at DATETIME NULL COMMENT 'Reported to graph.tidb.ai', - report_error VARCHAR(512) NULL COMMENT 'Report failure reason if reporting failed.', + id INTEGER NOT NULL AUTO_INCREMENT, + chat_id INTEGER NOT NULL, + message_id INTEGER NOT NULL, + trace_id BINARY(16) NOT NULL COMMENT 'Langfuse trace ID (UUID)', + action ENUM ('like', 'dislike') NOT NULL, + comment TEXT NOT NULL COMMENT 'Comments from user', + created_by VARCHAR(32) NOT NULL COMMENT 'User id', + created_at DATETIME NOT NULL COMMENT 'User submit feedback at', + knowledge_graph_detail JSON NOT NULL COMMENT 'Map, key is source URL, value is `like` or `dislike`.', + knowledge_graph_reported_at DATETIME NULL COMMENT 'Reported to graph.tidb.ai', + knowledge_graph_report_error TEXT NULL COMMENT 'Report failure reason if reporting failed.', PRIMARY KEY (id), UNIQUE INDEX (trace_id, created_by), - INDEX (created_at, reported_at) + INDEX (created_at, knowledge_graph_reported_at), + INDEX (chat_id, message_id) ); diff --git a/package.json b/package.json index 888596ad..7c0aeb5e 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "hast-util-select": "^6.0.2", "hast-util-to-text": "^4.0.0", "kysely": "^0.27.2", - "langfuse": "^3.10.0", + "langfuse": "^3.11.2", "liquidjs": "^10.10.0", "llamaindex": "^0.3.10", "luxon": "^3.4.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b1820649..d090b890 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -165,8 +165,8 @@ importers: specifier: ^0.27.2 version: 0.27.2 langfuse: - specifier: ^3.10.0 - version: 3.10.0 + specifier: ^3.11.2 + version: 3.11.2 liquidjs: specifier: ^10.10.0 version: 10.10.0 @@ -9902,18 +9902,18 @@ packages: resolution: {integrity: sha512-DmRvEfiR/NLpgsTbSxma2ldekhsdcd65+MNiKXyd/qj7w7X5e3cLkXxcj+MypsRDjPhHQ/CD5u3Eq1sBYzX0bw==} engines: {node: '>=14.0.0'} - /langfuse-core@3.10.0: - resolution: {integrity: sha512-ws0sLM0ua/0pb2DyDBbBIljUxXLiljIdwgdwPW5ewMlJzrFXJa4TdUNiWjoVvQdea2Y8Tf8KRWH2X8gesmkuMQ==} + /langfuse-core@3.11.2: + resolution: {integrity: sha512-iBhX0oJXXg34SHxY/a3oA5A6Nozd+82XtmRnJ9yO7eYPsyLwCqIWIgj++38W9ffll5lbhKStpNk4ZWvVX94kDw==} engines: {node: '>=18'} dependencies: mustache: 4.2.0 dev: false - /langfuse@3.10.0: - resolution: {integrity: sha512-hDLaIO3OJ8Vg9w0m3g+AdbyguPQeCXxUPsUqdcHet/hUr2hKVT1VjIYklVPlCPUzL5H0FMmTDrhROk94b4bVnQ==} + /langfuse@3.11.2: + resolution: {integrity: sha512-+XQNSZiWvHlpkIMYNCeuVNFu2l7KZbHD1FzozmOzV2vveDJkL+c9t0nby/WWnqFpAXJ2CA4LwGT7Al8uiFc7hg==} engines: {node: '>=18'} dependencies: - langfuse-core: 3.10.0 + langfuse-core: 3.11.2 dev: false /language-subtag-registry@0.3.22: diff --git a/src/app/(main)/(admin)/feedbacks/page.client.tsx b/src/app/(main)/(admin)/feedbacks/page.client.tsx new file mode 100644 index 00000000..9a297ba9 --- /dev/null +++ b/src/app/(main)/(admin)/feedbacks/page.client.tsx @@ -0,0 +1,53 @@ +'use client'; + +import { AdminPageHeading } from '@/components/admin-page-heading'; +import { DataTableRemote } from '@/components/data-table-remote'; +import type { Feedback } from '@/core/repositories/feedback'; +import type { ColumnDef } from '@tanstack/react-table'; +import { createColumnHelper } from '@tanstack/table-core'; +import { format } from 'date-fns'; +import { LinkIcon, ThumbsDownIcon, ThumbsUpIcon } from 'lucide-react'; +import Link from 'next/link'; + +export default function FeedbackPage () { + return ( + <> + + + idColumn="id" + api="/api/v1/feedbacks" + columns={columns} + /> + + ); +} + +const helper = createColumnHelper(); +const columns: ColumnDef[] = [ + helper.accessor('id', {}), + helper.accessor('chat_key', { + id: 'go_to_chat', + header: 'chat', + cell: (cell) => ( + + + {cell.row.original.chat_title} + + ), + }), + helper.accessor('action', { + cell: (cell) => { + switch (cell.getValue()) { + case 'like': + return Like; + case 'dislike': + return Dislike; + } + }, + }), + helper.accessor('comment', {}), + helper.accessor('created_by', {}), + helper.accessor('created_at', { + cell: cell => , + }), +]; diff --git a/src/app/(main)/(admin)/feedbacks/page.tsx b/src/app/(main)/(admin)/feedbacks/page.tsx new file mode 100644 index 00000000..992fdc07 --- /dev/null +++ b/src/app/(main)/(admin)/feedbacks/page.tsx @@ -0,0 +1,12 @@ +import { authGuard } from '@/lib/auth-server'; +import FeedbackPage from './page.client'; + +export default async function ServerDocumentsPage () { + await authGuard('admin'); + + return ( + + ); +} + +export const dynamic = 'force-dynamic'; diff --git a/src/app/(main)/nav.tsx b/src/app/(main)/nav.tsx index e436a283..04f68059 100644 --- a/src/app/(main)/nav.tsx +++ b/src/app/(main)/nav.tsx @@ -16,7 +16,7 @@ import type { Page } from '@/lib/database'; import { fetcher } from '@/lib/fetch'; import { cn } from '@/lib/utils'; import * as DialogPrimitive from '@radix-ui/react-dialog'; -import { ActivitySquareIcon, BinaryIcon, BotMessageSquareIcon, CogIcon, CommandIcon, FilesIcon, GlobeIcon, HomeIcon, ImportIcon, MenuIcon, MessagesSquareIcon, PlusIcon } from 'lucide-react'; +import { ActivitySquareIcon, BinaryIcon, BotMessageSquareIcon, CogIcon, CommandIcon, FilesIcon, GlobeIcon, HomeIcon, ImportIcon, MenuIcon, MessageCircleQuestionIcon, MessagesSquareIcon, PlusIcon } from 'lucide-react'; import { useSession } from 'next-auth/react'; import Link from 'next/link'; @@ -94,6 +94,7 @@ export function Nav () { title: 'Admin', items: [ { href: '/dashboard', title: 'Overview', icon: ActivitySquareIcon }, + { href: '/feedbacks', title: 'Feedbacks', icon: MessageCircleQuestionIcon }, { href: '/documents', title: 'Documents', icon: FilesIcon }, { href: '/indexes', title: 'Indexes', icon: BinaryIcon }, { href: '/chat-engines', title: 'Chat Engines', icon: BotMessageSquareIcon }, diff --git a/src/app/api/v1/chats/[id]/messages/[messageId]/feedback/route.ts b/src/app/api/v1/chats/[id]/messages/[messageId]/feedback/route.ts index 4bba92ce..486ab421 100644 --- a/src/app/api/v1/chats/[id]/messages/[messageId]/feedback/route.ts +++ b/src/app/api/v1/chats/[id]/messages/[messageId]/feedback/route.ts @@ -1,5 +1,5 @@ import { getChat, getChatMessage } from '@/core/repositories/chat'; -import { createKnowledgeGraphFeedback, findKnowledgeGraphFeedback } from '@/core/repositories/knowledge_graph_feedback'; +import { createFeedback, findFeedback } from '@/core/repositories/feedback'; import { getTraceId } from '@/core/services/feedback/utils'; import { defineHandler } from '@/lib/next/handler'; import { notFound } from 'next/navigation'; @@ -33,7 +33,7 @@ export const GET = defineHandler(({ } const userId = auth.user.id!; - return await findKnowledgeGraphFeedback(getTraceId(message.trace_url), userId); + return await findFeedback(getTraceId(message.trace_url), userId); }); export const POST = defineHandler(({ @@ -42,7 +42,8 @@ export const POST = defineHandler(({ messageId: z.coerce.number(), }), body: z.object({ - detail: z.record(z.enum(['like', 'dislike'])), + action: z.enum(['like', 'dislike']), + knowledge_graph_detail: z.record(z.enum(['like', 'dislike'])), comment: z.string(), }), auth: 'anonymous', @@ -69,7 +70,7 @@ export const POST = defineHandler(({ const traceId = getTraceId(message.trace_url); const userId = auth.user.id!; - const feedback = await findKnowledgeGraphFeedback(traceId, userId); + const feedback = await findFeedback(traceId, userId); // like whole answer @@ -79,8 +80,11 @@ export const POST = defineHandler(({ }, { status: 400 }); } - await createKnowledgeGraphFeedback({ - detail: body.detail, + await createFeedback({ + action: body.action, + chat_id: params.id, + message_id: params.messageId, + knowledge_graph_detail: body.knowledge_graph_detail, created_by: userId, trace_id: getTraceId(message.trace_url), created_at: new Date(), diff --git a/src/app/api/v1/chats/[id]/messages/[messageId]/trace/datasets/route.ts b/src/app/api/v1/chats/[id]/messages/[messageId]/trace/datasets/route.ts new file mode 100644 index 00000000..485c884d --- /dev/null +++ b/src/app/api/v1/chats/[id]/messages/[messageId]/trace/datasets/route.ts @@ -0,0 +1,51 @@ +import { getChat, getChatMessage } from '@/core/repositories/chat'; +import { getTraceId } from '@/core/services/feedback/utils'; +import { handleErrors } from '@/lib/fetch'; +import type { LangfuseDatasetsResponse } from '@/lib/langfuse/types'; +import { defineHandler } from '@/lib/next/handler'; +import { Langfuse } from 'langfuse'; +import { notFound } from 'next/navigation'; +import z from 'zod'; + +export const GET = defineHandler({ + params: z.object({ + id: z.coerce.number(), + messageId: z.coerce.number(), + }), +}, async ({ params }) => { + const chat = await getChat(params.id); + const message = await getChatMessage(params.messageId); + + if (!chat || !message) { + notFound(); + } + + if (!message.trace_url) { + notFound(); + } + + const traceId = getTraceId(message.trace_url); + + const datasets: LangfuseDatasetsResponse = await fetch(`https://us.cloud.langfuse.com/api/public/datasets`, { + method: 'GET', + headers: { + Authorization: `Basic ${btoa(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`)}`, + }, + cache: 'no-cache', + }).then(handleErrors).then(res => res.json()); + + const includedDatasets: string[] = []; + const client = new Langfuse(); + + await Promise.all(datasets.data.map(async dataset => { + const datasetDetails = await client.getDataset(dataset.name); + for (let item of datasetDetails.items) { + if ((item as any).sourceTraceId === traceId) { + includedDatasets.push(datasetDetails.name); + break; + } + } + })); + + return includedDatasets; +}); diff --git a/src/app/api/v1/feedbacks/route.ts b/src/app/api/v1/feedbacks/route.ts new file mode 100644 index 00000000..f2a9283a --- /dev/null +++ b/src/app/api/v1/feedbacks/route.ts @@ -0,0 +1,22 @@ +import { listFeedbacks } from '@/core/repositories/feedback'; +import { toPageRequest } from '@/lib/database'; +import { defineHandler } from '@/lib/next/handler'; +import z from 'zod'; + +export const GET = defineHandler({ + searchParams: z.object({ + chat_id: z.coerce.number().optional(), + message_id: z.coerce.number().optional(), + }), +}, async ({ request, searchParams }) => { + const { page, pageSize, sorting } = toPageRequest(request); + + return await listFeedbacks({ + page, + pageSize, + sorting, + ...searchParams, + }); +}); + +export const dynamic = 'force-dynamic'; diff --git a/src/app/api/v1/langfuse/datasets/[name]/items/route.ts b/src/app/api/v1/langfuse/datasets/[name]/items/route.ts new file mode 100644 index 00000000..5f43cf3c --- /dev/null +++ b/src/app/api/v1/langfuse/datasets/[name]/items/route.ts @@ -0,0 +1,30 @@ +import { handleErrors } from '@/lib/fetch'; +import { defineHandler } from '@/lib/next/handler'; +import { Langfuse } from 'langfuse'; +import z from 'zod'; + +export const POST = defineHandler({ + auth: 'admin', + params: z.object({ + name: z.string(), + }), + body: z.object({ + traceId: z.string(), + }), +}, async ({ params, body }) => { + const trace = await fetch(`https://us.cloud.langfuse.com/api/public/traces/${body.traceId}`, { + method: 'GET', + headers: { + Authorization: `Basic ${btoa(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`)}`, + }, + cache: 'no-cache', + }).then(handleErrors).then(res => res.json()); + + return await new Langfuse().createDatasetItem({ + datasetName: params.name, + input: trace.input, + metadata: trace.metadata, + expectedOutput: trace.output, + sourceTraceId: body.traceId, + }); +}); diff --git a/src/app/api/v1/langfuse/datasets/route.ts b/src/app/api/v1/langfuse/datasets/route.ts new file mode 100644 index 00000000..ab94edd7 --- /dev/null +++ b/src/app/api/v1/langfuse/datasets/route.ts @@ -0,0 +1,17 @@ +import { handleErrors } from '@/lib/fetch'; +import { defineHandler } from '@/lib/next/handler'; + +export const GET = defineHandler({ + auth: 'admin', +}, async ({}) => { + const response = await fetch(`https://us.cloud.langfuse.com/api/public/datasets`, { + method: 'GET', + headers: { + Authorization: `Basic ${btoa(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`)}`, + }, + }).then(handleErrors); + + return await response.json() +}); + +export const dynamic = 'force-dynamic'; diff --git a/src/components/chat/debug-info.tsx b/src/components/chat/debug-info.tsx index ec212670..5fb011ce 100644 --- a/src/components/chat/debug-info.tsx +++ b/src/components/chat/debug-info.tsx @@ -1,9 +1,10 @@ import { useChatEngineOptions } from '@/components/chat/context'; import { KnowledgeGraphDebugInfo } from '@/components/chat/knowledge-graph-debug-info'; +import { MessageLangfuse } from '@/components/chat/message-langfuse'; import type { ConversationMessageGroupProps } from '@/components/chat/use-grouped-conversation-messages'; import { Dialog, DialogContent, DialogHeader, DialogPortal, DialogTrigger } from '@/components/ui/dialog'; -import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area'; -import { WaypointsIcon, WorkflowIcon } from 'lucide-react'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { WorkflowIcon } from 'lucide-react'; import 'react-json-view-lite/dist/index.css'; export interface DebugInfoProps { @@ -23,6 +24,7 @@ export function DebugInfo ({ group }: DebugInfoProps) { Langfuse Tracing } + {graph_retriever?.enable && } {graph_retriever?.top_k &&
Knowledge Graph Top K: {graph_retriever.top_k}
} {graph_retriever?.reranker && (
Knowledge Graph Reranker: {graph_retriever.reranker.provider} {graph_retriever.reranker.options?.model}
)} diff --git a/src/components/chat/message-feedback.tsx b/src/components/chat/message-feedback.tsx index 2c1227a9..1f7ce508 100644 --- a/src/components/chat/message-feedback.tsx +++ b/src/components/chat/message-feedback.tsx @@ -1,22 +1,26 @@ import type { ContentSource } from '@/components/chat/use-message-feedback'; import { Button } from '@/components/ui/button'; -import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; import { Textarea } from '@/components/ui/textarea'; +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group'; +import type { Feedback } from '@/core/repositories/feedback'; import { Loader2Icon, ThumbsDownIcon, ThumbsUpIcon } from 'lucide-react'; import { type ReactElement, useEffect, useState } from 'react'; -export function MessageFeedback ({ initial, source, sourceLoading, onFeedback, children }: { initial?: { detail: Record, comment: string }, source: ContentSource | undefined, sourceLoading: boolean, onFeedback: (detail: Record, comment: string) => Promise, children: ReactElement }) { +export function MessageFeedback ({ initial, source, sourceLoading, onFeedback, children }: { initial?: Feedback, source: ContentSource | undefined, sourceLoading: boolean, onFeedback: (action: 'like' | 'dislike', detail: Record, comment: string) => Promise, children: ReactElement }) { const [open, setOpen] = useState(false); + const [action, setAction] = useState<'like' | 'dislike'>('like'); const [detail, setDetail] = useState>(() => (initial ?? {})); const [comment, setComment] = useState(initial?.comment ?? ''); const [running, setRunning] = useState(false); useEffect(() => { if (initial) { - setDetail(initial.detail); + setAction(initial.action); + setDetail(initial.knowledge_graph_detail); setComment(initial.comment); } - }, [initial]) + }, [initial]); const disabled = running || !!initial; @@ -25,55 +29,72 @@ export function MessageFeedback ({ initial, source, sourceLoading, onFeedback, c {children} - + Feedback -
Sources from Knowledge Graph
- {!source && sourceLoading &&
Loading...
} - {source && ( -
    - {source.markdownSources.kgRelationshipUrls.map(url => ( -
  • -
    - +
    +
    Do you like this answer
    + setAction(value as any)}> + + + Like + + + + Dislike + + +
    +
    +
    Sources from Knowledge Graph
    + {!source && sourceLoading &&
    Loading...
    } + {source && ( +     - )} -