-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add import documents from urls and build index for single/multi…
…ple documents API (#129) * feat: add upload multiple webpage API * feat: add build knowledge graph index API
- Loading branch information
Showing
25 changed files
with
566 additions
and
124 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
26 changes: 26 additions & 0 deletions
26
src/app/(main)/(admin)/explore/components/build-document-index-dialog.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import {BasicFormDialog} from "@/components/dialogs/basic-form-dialog"; | ||
|
|
||
| import { FormField, FormItem, FormLabel } from '@/components/ui/form'; | ||
| import { Textarea } from '@/components/ui/textarea'; | ||
| import {buildDocumentIndex, BuildDocumentIndexOptions} from '@/client/operations/documents'; | ||
| import type {ReactElement} from "react"; | ||
|
|
||
| export function BuildDocumentIndexDialog (trigger: ReactElement) { | ||
| return ( | ||
| <BasicFormDialog<BuildDocumentIndexOptions> | ||
| fromId="build-document-index-form" | ||
| trigger={trigger} | ||
| title="Build document index" | ||
| onSubmit={buildDocumentIndex} | ||
| submitButtonTitle={'Confirm'} | ||
| > | ||
| <FormItem> | ||
| <FormLabel>URL List</FormLabel> | ||
| <FormField | ||
| name="uriList" | ||
| render={({ field }) => <Textarea {...field} />} | ||
| /> | ||
| </FormItem> | ||
| </BasicFormDialog> | ||
| ); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| import {DataTableRowActions} from "@/app/(main)/(admin)/explore/components/data-table-row-actions"; | ||
| import {Tooltip, TooltipContent, TooltipTrigger} from "@/components/ui/tooltip"; | ||
| import type {Document} from "@/core/repositories/document"; | ||
| import type {CellContext, ColumnDef} from "@tanstack/react-table"; | ||
| import {createColumnHelper} from "@tanstack/table-core"; | ||
| import {format} from "date-fns"; | ||
| import {GithubIcon} from "lucide-react"; | ||
|
|
||
| const helper = createColumnHelper<Document>(); | ||
|
|
||
| const mono = (cell: CellContext<any, any>) => <span className="font-mono">{cell.getValue()}</span>; | ||
|
|
||
| const datetime = (cell: CellContext<any, any>) => <time>{format(cell.getValue(), 'yyyy-MM-dd HH:mm')}</time>; | ||
|
|
||
| const GITHUB_PAGE_URL_REGEXP = /^https?:\/\/github\.com\/([^\/]+)\/([^\/]+)(?:\/blob\/([^/]+))?/; | ||
|
|
||
| export const columns = [ | ||
| helper.accessor('id', {cell: mono}), | ||
| helper.accessor('name', { | ||
| cell: (cell: CellContext<any, any>) => <div className="flex space-x-2"> | ||
| <span className="max-w-[500px] truncate font-medium">{cell.getValue()}</span> | ||
| </div>, | ||
| }), | ||
| helper.accessor('mime', {cell: mono}), | ||
| helper.accessor('source_uri', { | ||
| cell: (cell: CellContext<any, any>) => { | ||
| const value = cell.getValue(); | ||
|
|
||
| const matched = GITHUB_PAGE_URL_REGEXP.exec(value); | ||
| if (matched) { | ||
| const [, owner, repo, branch] = matched; | ||
| return ( | ||
| <Tooltip> | ||
| <TooltipTrigger asChild> | ||
| <a href={value} target="_blank" className="flex items-center"> | ||
| <GithubIcon size="1em" className="mr-1"/> | ||
| <span>{owner}</span> | ||
| / | ||
| <span>{repo}</span> | ||
| {branch && <>/<span>{branch}</span></>} | ||
| </a> | ||
| </TooltipTrigger> | ||
| <TooltipContent className="text-xs"> | ||
| {value} | ||
| </TooltipContent> | ||
| </Tooltip> | ||
| ); | ||
| } | ||
|
|
||
| return <div className="flex space-x-2"> | ||
| <span className="max-w-[300px] truncate font-medium">{cell.getValue()}</span> | ||
| </div>; | ||
| } | ||
| }), | ||
| helper.accessor('hash', { | ||
| cell: (cell: CellContext<any, any>) => <span className="font-mono"> | ||
| { | ||
| cell.getValue() ? | ||
| cell.getValue().substring(0, 6) : | ||
| 'N/A' | ||
| } | ||
| </span> | ||
| }), | ||
| helper.accessor('created_at', { | ||
| cell: datetime, | ||
| enableSorting: true | ||
| }), | ||
| helper.accessor('last_modified_at', {cell: datetime}), | ||
| { | ||
| id: "actions", | ||
| cell: ({ row }) => <DataTableRowActions row={row} />, | ||
| }, | ||
| ] as ColumnDef<Document>[]; |
46 changes: 46 additions & 0 deletions
46
src/app/(main)/(admin)/explore/components/data-table-row-actions.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| "use client" | ||
|
|
||
| import {BuildDocumentIndexDialog} from "@/app/(main)/(admin)/explore/components/build-document-index-dialog"; | ||
| import { DotsHorizontalIcon } from "@radix-ui/react-icons" | ||
| import { Row } from "@tanstack/react-table" | ||
|
|
||
| import { Button } from "@/components/ui/button" | ||
| import { | ||
| DropdownMenu, | ||
| DropdownMenuContent, | ||
| DropdownMenuItem, | ||
| DropdownMenuSeparator, | ||
| DropdownMenuShortcut, | ||
| DropdownMenuTrigger, | ||
| } from "@/components/ui/dropdown-menu" | ||
|
|
||
|
|
||
| interface DataTableRowActionsProps<TData> { | ||
| row: Row<TData> | ||
| } | ||
|
|
||
| export function DataTableRowActions<TData>(props: DataTableRowActionsProps<TData>) { | ||
| return ( | ||
| <DropdownMenu> | ||
| <DropdownMenuTrigger asChild> | ||
| <Button | ||
| variant="ghost" | ||
| className="flex h-8 w-8 p-0 data-[state=open]:bg-muted" | ||
| > | ||
| <DotsHorizontalIcon className="h-4 w-4" /> | ||
| <span className="sr-only">Open menu</span> | ||
| </Button> | ||
| </DropdownMenuTrigger> | ||
| <DropdownMenuContent align="end" className="w-[160px]"> | ||
| <DropdownMenuItem disabled={true}> | ||
| Build index (Coming soon) | ||
| </DropdownMenuItem> | ||
| <DropdownMenuSeparator /> | ||
| <DropdownMenuItem disabled={true}> | ||
| Delete | ||
| <DropdownMenuShortcut>⌘⌫</DropdownMenuShortcut> | ||
| </DropdownMenuItem> | ||
| </DropdownMenuContent> | ||
| </DropdownMenu> | ||
| ) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| import {DefaultDocumentImportService, DocumentImportService} from "@/core/services/importing"; | ||
| import { defineHandler } from '@/lib/next/handler'; | ||
| import {baseRegistry} from "@/rag-spec/base"; | ||
| import {getFlow} from "@/rag-spec/createFlow"; | ||
| import { NextResponse } from "next/server"; | ||
| import {z} from "zod"; | ||
|
|
||
| const ImportDocumentsFromUrlsOptionsSchema = z.object({ | ||
| urls: z.string() | ||
| .url('The format of URL is incorrect.') | ||
| .array() | ||
| .min(1, 'Must provide at least one URL for importing.') | ||
| }); | ||
|
|
||
| export const POST = defineHandler({ | ||
| auth: 'admin', | ||
| body: ImportDocumentsFromUrlsOptionsSchema | ||
| }, async ({ body}) => { | ||
| const { urls } = body; | ||
|
|
||
| const encoder = new TextEncoder(); | ||
| const readableStream = new ReadableStream({ | ||
| async pull(controller) { | ||
| const service = new DefaultDocumentImportService({ flow: await getFlow(baseRegistry) }); | ||
| const taskIds = await DocumentImportService.createTasksByURLs(urls); | ||
| console.log('Create document import tasks: ', taskIds); | ||
|
|
||
| const process = await service.runTasks(10, taskIds, (process) => { | ||
| controller.enqueue(encoder.encode(JSON.stringify(process))); | ||
| }); | ||
|
|
||
| controller.enqueue(encoder.encode(JSON.stringify(process))); | ||
| controller.close(); | ||
| }, | ||
| }); | ||
| return new NextResponse(readableStream, { | ||
| headers: { | ||
| 'Content-Type': 'application/json; charset=utf-8', | ||
| }, | ||
| }); | ||
| }); | ||
|
|
||
| export const dynamic = 'force-dynamic'; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| import {getIndexByNameOrThrow} from "@/core/repositories/index_"; | ||
| import {DocumentIndexService} from "@/core/services/indexing"; | ||
| import { defineHandler } from '@/lib/next/handler'; | ||
| import {z} from "zod"; | ||
|
|
||
| const IndexDocumentsOptionsSchema = z.object({ | ||
| documentIds: z.number() | ||
| .int() | ||
| .array() | ||
| .min(1, 'Must provide at least one document'), | ||
| indexName: z.string() | ||
| }); | ||
|
|
||
| export const POST = defineHandler({ | ||
| auth: 'admin', | ||
| body: IndexDocumentsOptionsSchema | ||
| }, async ({ body}) => { | ||
| const { documentIds, indexName } = body; | ||
|
|
||
| const index = await getIndexByNameOrThrow(indexName); | ||
| const documentIdStr = documentIds.map((id) => `#${id}`).join(', ') | ||
| console.log(`Creating index for documents ${documentIdStr} with index <${indexName}> (provider: ${index.config.provider})`); | ||
|
|
||
| const service = new DocumentIndexService(); | ||
| await service.prepareProviders(); | ||
|
|
||
| // Create document index tasks. | ||
| const taskIds = await DocumentIndexService.createDocumentIndexTasksByDocumentIds(documentIds, index.id); | ||
| const taskIdStr = taskIds.map((id) => `#${id}`).join(', ') | ||
| console.log(`Create document index tasks ${taskIdStr}.`); | ||
|
|
||
| // Execute document index tasks. | ||
| const results = await Promise.allSettled( | ||
| taskIds.map(taskId => service.runDocumentIndexTask(taskId)) | ||
| ); | ||
| const succeed: number[] = []; | ||
| const failed: { taskId: number, reason: string; }[] = []; | ||
|
|
||
| results.forEach((result, i) => { | ||
| if (result.status === 'fulfilled') { | ||
| succeed.push(taskIds[i]); | ||
| } else { | ||
| failed.push({ | ||
| taskId: taskIds[i], | ||
| reason: result.reason.message | ||
| }); | ||
| } | ||
| }); | ||
|
|
||
| return { | ||
| succeed, | ||
| failed | ||
| } | ||
| }); | ||
|
|
||
| export const dynamic = 'force-dynamic'; |
Oops, something went wrong.