diff --git a/src/features/ai/components/agent-launch-input.tsx b/src/features/ai/components/agent-launch-input.tsx index 199db854a..1a2101e26 100644 --- a/src/features/ai/components/agent-launch-input.tsx +++ b/src/features/ai/components/agent-launch-input.tsx @@ -7,6 +7,7 @@ import { useAIChatStore } from "@/features/ai/stores/ai-chat.store"; import { useBufferStore } from "@/features/editor/stores/buffer.store"; import { useFileSystemStore } from "@/features/file-system/stores/file-system.store"; import type { FileEntry } from "@/features/file-system/types/app.types"; +import type { PastedImage } from "@/features/ai/types/chat-composer.types"; const EMPTY_PROJECT_FILES: FileEntry[] = []; @@ -48,6 +49,7 @@ export function AgentLaunchInput({ chatId, agentId: selectedAgentId, prompt: nextPrompt, + images: images && images.length > 0 ? images : undefined, selectedBufferIds: Array.from(selectedBufferIds), selectedFilesPaths: Array.from(selectedFilesPaths), editorSelections: composerContext.inputProps.selectedEditorContexts, diff --git a/src/features/ai/components/chat/ai-chat.tsx b/src/features/ai/components/chat/ai-chat.tsx index 498166427..1c55bf405 100644 --- a/src/features/ai/components/chat/ai-chat.tsx +++ b/src/features/ai/components/chat/ai-chat.tsx @@ -34,11 +34,9 @@ import { useAIChatStore } from "@/features/ai/stores/ai-chat.store"; import { useComposerContextSelection } from "@/features/ai/hooks/use-composer-context-selection"; import type { AcpEvent } from "@/features/ai/types/acp.types"; import type { ContextInfo } from "@/features/ai/types/ai-context.types"; -import type { - AgentMessageSubmitResult, - AIChatProps, - Message, -} from "@/features/ai/types/ai-chat.types"; +import type { AIChatProps, Message } from "@/features/ai/types/ai-chat.types"; +import type { PastedImage } from "@/features/ai/types/chat-composer.types"; +import { decodePastedImages } from "@/features/ai/utils/pasted-images"; import type { ChatAcpEvent } from "@/features/ai/types/chat-ui.types"; import { getFallbackAgentSessionTitle, @@ -98,9 +96,12 @@ const AIChat = memo(function AIChat({ const [isMessageSearchOpen, setIsMessageSearchOpen] = useState(false); const [messageSearchQuery, setMessageSearchQuery] = useState(""); const [activeMessageSearchIndex, setActiveMessageSearchIndex] = useState(0); - const composerContext = useComposerContextSelection(); - const { selectedBufferIds, selectedEditorContexts, selectedFilesPaths } = - composerContext.inputProps; + const [selectedBufferIds, setSelectedBufferIds] = useState>(new Set()); + const [selectedFilesPaths, setSelectedFilesPaths] = useState>(new Set()); + const [isSurfaceTyping, setIsSurfaceTyping] = useState(false); + const [surfaceStreamingMessageId, setSurfaceStreamingMessageId] = useState(null); + const [queueCount, setQueueCount] = useState(0); + const messageQueueRef = useRef<{ content: string; images?: PastedImage[] }[]>([]); const effectiveChatId = chatId ?? chatState.currentChatId; const currentChat = useMemo( () => chatState.chats.find((chat) => chat.id === effectiveChatId), @@ -424,8 +425,8 @@ const AIChat = memo(function AIChat({ async function processMessage( messageContent: string, - options: { editedUserMessageId?: string; targetChatId?: string } = {}, - ) { + options: { editedUserMessageId?: string; images?: PastedImage[] } = {}, + ) => { const store = useAIChatStore.getState(); const requestedChatId = options.targetChatId ?? effectiveChatId; const targetChat = requestedChatId @@ -466,6 +467,7 @@ const AIChat = memo(function AIChat({ ? existingMessages.slice(0, editedUserMessageIndex) : existingMessages, ); + const decodedImages = decodePastedImages(options.images ?? []); const userMessage: Message = editedUserMessageIndex >= 0 ? { @@ -478,6 +480,7 @@ const AIChat = memo(function AIChat({ content: trimmedMessageContent, role: "user", timestamp: new Date(), + images: decodedImages.length > 0 ? decodedImages : undefined, }; const assistantMessageId = createMessageId(); @@ -985,6 +988,8 @@ details: ${errorDetails || mainError} ); }, targetChatId, + undefined, + decodedImages, ); } catch (error) { console.error("Failed to start streaming:", error); @@ -996,30 +1001,35 @@ details: ${errorDetails || mainError} finishRunAndProcessQueue(targetChatId, runId); abortControllerRef.current = null; } - } + }; - const sendMessage = useCallback( - (messageContent: string): AgentMessageSubmitResult => { - if (!messageContent.trim()) return { accepted: false }; - const access = getAgentMessageAccess(currentAgentId, chatState.hasApiKey); - if (!access.accepted) { - showToast({ message: access.error ?? "This agent is not ready.", type: "error" }); - return access; - } - if (!isChatMessagesLoaded) { - const result = { accepted: false, error: "Wait for this session to finish loading." }; - showToast({ message: result.error, type: "error" }); - return result; - } + const processQueuedMessages = useCallback(async () => { + if (isSurfaceTyping || surfaceStreamingMessageId) { + return; + } + + const nextMessage = messageQueueRef.current.shift(); + setQueueCount(messageQueueRef.current.length); + if (nextMessage) { + console.log("Processing next queued message:", nextMessage.content); + await new Promise((resolve) => setTimeout(resolve, 500)); + await processMessage(nextMessage.content, { images: nextMessage.images }); + } + }, [isSurfaceTyping, surfaceStreamingMessageId]); - const targetChatId = effectiveChatId ?? useAIChatStore.getState().currentChatId; - if (targetChatId && useAIChatStore.getState().agentRuns[targetChatId]) { - chatActions.enqueueAgentMessage(targetChatId, messageContent); - return { accepted: true }; + const sendMessage = useCallback( + async (messageContent: string, images?: PastedImage[]) => { + const isAcp = isAcpAgent(currentAgentId); + // For ACP agents, we don't need an API key. + if (!messageContent.trim() || (!isAcp && !chatState.hasApiKey)) return; + + if (isSurfaceTyping || surfaceStreamingMessageId) { + messageQueueRef.current.push({ content: messageContent, images }); + setQueueCount(messageQueueRef.current.length); + return; } - void processMessage(messageContent); - return { accepted: true }; + await processMessage(messageContent, { images }); }, [ chatActions.enqueueAgentMessage, @@ -1032,7 +1042,9 @@ details: ${errorDetails || mainError} ); const handleSendMessage = useCallback( - (messageContent: string) => sendMessage(messageContent), + async (messageContent: string, images?: PastedImage[]) => { + await sendMessage(messageContent, images); + }, [sendMessage], ); @@ -1057,15 +1069,7 @@ details: ${errorDetails || mainError} pendingLaunch.editorSelections, ); chatActions.setPendingAgentLaunchRequest(null); - if (!pendingLaunch.prompt) return; - - const access = getAgentMessageAccess(pendingLaunch.agentId, chatState.hasApiKey); - if (!access.accepted) { - showToast({ message: access.error ?? "This agent is not ready.", type: "error" }); - return; - } - - void sendMessage(pendingLaunch.prompt); + void sendMessage(pendingLaunch.prompt, pendingLaunch.images); }, [ chatActions, effectiveChatId, diff --git a/src/features/ai/components/input/chat-input-bar.tsx b/src/features/ai/components/input/chat-input-bar.tsx index 97d593653..98e3a9366 100644 --- a/src/features/ai/components/input/chat-input-bar.tsx +++ b/src/features/ai/components/input/chat-input-bar.tsx @@ -949,6 +949,9 @@ const AIChatInputBar = memo(function AIChatInputBar({ if (inputRef.current) { inputRef.current.innerHTML = ""; } + + // Send the captured message + await onSendMessage(currentInput, currentImages.length > 0 ? currentImages : undefined); }; const focusInput = useCallback(() => inputRef.current?.focus(), []); diff --git a/src/features/ai/services/acp-stream-handler.ts b/src/features/ai/services/acp-stream-handler.ts index 8ff15c80e..3c5903989 100644 --- a/src/features/ai/services/acp-stream-handler.ts +++ b/src/features/ai/services/acp-stream-handler.ts @@ -9,7 +9,7 @@ import type { AgentConfig, } from "@/features/ai/types/acp.types"; import type { ContextInfo } from "@/features/ai/types/ai-context.types"; -import type { AgentCompletionResult } from "@/features/ai/types/agent-completion.types"; +import type { DecodedPastedImage } from "@/features/ai/utils/pasted-images"; import { useBufferStore } from "@/features/editor/stores/buffer.store"; import { useProjectStore } from "@/features/window/stores/project.store"; import { getAcpPathBaseName, toAcpFileUri } from "@/features/ai/lib/acp-file-uri"; @@ -90,13 +90,11 @@ export class AcpStreamHandler { await handler.ensureAgentRunning(); } - async start(userMessage: string, context: ContextInfo): Promise { - if (AcpStreamHandler.activeHandler && AcpStreamHandler.activeHandler !== this) { - this.handlers.onError( - "Another agent session is already running. Stop it before sending this prompt.", - ); - return; - } + async start( + userMessage: string, + context: ContextInfo, + images: readonly DecodedPastedImage[] = [], + ): Promise { try { AcpStreamHandler.activeHandler = this; await this.ensureAgentRunning(); @@ -106,7 +104,7 @@ export class AcpStreamHandler { await this.setupListeners(); this.awaitingFirstResponse = true; await withTimeout( - invoke("send_acp_prompt", { prompt: this.buildPrompt(userMessage, context) }), + invoke("send_acp_prompt", { prompt: this.buildPrompt(userMessage, context, images) }), ACP_PROMPT_TIMEOUT_MS, `${this.agentId} did not accept the prompt in time`, ); @@ -254,41 +252,47 @@ export class AcpStreamHandler { return `${this.agentId} is currently unavailable.`; } - private buildPrompt(userMessage: string, context: ContextInfo): AcpPromptContentBlock[] { + private buildPrompt( + userMessage: string, + context: ContextInfo, + images: readonly DecodedPastedImage[] = [], + ): AcpPromptContentBlock[] { // ACP slash commands must remain the first token in the prompt. // If we prepend context, agents interpret them as plain text. + const blocks: AcpPromptContentBlock[] = []; if (userMessage.trimStart().startsWith("/")) { - return [{ type: "text", text: userMessage }]; - } - - const contextPrompt = [buildContextPrompt(context), getFollowUpActionsInstruction()] - .filter(Boolean) - .join("\n\n"); - const blocks: AcpPromptContentBlock[] = [ - { type: "text", text: contextPrompt ? `${contextPrompt}\n\n${userMessage}` : userMessage }, - ]; - - const supportsEmbeddedContext = - useAIChatStore.getState().acpStatus?.agentCapabilities?.promptCapabilities.embeddedContext ?? - false; + blocks.push({ type: "text", text: userMessage }); + } else { + const contextPrompt = [buildContextPrompt(context), getFollowUpActionsInstruction()] + .filter(Boolean) + .join("\n\n"); + blocks.push({ + type: "text", + text: contextPrompt ? `${contextPrompt}\n\n${userMessage}` : userMessage, + }); - for (const file of context.mentionedFiles || []) { - if (supportsEmbeddedContext) { - blocks.push({ - type: "resource", - resource: { + const supportsEmbeddedContext = + useAIChatStore.getState().acpStatus?.agentCapabilities?.promptCapabilities + .embeddedContext ?? false; + + for (const file of context.mentionedFiles || []) { + if (supportsEmbeddedContext) { + blocks.push({ + type: "resource", + resource: { + uri: toAcpFileUri(file.path), + text: file.content, + mimeType: "text/plain", + }, + }); + } else { + blocks.push({ + type: "resource_link", uri: toAcpFileUri(file.path), - text: file.content, + name: getAcpPathBaseName(file.path), mimeType: "text/plain", - }, - }); - } else { - blocks.push({ - type: "resource_link", - uri: toAcpFileUri(file.path), - name: getAcpPathBaseName(file.path), - mimeType: "text/plain", - }); + }); + } } } @@ -309,6 +313,10 @@ export class AcpStreamHandler { }); } + for (const image of images) { + blocks.push({ type: "image", data: image.data, mediaType: image.mediaType }); + } + return blocks; } diff --git a/src/features/ai/services/ai-chat-service.ts b/src/features/ai/services/ai-chat-service.ts index 0e400a3a3..f7e2479be 100644 --- a/src/features/ai/services/ai-chat-service.ts +++ b/src/features/ai/services/ai-chat-service.ts @@ -18,6 +18,7 @@ import { } from "@/features/ai/services/providers/ai-provider-registry"; import { isOllamaCloudUrl } from "@/features/ai/services/providers/ollama-provider"; import { processStreamingResponse } from "@/utils/stream-utils"; +import type { DecodedPastedImage } from "@/features/ai/utils/pasted-images"; import { getProviderApiToken } from "@/features/ai/services/ai-token-service"; import { resolveChatCompletionTokenLimit } from "@/features/ai/lib/chat-completion-budget"; import { @@ -153,6 +154,7 @@ export const getChatCompletionStream = async ( onResourceChunk?: (uri: string, name: string | null) => void, chatId?: string, systemPromptOverride?: string, + images?: readonly DecodedPastedImage[], ): Promise => { try { if (agentId === CODEX_INTEGRATION_ID) { @@ -196,7 +198,7 @@ export const getChatCompletionStream = async ( }, chatId, ); - await handler.start(userMessage, context); + await handler.start(userMessage, context, images); return; } diff --git a/src/features/ai/stores/ai-chat/ai-chat-store.types.ts b/src/features/ai/stores/ai-chat/ai-chat-store.types.ts index 4fd39c370..b4c147cc1 100644 --- a/src/features/ai/stores/ai-chat/ai-chat-store.types.ts +++ b/src/features/ai/stores/ai-chat/ai-chat-store.types.ts @@ -12,6 +12,7 @@ import type { Message, OutputStyle, } from "@/features/ai/types/ai-chat.types"; +import type { PastedImage } from "@/features/ai/types/chat-composer.types"; import type { ProviderModel } from "@/features/ai/services/providers/ai-provider-interface"; import type { EditorSelectionContext } from "@/features/ai/types/ai-context.types"; @@ -23,7 +24,8 @@ export interface AIWorkspaceSessionSnapshot { interface PendingAgentLaunchRequest { chatId: string; agentId: AgentType; - prompt: string | null; + prompt: string; + images?: PastedImage[]; selectedBufferIds: string[]; selectedFilesPaths: string[]; editorSelections: EditorSelectionContext[]; diff --git a/src/features/ai/tests/pasted-images.test.ts b/src/features/ai/tests/pasted-images.test.ts new file mode 100644 index 000000000..1895193de --- /dev/null +++ b/src/features/ai/tests/pasted-images.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vite-plus/test"; +import type { PastedImage } from "@/features/ai/types/chat-composer.types"; +import { decodePastedImage, decodePastedImages } from "@/features/ai/utils/pasted-images"; + +function makePastedImage(dataUrl: string): PastedImage { + return { + id: "image-1", + dataUrl, + name: "pasted.png", + size: 1024, + }; +} + +describe("decodePastedImage", () => { + it("decodes a base64 data URL into data and media type", () => { + expect(decodePastedImage(makePastedImage("data:image/png;base64,aGVsbG8="))).toEqual({ + mediaType: "image/png", + data: "aGVsbG8=", + }); + }); + + it("decodes a data URL without the base64 marker", () => { + expect(decodePastedImage(makePastedImage("data:image/jpeg,rawdata"))).toEqual({ + mediaType: "image/jpeg", + data: "rawdata", + }); + }); + + it("returns null for a non-data URL", () => { + expect(decodePastedImage(makePastedImage("https://example.com/image.png"))).toBeNull(); + expect(decodePastedImage(makePastedImage(""))).toBeNull(); + }); +}); + +describe("decodePastedImages", () => { + it("decodes every valid image and skips invalid ones", () => { + const decoded = decodePastedImages([ + makePastedImage("data:image/png;base64,aaa"), + makePastedImage("not-a-data-url"), + makePastedImage("data:image/webp;base64,bbb"), + ]); + + expect(decoded).toEqual([ + { mediaType: "image/png", data: "aaa" }, + { mediaType: "image/webp", data: "bbb" }, + ]); + }); + + it("returns an empty list when there is nothing to decode", () => { + expect(decodePastedImages([])).toEqual([]); + expect(decodePastedImages([makePastedImage("broken")])).toEqual([]); + }); +}); diff --git a/src/features/ai/types/acp.types.ts b/src/features/ai/types/acp.types.ts index 4f6286347..fde989797 100644 --- a/src/features/ai/types/acp.types.ts +++ b/src/features/ai/types/acp.types.ts @@ -75,6 +75,7 @@ type AcpContentBlock = export type AcpPromptContentBlock = | { type: "text"; text: string } + | { type: "image"; data: string; mediaType: string } | { type: "resource_link"; uri: string; name: string; mimeType?: string | null } | { type: "resource"; diff --git a/src/features/ai/types/ai-chat.types.ts b/src/features/ai/types/ai-chat.types.ts index be93e30ab..7b1a11edc 100644 --- a/src/features/ai/types/ai-chat.types.ts +++ b/src/features/ai/types/ai-chat.types.ts @@ -5,7 +5,7 @@ import type { } from "@/features/ai/types/acp.types"; import type { ChatFollowUpAction } from "@/features/ai/lib/follow-up-actions"; import type { FileEntry } from "@/features/file-system/types/app.types"; -import type { EditorSelectionContext } from "@/features/ai/types/ai-context.types"; +import type { PastedImage } from "@/features/ai/types/chat-composer.types"; import type { PaneContent } from "@/features/panes/types/pane-content.types"; import type { GenerativeUIView } from "@/extensions/ui/types/generative-ui"; @@ -117,6 +117,6 @@ export interface AIChatInputBarProps { presentation?: "default" | "initial"; autoFocus?: boolean; onAgentChange?: (agentId: AgentType) => void; - onSendMessage: (message: string) => AgentMessageSubmitResult; + onSendMessage: (message: string, images?: PastedImage[]) => Promise; onStopStreaming: () => void; } diff --git a/src/features/ai/utils/pasted-images.ts b/src/features/ai/utils/pasted-images.ts new file mode 100644 index 000000000..b3fc286d5 --- /dev/null +++ b/src/features/ai/utils/pasted-images.ts @@ -0,0 +1,25 @@ +import type { PastedImage } from "@/features/ai/types/chat-composer.types"; + +export interface DecodedPastedImage { + data: string; + mediaType: string; +} + +const DATA_URL_PATTERN = /^data:([^;,]+)(;base64)?,/; + +export function decodePastedImage(image: PastedImage): DecodedPastedImage | null { + const match = DATA_URL_PATTERN.exec(image.dataUrl); + if (!match) return null; + + return { + mediaType: match[1], + data: image.dataUrl.slice(match[0].length), + }; +} + +export function decodePastedImages(images: readonly PastedImage[]): DecodedPastedImage[] { + return images.flatMap((image) => { + const decoded = decodePastedImage(image); + return decoded ? [decoded] : []; + }); +}