diff --git a/workspaces/intelligent-assistant/.changeset/screen-context-chip-ux.md b/workspaces/intelligent-assistant/.changeset/screen-context-chip-ux.md new file mode 100644 index 00000000000..2453db20edb --- /dev/null +++ b/workspaces/intelligent-assistant/.changeset/screen-context-chip-ux.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor +--- + +Add screen context UX: kebab opt-in, context chip (recording/paused/unavailable), and gated DOM and screenshot attachments on send per RHIDP-14319. diff --git a/workspaces/intelligent-assistant/e2e-tests/fixtures/responses.ts b/workspaces/intelligent-assistant/e2e-tests/fixtures/responses.ts index b293853d2aa..07d912d3402 100644 --- a/workspaces/intelligent-assistant/e2e-tests/fixtures/responses.ts +++ b/workspaces/intelligent-assistant/e2e-tests/fixtures/responses.ts @@ -27,6 +27,7 @@ export const models = [ type: 'model', owned_by: 'library', provider_id: 'mock-provider-1', + supportsVision: false, }, { identifier: 'mock-provider-1/mock-model-2', @@ -36,6 +37,7 @@ export const models = [ type: 'model', owned_by: 'library', provider_id: 'mock-provider-1', + supportsVision: true, }, ]; diff --git a/workspaces/intelligent-assistant/e2e-tests/lightspeed.conversation.test.ts b/workspaces/intelligent-assistant/e2e-tests/lightspeed.conversation.test.ts index 0617f33c203..369143e369a 100644 --- a/workspaces/intelligent-assistant/e2e-tests/lightspeed.conversation.test.ts +++ b/workspaces/intelligent-assistant/e2e-tests/lightspeed.conversation.test.ts @@ -411,6 +411,37 @@ test.describe('Intelligent assistant conversation', () => { await sharedPage.keyboard.press('Escape'); }); + test('Vision models show screenshot indicator in the dropdown list only', async () => { + const visionAriaLabel = + translations['modelSelector.visionScreenshot.ariaLabel']; + const toggle = sharedPage.locator( + `button[aria-label="${translations['aria.chatbotSelector']}"]`, + ); + + await expect(toggle.getByLabel(visionAriaLabel)).toHaveCount(0); + + await toggle.click(); + + const visionItem = sharedPage.getByRole('menuitem', { + name: /mock-model-2/, + }); + await expect(visionItem.getByLabel(visionAriaLabel)).toBeVisible(); + + const textItem = sharedPage.getByRole('menuitem', { + name: 'mock-model-1', + }); + await expect(textItem.getByLabel(visionAriaLabel)).toHaveCount(0); + + await expect( + visionItem.locator('.lightspeed-model-tick-slot'), + ).toBeVisible(); + await expect( + textItem.locator('.lightspeed-model-tick-slot'), + ).toBeVisible(); + + await sharedPage.keyboard.press('Escape'); + }); + test('Model selector becomes disabled after sending a message', async () => { await sendMessage( LIGHTSPEED_E2E_DEFAULT_BOT_QUERY, diff --git a/workspaces/intelligent-assistant/e2e-tests/lightspeed.screen-context.test.ts b/workspaces/intelligent-assistant/e2e-tests/lightspeed.screen-context.test.ts new file mode 100644 index 00000000000..04dd207a87a --- /dev/null +++ b/workspaces/intelligent-assistant/e2e-tests/lightspeed.screen-context.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, type Page } from '@playwright/test'; +import { + conversations, + generateQueryResponse, + modelBaseUrl, +} from './fixtures/responses'; +import { + expectConversationArea, + openChatbot, + selectDisplayMode, + waitForBackstageCatalogReady, +} from './pages/LightspeedPage'; +import { + bootstrapLightspeedE2ePage, + LIGHTSPEED_E2E_DEFAULT_BOT_QUERY, +} from './utils/lightspeedE2eSetup'; +import { sendMessage } from './utils/testHelper'; +import { openChatbotSettings } from './utils/chatManagement'; +import { mockQuery } from './utils/devMode'; +import { + disableScreenContextViaKebab, + enableScreenContextViaKebab, + expectScreenContextChipHidden, + expectScreenContextPausedVisible, + expectScreenContextRecordingVisible, + expectScreenContextUnavailableVisible, + pauseScreenContextChip, + resumeScreenContextChip, + selectEnableScreenContext, + verifyEnableScreenContextOption, +} from './utils/screenContext'; +import type { LightspeedMessages } from './utils/translations'; + +test.describe('Intelligent assistant screen context', () => { + let translations: LightspeedMessages; + let sharedPage: Page; + + test.beforeAll(async ({ browser }) => { + const boot = await bootstrapLightspeedE2ePage(browser); + sharedPage = boot.page; + translations = boot.translations; + }); + + test.beforeEach(async () => { + await sharedPage.goto('/catalog'); + await waitForBackstageCatalogReady(sharedPage); + await openChatbot(sharedPage, translations); + }); + + test('kebab Enable shows recording chip; Disable hides it', async () => { + await expectScreenContextChipHidden(sharedPage); + + // Verify the enable item, then click it in the same open menu. + // Escape closes the chatbot panel (not just the menu), so do not dismiss + // and reopen Options between verify and enable. + await openChatbotSettings(sharedPage, translations); + await verifyEnableScreenContextOption(sharedPage, translations); + await selectEnableScreenContext(sharedPage, translations); + await expectScreenContextRecordingVisible(sharedPage); + + await disableScreenContextViaKebab(sharedPage, translations); + await expectScreenContextChipHidden(sharedPage); + }); + + test('chip pause/resume toggles Context: paused label', async () => { + await enableScreenContextViaKebab(sharedPage, translations); + await pauseScreenContextChip(sharedPage); + await expectScreenContextPausedVisible(sharedPage, translations); + + await resumeScreenContextChip(sharedPage, translations); + await expectScreenContextRecordingVisible(sharedPage); + + await disableScreenContextViaKebab(sharedPage, translations); + }); + + test('fullscreen shows Context: unavailable', async () => { + await enableScreenContextViaKebab(sharedPage, translations); + await selectDisplayMode(sharedPage, translations, 'Fullscreen'); + await expectConversationArea(sharedPage, translations, 'Fullscreen'); + // Guest sessions do not persist sharing across the fullscreen remount; + // re-enable so the unavailable chip can render on the new surface. + // Do not use enableScreenContextViaKebab — it asserts the recording chip. + await openChatbotSettings(sharedPage, translations); + await verifyEnableScreenContextOption(sharedPage, translations); + await selectEnableScreenContext(sharedPage, translations); + await expectScreenContextUnavailableVisible(sharedPage, translations); + }); + + test('paused send omits screen-context attachments', async () => { + await enableScreenContextViaKebab(sharedPage, translations); + await pauseScreenContextChip(sharedPage); + await expectScreenContextPausedVisible(sharedPage, translations); + + let capturedAttachments: Array<{ attachment_type?: string }> | undefined; + + await sharedPage.unroute(`${modelBaseUrl}/v1/query`); + await sharedPage.route(`${modelBaseUrl}/v1/query`, async route => { + const payload = route.request().postDataJSON(); + capturedAttachments = payload.attachments; + if (payload.conversation_id) { + conversations[1].conversation_id = payload.conversation_id; + } + const conversationId = + conversations[1].conversation_id ?? conversations[0].conversation_id; + await route.fulfill({ + body: generateQueryResponse(conversationId), + }); + }); + + await sendMessage( + LIGHTSPEED_E2E_DEFAULT_BOT_QUERY, + sharedPage, + translations, + ); + + const attachments = capturedAttachments ?? []; + expect( + attachments.some( + a => + a.attachment_type === 'image' || + a.attachment_type === 'configuration', + ), + ).toBe(false); + + await sharedPage.unroute(`${modelBaseUrl}/v1/query`); + await mockQuery( + sharedPage, + LIGHTSPEED_E2E_DEFAULT_BOT_QUERY, + conversations, + ); + + await disableScreenContextViaKebab(sharedPage, translations); + }); +}); diff --git a/workspaces/intelligent-assistant/e2e-tests/utils/chatManagement.ts b/workspaces/intelligent-assistant/e2e-tests/utils/chatManagement.ts index 90c32afb7d2..faba2ef796d 100644 --- a/workspaces/intelligent-assistant/e2e-tests/utils/chatManagement.ts +++ b/workspaces/intelligent-assistant/e2e-tests/utils/chatManagement.ts @@ -307,9 +307,11 @@ export const openChatbotSettings = async ( page: Page, translations: LightspeedMessages, ) => { - await page - .getByRole('button', { name: translations['aria.options.label'] }) - .click(); + const options = page + .locator('.pf-chatbot__header') + .getByRole('button', { name: translations['aria.options.label'] }); + await expect(options).toBeVisible({ timeout: 15_000 }); + await options.click(); }; export const verifyChatbotSettingsVisible = async ( diff --git a/workspaces/intelligent-assistant/e2e-tests/utils/screenContext.ts b/workspaces/intelligent-assistant/e2e-tests/utils/screenContext.ts new file mode 100644 index 00000000000..2e196a938ea --- /dev/null +++ b/workspaces/intelligent-assistant/e2e-tests/utils/screenContext.ts @@ -0,0 +1,125 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect, type Page } from '@playwright/test'; +import type { LightspeedMessages } from './translations'; +import { openChatbotSettings } from './chatManagement'; + +export const screenContextChip = (page: Page) => + page.locator('.lightspeed-page-context-label'); + +export const expectScreenContextChipHidden = async (page: Page) => { + await expect(screenContextChip(page)).toHaveCount(0); +}; + +export const expectScreenContextRecordingVisible = async (page: Page) => { + await expect( + page.locator('.lightspeed-page-context-label-recording'), + ).toBeVisible(); +}; + +export const expectScreenContextPausedVisible = async ( + page: Page, + _translations: LightspeedMessages, +) => { + await expect( + page.locator('.lightspeed-page-context-label-paused'), + ).toBeVisible(); +}; + +export const expectScreenContextUnavailableVisible = async ( + page: Page, + _translations: LightspeedMessages, +) => { + await expect( + page.locator('.lightspeed-page-context-label-unavailable'), + ).toBeVisible({ timeout: 15_000 }); +}; + +export const verifyEnableScreenContextOption = async ( + page: Page, + translations: LightspeedMessages, +) => { + await expect( + page.getByRole('menuitem', { + name: `${translations['settings.screenContext.enable']} ${translations['settings.screenContext.disabled.description']}`, + }), + ).toBeVisible(); +}; + +export const verifyDisableScreenContextOption = async ( + page: Page, + translations: LightspeedMessages, +) => { + await expect( + page.getByRole('menuitem', { + name: `${translations['settings.screenContext.disable']} ${translations['settings.screenContext.enabled.description']}`, + }), + ).toBeVisible(); +}; + +export const selectEnableScreenContext = async ( + page: Page, + translations: LightspeedMessages, +) => { + await page + .getByRole('menuitem', { + name: translations['settings.screenContext.enable'], + }) + .click(); +}; + +export const selectDisableScreenContext = async ( + page: Page, + translations: LightspeedMessages, +) => { + await page + .getByRole('menuitem', { + name: translations['settings.screenContext.disable'], + }) + .click(); +}; + +export const enableScreenContextViaKebab = async ( + page: Page, + translations: LightspeedMessages, +) => { + await openChatbotSettings(page, translations); + await verifyEnableScreenContextOption(page, translations); + await selectEnableScreenContext(page, translations); + await expectScreenContextRecordingVisible(page); +}; + +export const disableScreenContextViaKebab = async ( + page: Page, + translations: LightspeedMessages, +) => { + await openChatbotSettings(page, translations); + await verifyDisableScreenContextOption(page, translations); + await selectDisableScreenContext(page, translations); + await expectScreenContextChipHidden(page); +}; + +export const pauseScreenContextChip = async (page: Page) => { + await page.locator('.lightspeed-page-context-label-recording').click(); +}; + +export const resumeScreenContextChip = async ( + page: Page, + _translations: LightspeedMessages, +) => { + await page.locator('.lightspeed-page-context-label-paused').click(); +}; diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/README.md b/workspaces/intelligent-assistant/plugins/intelligent-assistant/README.md index 2bddc775153..b1f7f5e84f4 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/README.md +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/README.md @@ -161,7 +161,18 @@ Intelligent assistant supports multiple **display modes** from Settings (for exa ### Screen Context -When enabled, the assistant automatically captures structured page context from the current RHDH viewport and attaches it to each message. This provides the LLM with headings, tables, alerts, form fields, filters, and other on-screen content for more accurate, context-aware responses. +Screen context attaches structured page content from the current RHDH viewport to chat messages (headings, tables, alerts, form fields, filters, and optional screenshots) so the LLM can give more accurate, context-aware responses. + +Enablement has two levels: + +1. **Administrator** — set `screen-context.enabled: true` in `app-config.yaml` (default: `false`). +2. **User** — opt in from the chatbot options (kebab) menu with **Enable screen context**. Sharing is off until the user opts in. + +When sharing is on, a **context chip** in the message bar shows the current page label and state: + +- **Recording** — context will be attached on send; click the chip to pause. +- **Paused** — context is not attached; click to resume. +- **Unavailable** — fullscreen mode; switch to Overlay or Docked to use screen context. Configure in `app-config.yaml`: @@ -176,7 +187,8 @@ intelligent-assistant: maxChars: 8000 # Max characters extracted per page (default: 8000) ``` -- `screen-context.enabled` — enables the full screen-context feature (DOM extraction and optional screenshots). +- `screen-context.enabled` — enables the full screen-context feature (DOM extraction and optional screenshots). Users must still opt in via the kebab menu. +- `screen-context.screenshots.enabled` — toggles screenshot capture. Screenshots are only attached when the selected model supports vision (`supportsVision`). - `screen-context.dom-extraction.enabled` — toggles DOM text extraction independently of screenshots. - `screen-context.dom-extraction.maxChars` — caps the extracted text size to control LLM token usage. diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/report-alpha.api.md b/workspaces/intelligent-assistant/plugins/intelligent-assistant/report-alpha.api.md index 581b4d88891..7af5799c35e 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/report-alpha.api.md +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/report-alpha.api.md @@ -149,6 +149,9 @@ export const intelligentAssistantTranslationRef: TranslationRef< readonly 'chatbox.fileUpload.failed': string; readonly 'chatbox.fileUpload.infoText': string; readonly 'modelSelector.disabledTooltip': string; + readonly 'modelSelector.visionScreenshot.line1': string; + readonly 'modelSelector.visionScreenshot.line2': string; + readonly 'modelSelector.visionScreenshot.ariaLabel': string; readonly 'aria.chatbotSelector': string; readonly 'aria.important': string; readonly 'aria.chatHistoryMenu': string; @@ -228,6 +231,27 @@ export const intelligentAssistantTranslationRef: TranslationRef< readonly 'settings.savedPrompts.disable': string; readonly 'settings.savedPrompts.enabled.description': string; readonly 'settings.savedPrompts.disabled.description': string; + readonly 'settings.screenContext.enable': string; + readonly 'settings.screenContext.disable': string; + readonly 'settings.screenContext.enabled.description': string; + readonly 'settings.screenContext.disabled.description': string; + readonly 'contextChip.label.paused': string; + readonly 'contextChip.label.unavailable': string; + readonly 'contextChip.label.softwareTemplates': string; + readonly 'contextChip.tooltip.askAbout': string; + readonly 'contextChip.tooltip.template': string; + readonly 'contextChip.tooltip.search': string; + readonly 'contextChip.tooltip.paused': string; + readonly 'contextChip.tooltip.unavailable': string; + readonly 'contextChip.tooltip.line2.fullContext': string; + readonly 'contextChip.tooltip.line2.adminLimited': string; + readonly 'contextChip.tooltip.line2.screenshotOnly': string; + readonly 'contextChip.tooltip.line2.domOffNoVision': string; + readonly 'contextChip.tooltip.line2.textOnlyNoVision': string; + readonly 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': string; + readonly 'contextChip.tooltip.line2.textOnlyCombined': string; + readonly 'contextChip.aria.pause': string; + readonly 'contextChip.aria.resume': string; readonly 'settings.panel.title': string; readonly 'settings.mcp.label': string; readonly 'settings.prompt.label': string; diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx index 880e866144a..8f4d9853c09 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx @@ -100,6 +100,7 @@ import { useSavedPromptActions, useSavedPrompts, useSavedPromptsSettings, + useScreenContextSettings, useSettingsPanelUrlState, useSortSettings, useStopConversation, @@ -117,13 +118,13 @@ import { isSavedPromptConversationId, SAVED_PROMPT_CONVERSATION_ID_PREFIX, } from '../utils/buildConversationHistoryGroups'; -import { extractPageContext } from '../utils/dom-extractor'; import { ChatbotFootnoteWithIcon, getCategorizeMessages, getFootnoteProps, SortOption, } from '../utils/lightspeed-chatbox-utils'; +import { buildScreenContextAttachments } from '../utils/screen-context-utils'; import Attachment from './Attachment'; import { useFileAttachmentContext } from './AttachmentContext'; import { CollapsedHistoryStrip } from './CollapsedHistoryStrip'; @@ -144,6 +145,7 @@ import { } from './notebooks/SidebarCollapseIcon'; import { RenameConversationModal } from './RenameConversationModal'; import { SavedPromptMenuItems } from './SavedPromptMenuItems'; +import { ScreenContextChip } from './ScreenContextChip'; import { SettingsPanel } from './SettingsPanel'; import { ToastAlertGroup } from './ToastAlertGroup'; @@ -364,6 +366,15 @@ const StyledMessageBar = styled(MessageBar)(({ theme }) => ({ }, })); +const MessageBarActionsRow = styled('div')({ + display: 'flex', + flex: '1 1 0', + minWidth: 0, + overflow: 'hidden', + alignItems: 'center', + gap: 8, +}); + const StyledSelectList = styled(SelectList)({ padding: 0, margin: 0, @@ -593,7 +604,12 @@ type LightspeedChatProps = { avatar?: string; profileLoading: boolean; handleSelectedModel: (item: string) => void; - models: { label: string; value: string; provider: string }[]; + models: { + label: string; + value: string; + provider: string; + supportsVision?: boolean; + }[]; }; export const LightspeedChat = ({ @@ -625,6 +641,10 @@ export const LightspeedChat = ({ configApi.getOptionalNumber( 'intelligent-assistant.screen-context.dom-extraction.maxChars', ) ?? 8000; + const screenshotsEnabled = + configApi.getOptionalBoolean( + 'intelligent-assistant.screen-context.screenshots.enabled', + ) ?? true; const notebooksRouteMatch = useMatch(`${LIGHTSPEED_PATH}/notebooks`); const notebookViewRouteMatch = useMatch( @@ -1051,6 +1071,26 @@ export const LightspeedChat = ({ const { isSavedPromptsEnabled, handleSavedPromptsToggle } = useSavedPromptsSettings(user); + const { + isScreenContextSharingEnabled, + isScreenContextPaused, + handleScreenContextSharingToggle, + toggleScreenContextPaused, + } = useScreenContextSettings(user); + + const selectedModelSupportsVision = useMemo( + () => models.find(m => m.value === selectedModel)?.supportsVision ?? false, + [models, selectedModel], + ); + + let screenContextChipState: 'recording' | 'paused' | 'unavailable' = + 'recording'; + if (isFullscreenMode) { + screenContextChipState = 'unavailable'; + } else if (isScreenContextPaused) { + screenContextChipState = 'paused'; + } + const { selectedSort, handleSortChange } = useSortSettings(user); const { @@ -1296,29 +1336,32 @@ export const LightspeedChat = ({ prompt: message.toString(), }), ); - const allAttachments = getAttachments(fileContents); - if (screenContextEnabled && domExtractionEnabled) { + const messageText = message.toString(); + setIsSendButtonDisabled(true); + + void (async () => { + const allAttachments = getAttachments(fileContents); try { - const domContext = extractPageContext({ - maxChars: domExtractionMaxChars, + const screenAttachments = await buildScreenContextAttachments({ + adminEnabled: screenContextEnabled, + sharingEnabled: isScreenContextSharingEnabled, + paused: isScreenContextPaused, + isFullscreen: isFullscreenMode, + domEnabled: domExtractionEnabled, + screenshotsEnabled, + supportsVision: selectedModelSupportsVision, + domExtractionMaxChars, }); - if (domContext) { - allAttachments.push({ - attachment_type: 'configuration', - content_type: 'text/plain', - content: domContext, - }); - } + allAttachments.push(...screenAttachments); } catch { - // DOM extraction failure is non-fatal; proceed without page context + // Screen context failure is non-fatal } - } - handleInputPrompt(message.toString(), allAttachments); - setIsSendButtonDisabled(true); - setFileContents([]); - setDraftMessage(''); + handleInputPrompt(messageText, allAttachments); + setFileContents([]); + setDraftMessage(''); + })(); }; const handleApplySavedPrompt = useCallback( @@ -1961,13 +2004,26 @@ export const LightspeedChat = ({ }, }} additionalActions={ - 0} - disabledTooltip={t('modelSelector.disabledTooltip')} - /> + + 0} + disabledTooltip={t('modelSelector.disabledTooltip')} + /> + {screenContextEnabled && + isScreenContextSharingEnabled && + showChatPanel && ( + + )} + } forceMultilineLayout allowedFileTypes={supportedFileTypes} @@ -2075,6 +2131,7 @@ export const LightspeedChat = ({ isCompact={!isFullscreenMode} compactDrawerOpen={!isFullscreenMode && isChatHistoryDrawerOpen} mcpDrawerFix={isSettingsOpen && !isChatHistoryDrawerOpen} + data-screen-capture-exclude > {showCompactSettings ? ( settingsPanel @@ -2176,6 +2233,9 @@ export const LightspeedChat = ({ setDisplayMode={setDisplayModeFromHeader} displayMode={displayMode} onPinnedChatsToggle={handlePinningChatsToggle} + screenContextAdminEnabled={screenContextEnabled} + isScreenContextSharingEnabled={isScreenContextSharingEnabled} + onScreenContextSharingToggle={handleScreenContextSharingToggle} onMcpSettingsClick={() => { openSettings( mcpToolsPermissionResolved diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedChatBoxHeader.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedChatBoxHeader.tsx index 33fa845518d..3ac92820a71 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedChatBoxHeader.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedChatBoxHeader.tsx @@ -54,6 +54,9 @@ type LightspeedChatBoxHeaderProps = { onPinnedChatsToggle: (state: boolean) => void; isSavedPromptsEnabled: boolean; onSavedPromptsToggle: (state: boolean) => void; + screenContextAdminEnabled?: boolean; + isScreenContextSharingEnabled?: boolean; + onScreenContextSharingToggle?: (state: boolean) => void; onMcpSettingsClick: () => void; isModelSelectorDisabled?: boolean; hideModelSelector?: boolean; @@ -99,6 +102,9 @@ export const LightspeedChatBoxHeader = ({ onPinnedChatsToggle, isSavedPromptsEnabled, onSavedPromptsToggle, + screenContextAdminEnabled = false, + isScreenContextSharingEnabled = false, + onScreenContextSharingToggle, onMcpSettingsClick, isModelSelectorDisabled = false, hideModelSelector = false, @@ -288,6 +294,32 @@ export const LightspeedChatBoxHeader = ({ {t('settings.savedPrompts.enable')} )} + {screenContextAdminEnabled && + (isScreenContextSharingEnabled ? ( + } + description={t( + 'settings.screenContext.enabled.description', + )} + onClick={() => onScreenContextSharingToggle?.(false)} + > + {t('settings.screenContext.disable')} + + ) : ( + } + description={t( + 'settings.screenContext.disabled.description', + )} + onClick={() => onScreenContextSharingToggle?.(true)} + > + {t('settings.screenContext.enable')} + + ))} { label: m.provider_resource_id, value: m.provider_resource_id, provider: m.provider_id, + supportsVision: m.supportsVision, })) : [], [models], diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedDrawerProvider.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedDrawerProvider.tsx index 57af93f6bd9..85c4dec5fa3 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedDrawerProvider.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightspeedDrawerProvider.tsx @@ -65,6 +65,7 @@ export const LightspeedDrawerProvider = ({ children }: PropsWithChildren) => { onEscapePress={() => closeChatbot()} ouiaId="LightspeedChatbotModal" aria-labelledby="lightspeed-chatpopup-modal" + data-screen-capture-exclude > diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/MessageBarModelSelector.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/MessageBarModelSelector.tsx index b2a98fe3c51..471fa005cdf 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/MessageBarModelSelector.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/MessageBarModelSelector.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Ref, useEffect, useState } from 'react'; +import { Fragment, Ref, useEffect, useMemo, useState } from 'react'; import { styled } from '@mui/material/styles'; import Typography from '@mui/material/Typography'; @@ -26,13 +26,22 @@ import { MenuToggleElement, Tooltip, } from '@patternfly/react-core'; -import { AngleDownIcon } from '@patternfly/react-icons'; +import { + AngleDownIcon, + CheckIcon, + OutlinedImageIcon, +} from '@patternfly/react-icons'; import { useTranslation } from '../hooks/useTranslation'; type MessageBarModelSelectorProps = { selectedModel: string; - models: { label: string; value: string; provider: string }[]; + models: { + label: string; + value: string; + provider: string; + supportsVision?: boolean; + }[]; onSelect: (model: string) => void; disabled?: boolean; disabledTooltip?: string; @@ -59,11 +68,73 @@ const SelectorToggle = styled(MenuToggle)(({ theme }) => ({ }, })); +const VisionIndicatorIcon = styled(OutlinedImageIcon)(({ theme }) => ({ + width: 16, + height: 16, + color: theme.palette.text.secondary, + flexShrink: 0, +})); + +const VisionIndicatorWrap = styled('span')({ + display: 'inline-flex', + alignItems: 'center', + lineHeight: 1, +}); + +/** Fixed-width slot so vision icons align across rows (empty when unsupported). */ +const VisionSlot = styled('span')({ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + flexShrink: 0, + width: 16, + height: 16, +}); + +/** Fixed-width slot so the select tick aligns across rows (empty when not selected). */ +const TickSlot = styled('span')(({ theme }) => ({ + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + flexShrink: 0, + width: '1em', + height: '1em', + color: theme.palette.primary.main, +})); + +const ModelLabel = styled('span')({ + flex: '1 1 auto', + minWidth: 0, + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', +}); + +const ModelItemContent = styled('span')({ + display: 'flex', + alignItems: 'center', + gap: 8, + width: '100%', + minWidth: 0, +}); + const StyledDropdown = styled(Dropdown)({ '& ul, & li': { padding: 0, margin: 0, }, + '& .pf-v6-c-menu__item-main': { + width: '100%', + }, + '& .pf-v6-c-menu__item-text': { + flex: '1 1 auto', + minWidth: 0, + overflow: 'hidden', + }, + // We render our own fixed tick column; hide PF's conditional select icon. + '& .pf-v6-c-menu__item-select-icon': { + display: 'none', + }, }); export const MessageBarModelSelector = ({ @@ -85,6 +156,17 @@ export const MessageBarModelSelector = ({ const selectedModelLabel = models.find(m => m.value === selectedModel)?.label ?? selectedModel; + const visionScreenshotTooltip = useMemo( + () => ( + + {t('modelSelector.visionScreenshot.line1')} +
+ {t('modelSelector.visionScreenshot.line2')} + + ), + [t], + ); + const toggle = (toggleRef: Ref) => ( 10 ? '240px' : undefined} > - {models.map(model => ( - - {model.label} - - ))} + {models.map(model => { + const isSelected = selectedModel === model.value; + return ( + + + {model.label} + {model.supportsVision ? ( + + + event.stopPropagation()} + onMouseDown={event => event.stopPropagation()} + > + + + + + ) : ( + + )} + + {isSelected ? : null} + + + + ); + })} ); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/ScreenContextChip.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/ScreenContextChip.tsx new file mode 100644 index 00000000000..8774882bb0c --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/ScreenContextChip.tsx @@ -0,0 +1,236 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Fragment, useMemo } from 'react'; +import { useLocation } from 'react-router-dom'; + +import { keyframes, styled } from '@mui/material/styles'; +import { Label, Tooltip } from '@patternfly/react-core'; +import { PauseIcon } from '@patternfly/react-icons'; + +import { useScreenContextLabelRevision } from '../hooks/useScreenContextLabelRevision'; +import { useTranslation } from '../hooks/useTranslation'; +import { + getScreenContextTooltipLine1Text, + getScreenContextTooltipLine2Key, + resolveScreenContextChipLabel, + ScreenContextTooltipLine2Key, + SOFTWARE_TEMPLATES_LIST_CHIP_LABEL, +} from '../utils/screen-context-utils'; + +export type ScreenContextChipState = 'recording' | 'paused' | 'unavailable'; + +/** Prototype max width; shorter labels shrink to content. */ +const CHIP_MAX_WIDTH_PX = 150; +const CHIP_HEIGHT_PX = 24; + +const pulse = keyframes` + 0% { opacity: 0.45; transform: scale(0.92); } + 50% { opacity: 1; transform: scale(1); } + 100% { opacity: 0.45; transform: scale(0.92); } +`; + +const ChipWrap = styled('div')({ + display: 'inline-flex', + height: CHIP_HEIGHT_PX, + maxWidth: CHIP_MAX_WIDTH_PX, + minWidth: 0, + flexShrink: 0, +}); + +const FixedSizeChipLabel = styled(Label)({ + height: CHIP_HEIGHT_PX, + maxWidth: CHIP_MAX_WIDTH_PX, + width: 'max-content', + boxSizing: 'border-box', + display: 'inline-flex', + alignItems: 'center', + flexShrink: 0, + '--pf-v6-c-label--PaddingBlockStart': '0', + '--pf-v6-c-label--PaddingBlockEnd': '0', + '--pf-v6-c-label--MinWidth': 'auto', + '& .pf-v6-c-label__content': { + maxWidth: '100%', + height: '100%', + minWidth: 0, + boxSizing: 'border-box', + }, + '& .pf-v6-c-label__text': { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + minWidth: 0, + }, +}); + +const RecordingDot = styled('span')({ + display: 'inline-block', + width: 8, + height: 8, + borderRadius: '50%', + backgroundColor: 'var(--pf-t--global--icon--color--status--danger--default)', + animation: `${pulse} 1.4s ease-in-out infinite`, +}); + +const ChipText = styled('span')({ + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + minWidth: 0, +}); + +const LINE2_I18N: Record = { + fullContext: 'contextChip.tooltip.line2.fullContext', + adminLimited: 'contextChip.tooltip.line2.adminLimited', + screenshotOnly: 'contextChip.tooltip.line2.screenshotOnly', + domOffNoVision: 'contextChip.tooltip.line2.domOffNoVision', + textOnlyNoVision: 'contextChip.tooltip.line2.textOnlyNoVision', + textOnlyAdminScreenshotsOff: + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff', + textOnlyCombined: 'contextChip.tooltip.line2.textOnlyCombined', +}; + +export type ScreenContextChipProps = { + state: ScreenContextChipState; + chipLabel?: string; + domEnabled: boolean; + screenshotsEnabled: boolean; + supportsVision: boolean; + onTogglePaused?: () => void; +}; + +export const ScreenContextChip = ({ + state, + chipLabel, + domEnabled, + screenshotsEnabled, + supportsVision, + onTogglePaused, +}: ScreenContextChipProps) => { + const { t } = useTranslation(); + const routerLocation = useLocation(); + useScreenContextLabelRevision(); + + const { label: resolvedLabel, routeKind } = resolveScreenContextChipLabel({ + pathname: routerLocation.pathname, + search: routerLocation.search, + }); + + const label = chipLabel ?? resolvedLabel; + const displayLabel = + label === SOFTWARE_TEMPLATES_LIST_CHIP_LABEL + ? t('contextChip.label.softwareTemplates') + : label; + + const tooltipContent = useMemo(() => { + if (state === 'unavailable') { + return t('contextChip.tooltip.unavailable'); + } + if (state === 'paused') { + return t('contextChip.tooltip.paused'); + } + + const line1 = getScreenContextTooltipLine1Text( + { + chipLabel: displayLabel, + routeKind, + pathname: routerLocation.pathname, + }, + (key, options) => t(key as any, options as any), + ); + const line2Key = getScreenContextTooltipLine2Key({ + domEnabled, + screenshotsEnabled, + supportsVision, + }); + const line2 = t(LINE2_I18N[line2Key] as any, {} as any); + return ( + + {line1} +
+ {line2} + + ); + }, [ + state, + displayLabel, + routeKind, + routerLocation.pathname, + domEnabled, + screenshotsEnabled, + supportsVision, + t, + ]); + + if (state === 'unavailable') { + return ( + + + + {t('contextChip.label.unavailable')} + + + + ); + } + + if (state === 'paused') { + return ( + + + } + className="lightspeed-page-context-chip-clickable lightspeed-context-chip-paused-label" + aria-label={t('contextChip.aria.resume')} + > + {t('contextChip.label.paused')} + + + + ); + } + + return ( + + + } + className="lightspeed-context-chip-recording-label" + aria-label={t('contextChip.aria.pause' as any, { + label: displayLabel, + })} + > + {displayLabel} + + + + ); +}; diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChat.test.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChat.test.tsx index a6011987ffa..70413f4c887 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChat.test.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChat.test.tsx @@ -19,6 +19,7 @@ import { configApiRef, IdentityApi, identityApiRef, + storageApiRef, } from '@backstage/core-plugin-api'; import { usePermission } from '@backstage/plugin-permission-react'; import { mockApis, TestApiProvider } from '@backstage/test-utils'; @@ -257,6 +258,7 @@ const setupLightspeedChat = ( apis={[ [identityApiRef, identityApi], [configApiRef, configApi], + [storageApiRef, mockApis.storage()], [lightspeedApiRef, mockLightspeedApi], [notebooksApiRef, mockNotebooksApi], ]} diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChatBoxHeader.test.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChatBoxHeader.test.tsx index 8dcf5a185e6..5bff09b9b9c 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChatBoxHeader.test.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/LightspeedChatBoxHeader.test.tsx @@ -93,4 +93,51 @@ describe('LightspeedChatBoxHeader', () => { screen.queryByText('MCP and Prompt Settings'), ).not.toBeInTheDocument(); }); + + it('hides screen context kebab items when admin flag is off', () => { + render( + , + ); + + fireEvent.click(screen.getByLabelText('Options')); + expect(screen.queryByText('Enable screen context')).not.toBeInTheDocument(); + expect( + screen.queryByText('Disable screen context'), + ).not.toBeInTheDocument(); + }); + + it('shows Enable screen context when admin on and sharing off', () => { + const onToggle = jest.fn(); + render( + , + ); + + fireEvent.click(screen.getByLabelText('Options')); + fireEvent.click(screen.getByText('Enable screen context')); + expect(onToggle).toHaveBeenCalledWith(true); + }); + + it('shows Disable screen context when admin on and sharing on', () => { + const onToggle = jest.fn(); + render( + , + ); + + fireEvent.click(screen.getByLabelText('Options')); + fireEvent.click(screen.getByText('Disable screen context')); + expect(onToggle).toHaveBeenCalledWith(false); + }); }); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/MessageBarModelSelector.test.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/MessageBarModelSelector.test.tsx index d7dacbeb300..ee2731eee85 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/MessageBarModelSelector.test.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/MessageBarModelSelector.test.tsx @@ -224,6 +224,68 @@ describe('MessageBarModelSelector', () => { expect(screen.getByText('Granite 3.3')).toBeInTheDocument(); }); + it('shows vision screenshot indicator on vision-capable models in the list', async () => { + const modelsWithVision = [ + { + label: 'Granite Vision', + value: 'granite-vision', + provider: 'ibm', + supportsVision: true, + }, + { + label: 'Granite Text', + value: 'granite-text', + provider: 'ibm', + supportsVision: false, + }, + ]; + + render( + , + ); + + const toggle = screen.getByRole('button', { name: 'Chatbot selector' }); + expect( + screen.queryByLabelText('Vision model screenshot context'), + ).not.toBeInTheDocument(); + + await userEvent.click(toggle); + + await waitFor(() => { + expect( + screen.getByLabelText('Vision model screenshot context'), + ).toBeInTheDocument(); + }); + + const visionItem = screen.getByRole('menuitem', { + name: /Granite Vision/, + }); + const textItem = screen.getByRole('menuitem', { name: 'Granite Text' }); + + expect( + visionItem.querySelector('.lightspeed-model-vision-slot'), + ).toBeInTheDocument(); + expect( + textItem.querySelector('.lightspeed-model-vision-slot'), + ).toBeInTheDocument(); + expect( + textItem.querySelector('[aria-label="Vision model screenshot context"]'), + ).not.toBeInTheDocument(); + + // Fixed tick column on every row; checkmark only on the selected item. + expect( + visionItem.querySelector('.lightspeed-model-tick-slot') + ?.childElementCount, + ).toBe(0); + expect( + textItem.querySelector('.lightspeed-model-tick-slot')?.childElementCount, + ).toBe(1); + }); + it('should render with empty models list', () => { render( , + initialPath = '/catalog/default/component/demo', +) => { + window.history.pushState({}, '', initialPath); + return render( + + + , + ); +}; + +jest.mock('../../hooks/useTranslation', () => ({ + useTranslation: () => ({ + t: (key: string, options?: { label?: string }) => { + if (options?.label) { + return `${key}:${options.label}`; + } + return key; + }, + }), +})); + +describe('ScreenContextChip', () => { + it('renders recording state with chip label', () => { + renderChip({ + state: 'recording', + chipLabel: 'Prototype — Sample documentation', + domEnabled: true, + screenshotsEnabled: true, + supportsVision: false, + }); + + expect( + screen.getByText('Prototype — Sample documentation'), + ).toBeInTheDocument(); + expect( + screen.getByLabelText( + 'contextChip.aria.pause:Prototype — Sample documentation', + ), + ).toBeInTheDocument(); + }); + + it('renders paused label', () => { + renderChip({ + state: 'paused', + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }); + + expect(screen.getByText('contextChip.label.paused')).toBeInTheDocument(); + }); + + it('renders unavailable label in fullscreen', () => { + renderChip({ + state: 'unavailable', + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }); + + expect( + screen.getByText('contextChip.label.unavailable'), + ).toBeInTheDocument(); + }); + + it('resolves chip label from DOM when route changes', () => { + document.body.innerHTML = ` +
+
+

Entity Alpha

+
+
+ `; + renderChip( + { + state: 'recording', + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }, + '/catalog/default/component/alpha', + ); + expect( + screen.getByLabelText('contextChip.aria.pause:Entity Alpha'), + ).toBeInTheDocument(); + }); + + it('updates chip text when the URL changes without refresh', () => { + document.body.innerHTML = ` +
+
+

Entity Alpha

+
+
+ `; + renderChip( + { + state: 'recording', + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }, + '/catalog/default/component/alpha', + ); + expect( + screen.getByLabelText('contextChip.aria.pause:Entity Alpha'), + ).toBeInTheDocument(); + + document.querySelector('.bui-HeaderTitle')!.textContent = 'Entity Beta'; + act(() => { + window.history.pushState({}, '', '/catalog/default/component/beta'); + }); + + expect( + screen.getByLabelText('contextChip.aria.pause:Entity Beta'), + ).toBeInTheDocument(); + }); + + it('shows search query on search route', () => { + renderChip( + { + state: 'recording', + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }, + '/search?query=payments-api', + ); + expect(screen.getByText('payments-api')).toBeInTheDocument(); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/iaRbacPermissionGating.test.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/iaRbacPermissionGating.test.tsx index 219ddc85b29..d65180a379d 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/iaRbacPermissionGating.test.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/__tests__/iaRbacPermissionGating.test.tsx @@ -20,6 +20,7 @@ import { configApiRef, IdentityApi, identityApiRef, + storageApiRef, } from '@backstage/core-plugin-api'; import { usePermission } from '@backstage/plugin-permission-react'; import { mockApis, TestApiProvider } from '@backstage/test-utils'; @@ -361,6 +362,7 @@ const setupLightspeedChat = (initialPath = '/intelligent-assistant') => ( apis={[ [identityApiRef, identityApi], [configApiRef, configApi], + [storageApiRef, mockApis.storage()], [lightspeedApiRef, mockLightspeedApi], [notebooksApiRef, mockNotebooksApi], ]} @@ -393,6 +395,7 @@ const setupLightspeedChatContainer = ( apis={[ [identityApiRef, identityApi], [configApiRef, configApi], + [storageApiRef, mockApis.storage()], [lightspeedApiRef, mockLightspeedApi], [notebooksApiRef, mockNotebooksApi], ]} diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/screenContextLabelSubscription.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/screenContextLabelSubscription.test.ts new file mode 100644 index 00000000000..3f2904904d6 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/screenContextLabelSubscription.test.ts @@ -0,0 +1,74 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + getScreenContextLabelSnapshot, + isInsideScreenCaptureExclude, + resetScreenContextLabelSnapshotCache, +} from '../screenContextLabelSubscription'; + +describe('getScreenContextLabelSnapshot', () => { + beforeEach(() => { + resetScreenContextLabelSnapshotCache(); + document.body.innerHTML = `
`; + window.history.pushState( + {}, + '', + '/create/templates/default/argocd-template', + ); + }); + + it('includes resolved chip label so template card titles refresh the snapshot', () => { + expect(getScreenContextLabelSnapshot()).toContain('argocd-template'); + + const main = document.querySelector('main'); + if (main) { + main.innerHTML = ` +

+ Add ArgoCD to an existing project template +

+ `; + } + + const snapshot = getScreenContextLabelSnapshot(); + expect(snapshot).toContain('Add ArgoCD to an existing project templ'); + expect(snapshot.endsWith('argocd-template')).toBe(false); + }); +}); + +describe('isInsideScreenCaptureExclude', () => { + beforeEach(() => { + document.body.innerHTML = ''; + }); + + it('returns true for a Text node inside an excluded element', () => { + document.body.innerHTML = ` +
+

streaming

+
+ `; + const textNode = document.getElementById('stream')?.firstChild; + expect(textNode?.nodeType).toBe(Node.TEXT_NODE); + expect(isInsideScreenCaptureExclude(textNode as Node)).toBe(true); + }); + + it('returns false for a Text node outside an excluded element', () => { + document.body.innerHTML = `

Catalog

`; + const textNode = document.getElementById('page-title')?.firstChild; + expect(textNode?.nodeType).toBe(Node.TEXT_NODE); + expect(isInsideScreenCaptureExclude(textNode as Node)).toBe(false); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useScreenContextLabelRevision.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useScreenContextLabelRevision.test.ts new file mode 100644 index 00000000000..68dd7c6df82 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useScreenContextLabelRevision.test.ts @@ -0,0 +1,60 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { act, renderHook } from '@testing-library/react'; + +import { resetScreenContextLabelSnapshotCache } from '../screenContextLabelSubscription'; +import { useScreenContextLabelRevision } from '../useScreenContextLabelRevision'; + +describe('useScreenContextLabelRevision', () => { + beforeEach(() => { + resetScreenContextLabelSnapshotCache(); + document.body.innerHTML = ` +
+

Page A

+
+ `; + window.history.pushState({}, '', '/page-a'); + }); + + it('updates when the URL changes', () => { + const { result } = renderHook(() => useScreenContextLabelRevision()); + expect(result.current).toContain('/page-a'); + + act(() => { + window.history.pushState({}, '', '/page-b'); + }); + + expect(result.current).toContain('/page-b'); + }); + + it('updates when the page title in chrome changes', async () => { + const { result } = renderHook(() => useScreenContextLabelRevision()); + expect(result.current).toContain('Page A'); + + await act(async () => { + const title = document.querySelector('.bui-HeaderTitle'); + if (title) { + title.textContent = 'Page B'; + } + await new Promise(resolve => { + requestAnimationFrame(() => resolve()); + }); + }); + + expect(result.current).toContain('Page B'); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useScreenContextSettings.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useScreenContextSettings.test.ts new file mode 100644 index 00000000000..4ef144e0962 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useScreenContextSettings.test.ts @@ -0,0 +1,123 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createElement, ReactNode } from 'react'; + +import { StorageApi, storageApiRef } from '@backstage/core-plugin-api'; +import { mockApis, TestApiProvider } from '@backstage/test-utils'; + +import { act, renderHook, waitFor } from '@testing-library/react'; + +import { useScreenContextSettings } from '../useScreenContextSettings'; + +describe('useScreenContextSettings', () => { + const mockUser = 'user:default/john'; + const guestUser = 'user:default/guest'; + let mockStorageApi: StorageApi; + + const createWrapper = (storageApi: StorageApi) => { + return ({ children }: { children: ReactNode }) => + createElement(TestApiProvider, { + apis: [[storageApiRef, storageApi]], + children, + }); + }; + + beforeEach(() => { + mockStorageApi = mockApis.storage(); + }); + + it('defaults sharing to false and paused to false for signed-in user', () => { + const { result } = renderHook(() => useScreenContextSettings(mockUser), { + wrapper: createWrapper(mockStorageApi), + }); + + expect(result.current.isScreenContextSharingEnabled).toBe(false); + expect(result.current.isScreenContextPaused).toBe(false); + }); + + it('toggles sharing and resets paused when disabling', async () => { + const { result } = renderHook(() => useScreenContextSettings(mockUser), { + wrapper: createWrapper(mockStorageApi), + }); + + act(() => { + result.current.handleScreenContextSharingToggle(true); + }); + + await waitFor(() => { + expect(result.current.isScreenContextSharingEnabled).toBe(true); + }); + + act(() => { + result.current.setScreenContextPaused(true); + }); + + await waitFor(() => { + expect(result.current.isScreenContextPaused).toBe(true); + }); + + act(() => { + result.current.handleScreenContextSharingToggle(false); + }); + + await waitFor(() => { + expect(result.current.isScreenContextSharingEnabled).toBe(false); + expect(result.current.isScreenContextPaused).toBe(false); + }); + }); + + it('toggles paused state', async () => { + const { result } = renderHook(() => useScreenContextSettings(mockUser), { + wrapper: createWrapper(mockStorageApi), + }); + + act(() => { + result.current.handleScreenContextSharingToggle(true); + result.current.toggleScreenContextPaused(); + }); + + await waitFor(() => { + expect(result.current.isScreenContextPaused).toBe(true); + }); + + act(() => { + result.current.toggleScreenContextPaused(); + }); + + await waitFor(() => { + expect(result.current.isScreenContextPaused).toBe(false); + }); + }); + + it('does not persist for guest users', async () => { + const { result } = renderHook(() => useScreenContextSettings(guestUser), { + wrapper: createWrapper(mockStorageApi), + }); + + act(() => { + result.current.handleScreenContextSharingToggle(true); + }); + + await waitFor(() => { + expect(result.current.isScreenContextSharingEnabled).toBe(true); + }); + + const bucket = mockStorageApi.forBucket('lightspeed'); + expect( + bucket.snapshot('screenContextSharingEnabled').value, + ).toBeUndefined(); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useWindowLocation.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useWindowLocation.test.ts new file mode 100644 index 00000000000..247189a36e7 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/__tests__/useWindowLocation.test.ts @@ -0,0 +1,36 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { act, renderHook } from '@testing-library/react'; + +import { useWindowLocation } from '../useWindowLocation'; + +describe('useWindowLocation', () => { + beforeEach(() => { + window.history.pushState({}, '', '/'); + }); + + it('updates when history.pushState changes the URL', () => { + const { result } = renderHook(() => useWindowLocation()); + expect(result.current.pathname).toBe('/'); + + act(() => { + window.history.pushState({}, '', '/catalog/default/component/demo'); + }); + + expect(result.current.pathname).toBe('/catalog/default/component/demo'); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/index.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/index.ts index e532d8e0dda..a1f5edfd2f1 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/index.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/index.ts @@ -32,6 +32,9 @@ export * from './useDisplayModeSettings'; export * from './notebooks/useNotebookSession'; export * from './notebooks/useNotebookSessions'; export * from './usePinnedChatsSettings'; +export * from './useScreenContextSettings'; +export * from './useScreenContextLabelRevision'; +export * from './useWindowLocation'; export * from './useSavedPromptsSettings'; export * from './useSavedPrompts'; export * from './useSavedPromptActions'; diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/screenContextLabelSubscription.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/screenContextLabelSubscription.ts new file mode 100644 index 00000000000..d5e03f3c909 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/screenContextLabelSubscription.ts @@ -0,0 +1,156 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PAGE_CHROME_ROOT_SELECTORS } from '../utils/dom-selectors'; +import { resolveScreenContextChipLabel } from '../utils/screen-context-utils'; + +const listeners = new Set<() => void>(); +let historyPatched = false; +let pageChromeObserver: MutationObserver | undefined; +let notifyScheduled = false; + +function notifyScreenContextLabelListeners() { + listeners.forEach(listener => listener()); +} + +function scheduleNotifyScreenContextLabelListeners() { + if (notifyScheduled || typeof window === 'undefined') { + return; + } + notifyScheduled = true; + window.requestAnimationFrame(() => { + notifyScheduled = false; + notifyScreenContextLabelListeners(); + }); +} + +/** @internal Exported for unit tests. */ +export function isInsideScreenCaptureExclude(node: Node): boolean { + const el = node instanceof Element ? node : node.parentElement; + return Boolean(el?.closest('[data-screen-capture-exclude]')); +} + +function shouldReactToMutation(mutations: MutationRecord[]): boolean { + for (const mutation of mutations) { + if ( + mutation.target instanceof Node && + isInsideScreenCaptureExclude(mutation.target) + ) { + continue; + } + return true; + } + return false; +} + +const observedChromeNodes = new WeakSet(); + +function collectPageChromeObserveTargets(): Node[] { + const observeTargets: Node[] = []; + const titleEl = document.querySelector('title'); + if (titleEl) { + observeTargets.push(titleEl); + } + for (const selector of PAGE_CHROME_ROOT_SELECTORS) { + const node = document.querySelector(selector); + if (node && !observeTargets.includes(node)) { + observeTargets.push(node); + } + } + if (observeTargets.length === 0 && document.body) { + observeTargets.push(document.body); + } + return observeTargets; +} + +function ensurePageChromeObserver() { + if (typeof window === 'undefined') { + return; + } + + if (!pageChromeObserver) { + pageChromeObserver = new MutationObserver(mutations => { + if (shouldReactToMutation(mutations)) { + scheduleNotifyScreenContextLabelListeners(); + } + }); + } + + for (const target of collectPageChromeObserveTargets()) { + if (observedChromeNodes.has(target)) { + continue; + } + observedChromeNodes.add(target); + pageChromeObserver.observe(target, { + subtree: true, + childList: true, + characterData: true, + }); + } +} + +/** + * Patch at module load so we wrap Backstage / React Router history wrappers too. + */ +export function patchBrowserHistoryForScreenContext() { + if (historyPatched || typeof window === 'undefined') { + return; + } + historyPatched = true; + + window.addEventListener('popstate', notifyScreenContextLabelListeners); + window.addEventListener('hashchange', notifyScreenContextLabelListeners); + + const { pushState, replaceState } = window.history; + window.history.pushState = function pushStatePatched(...args) { + pushState.apply(this, args); + notifyScreenContextLabelListeners(); + }; + window.history.replaceState = function replaceStatePatched(...args) { + replaceState.apply(this, args); + notifyScreenContextLabelListeners(); + }; +} + +patchBrowserHistoryForScreenContext(); + +let cachedSnapshot = ''; + +export function getScreenContextLabelSnapshot(): string { + const pathname = window.location.pathname; + const search = window.location.search; + const { label } = resolveScreenContextChipLabel({ pathname, search }); + const next = `${pathname}\0${search}\0${label}`; + if (next !== cachedSnapshot) { + cachedSnapshot = next; + } + return cachedSnapshot; +} + +export function subscribeToScreenContextLabel( + onStoreChange: () => void, +): () => void { + ensurePageChromeObserver(); + listeners.add(onStoreChange); + return () => { + listeners.delete(onStoreChange); + }; +} + +/** @internal For unit tests only. */ +export function resetScreenContextLabelSnapshotCache(): void { + cachedSnapshot = ''; +} diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useScreenContextLabelRevision.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useScreenContextLabelRevision.ts new file mode 100644 index 00000000000..eaab20b6cce --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useScreenContextLabelRevision.ts @@ -0,0 +1,34 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useSyncExternalStore } from 'react'; + +import { + getScreenContextLabelSnapshot, + subscribeToScreenContextLabel, +} from './screenContextLabelSubscription'; + +/** + * Re-renders when the browser URL or main page chrome title changes + * (history navigation, async header updates after route transitions). + */ +export function useScreenContextLabelRevision(): string { + return useSyncExternalStore( + subscribeToScreenContextLabel, + getScreenContextLabelSnapshot, + getScreenContextLabelSnapshot, + ); +} diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useScreenContextSettings.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useScreenContextSettings.ts new file mode 100644 index 00000000000..6fc16f7b692 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useScreenContextSettings.ts @@ -0,0 +1,161 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { useCallback, useEffect, useState } from 'react'; + +import { storageApiRef, useApi } from '@backstage/core-plugin-api'; + +import { isGuestUser } from '../utils/user-utils'; + +const BUCKET_NAME = 'lightspeed'; +const SHARING_ENABLED_KEY = 'screenContextSharingEnabled'; +const PAUSED_KEY = 'screenContextPaused'; + +type UseScreenContextSettingsReturn = { + isScreenContextSharingEnabled: boolean; + isScreenContextPaused: boolean; + handleScreenContextSharingToggle: (enabled: boolean) => void; + toggleScreenContextPaused: () => void; + setScreenContextPaused: (paused: boolean) => void; +}; + +/** + * Persists screen context user preference (opt-in sharing + pause state). + */ +export const useScreenContextSettings = ( + user: string | undefined, +): UseScreenContextSettingsReturn => { + const storageApi = useApi(storageApiRef); + const bucket = storageApi.forBucket(BUCKET_NAME); + + const [isScreenContextSharingEnabled, setIsScreenContextSharingEnabled] = + useState(false); + const [isScreenContextPaused, setIsScreenContextPaused] = useState(false); + + const shouldPersist = Boolean(user) && !isGuestUser(user); + + useEffect(() => { + if (!user) { + setIsScreenContextSharingEnabled(false); + setIsScreenContextPaused(false); + return undefined; + } + + if (isGuestUser(user)) { + setIsScreenContextSharingEnabled(false); + setIsScreenContextPaused(false); + return undefined; + } + + try { + const sharingSnapshot = bucket.snapshot(SHARING_ENABLED_KEY); + const pausedSnapshot = bucket.snapshot(PAUSED_KEY); + setIsScreenContextSharingEnabled(sharingSnapshot.value ?? false); + setIsScreenContextPaused(pausedSnapshot.value ?? false); + } catch (error) { + // eslint-disable-next-line no-console + console.error( + 'Error reading screen context settings from storage:', + error, + ); + } + + const sharingSubscription = bucket + .observe$(SHARING_ENABLED_KEY) + .subscribe({ + next: snapshot => { + setIsScreenContextSharingEnabled(snapshot.value ?? false); + }, + error: error => { + // eslint-disable-next-line no-console + console.error('Error observing screenContextSharingEnabled:', error); + }, + }); + + const pausedSubscription = bucket.observe$(PAUSED_KEY).subscribe({ + next: snapshot => { + setIsScreenContextPaused(snapshot.value ?? false); + }, + error: error => { + // eslint-disable-next-line no-console + console.error('Error observing screenContextPaused:', error); + }, + }); + + return () => { + sharingSubscription.unsubscribe(); + pausedSubscription.unsubscribe(); + }; + }, [bucket, user]); + + const persistSharing = useCallback( + (enabled: boolean) => { + if (!shouldPersist) return; + try { + bucket.set(SHARING_ENABLED_KEY, enabled); + } catch (error) { + // eslint-disable-next-line no-console + console.error('Error saving screen context sharing state:', error); + } + }, + [bucket, shouldPersist], + ); + + const persistPaused = useCallback( + (paused: boolean) => { + if (!shouldPersist) return; + try { + bucket.set(PAUSED_KEY, paused); + } catch (error) { + // eslint-disable-next-line no-console + console.error('Error saving screen context paused state:', error); + } + }, + [bucket, shouldPersist], + ); + + const handleScreenContextSharingToggle = useCallback( + (enabled: boolean) => { + if (!user) return; + + setIsScreenContextSharingEnabled(enabled); + setIsScreenContextPaused(false); + persistPaused(false); + persistSharing(enabled); + }, + [user, persistSharing, persistPaused], + ); + + const setScreenContextPaused = useCallback( + (paused: boolean) => { + if (!user) return; + setIsScreenContextPaused(paused); + persistPaused(paused); + }, + [user, persistPaused], + ); + + const toggleScreenContextPaused = useCallback(() => { + setScreenContextPaused(!isScreenContextPaused); + }, [isScreenContextPaused, setScreenContextPaused]); + + return { + isScreenContextSharingEnabled, + isScreenContextPaused, + handleScreenContextSharingToggle, + toggleScreenContextPaused, + setScreenContextPaused, + }; +}; diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useWindowLocation.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useWindowLocation.ts new file mode 100644 index 00000000000..526e14fd56c --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/hooks/useWindowLocation.ts @@ -0,0 +1,62 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { useSyncExternalStore } from 'react'; + +import { + patchBrowserHistoryForScreenContext, + subscribeToScreenContextLabel, +} from './screenContextLabelSubscription'; + +export type WindowLocationSnapshot = { + pathname: string; + search: string; +}; + +patchBrowserHistoryForScreenContext(); + +let cachedSnapshot: WindowLocationSnapshot = { + pathname: '', + search: '', +}; + +function getWindowLocationSnapshot(): WindowLocationSnapshot { + const pathname = window.location.pathname; + const search = window.location.search; + if ( + cachedSnapshot.pathname !== pathname || + cachedSnapshot.search !== search + ) { + cachedSnapshot = { pathname, search }; + } + return cachedSnapshot; +} + +function subscribeToWindowLocation(onStoreChange: () => void): () => void { + return subscribeToScreenContextLabel(onStoreChange); +} + +/** + * Subscribes to browser URL changes (including history.pushState/replaceState). + * Use when the host app navigates outside the same React Router tree as Lightspeed. + */ +export function useWindowLocation(): WindowLocationSnapshot { + return useSyncExternalStore( + subscribeToWindowLocation, + getWindowLocationSnapshot, + getWindowLocationSnapshot, + ); +} diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/index.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/index.tsx index 1592dbc2e13..5d741f360d3 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/index.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/index.tsx @@ -43,6 +43,9 @@ import { notebooksApiRef } from './api/notebooksApi'; import { NotebooksApiClient } from './api/NotebooksApiClient'; import { LightspeedChatContainer as LightspeedChatContainerElement } from './components/LightspeedChatContainer'; import { LightspeedDrawerProvider as LightspeedProvider } from './components/LightspeedDrawerProvider'; + +import './hooks/screenContextLabelSubscription'; + import { LightspeedFABContent as LightspeedFABComponent } from './components/LightspeedFABContent'; import { LIGHTSPEED_APP_DRAWER_ID, diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/de.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/de.ts index de6bce89053..1ce438c4fa5 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/de.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/de.ts @@ -26,6 +26,11 @@ const intelligentAssistantTranslationDe = createTranslationMessages({ ref: intelligentAssistantTranslationRef, messages: { 'aria.chatHistoryMenu': 'Chatverlauf-Menü', + 'modelSelector.visionScreenshot.line1': 'Bildanalyse wird unterstützt.', + 'modelSelector.visionScreenshot.line2': + 'Ein Screenshot wird Ihrer Nachricht beigefügt.', + 'modelSelector.visionScreenshot.ariaLabel': + 'Screenshot-Kontext für Vision-Modell', 'modelSelector.disabledTooltip': 'Jede Chatsitzung unterstützt nur ein Modell. Um das Modell zu wechseln, starten Sie einen neuen Chat.', 'aria.chatbotSelector': 'Chatbot-Auswahl', @@ -354,6 +359,39 @@ const intelligentAssistantTranslationDe = createTranslationMessages({ 'settings.savedPrompts.enable': 'Gespeicherte Prompts aktivieren', 'settings.savedPrompts.enabled.description': 'Gespeicherte Prompts sind derzeit aktiviert', + 'settings.screenContext.enable': 'Bildschirmkontext aktivieren', + 'settings.screenContext.disable': 'Bildschirmkontext deaktivieren', + 'settings.screenContext.enabled.description': + 'Die Freigabe des Bildschirmkontexts ist derzeit aktiviert', + 'settings.screenContext.disabled.description': + 'Die Freigabe des Bildschirmkontexts ist derzeit deaktiviert', + 'contextChip.label.paused': 'Kontext: pausiert', + 'contextChip.label.unavailable': 'Kontext: nicht verfügbar', + 'contextChip.label.softwareTemplates': 'Software-Templates', + 'contextChip.tooltip.askAbout': 'Fragen Sie zu {{label}}.', + 'contextChip.tooltip.template': + 'Fragen Sie, wie Sie das Template {{label}} ausfüllen.', + 'contextChip.tooltip.search': 'Fragen Sie zu Ihrer Suche: {{label}}.', + 'contextChip.tooltip.paused': + 'Der Bildschirmkontext ist pausiert. Klicken Sie, um die Freigabe Ihres aktuellen Bildschirms an den intelligenten Assistenten fortzusetzen.', + 'contextChip.tooltip.unavailable': + 'Der Bildschirmkontext ist im Vollbildmodus nicht verfügbar. Wechseln Sie zum Overlay- oder Andockmodus, um ihn zu aktivieren.', + 'contextChip.tooltip.line2.fullContext': + 'Seitentext und ein Screenshot werden mit Ihrer Nachricht gesendet.', + 'contextChip.tooltip.line2.adminLimited': + 'Die Freigabe des Bildschirmkontexts ist durch Administratoreinstellungen eingeschränkt.', + 'contextChip.tooltip.line2.screenshotOnly': + 'Die Textextraktion ist von Ihrem Administrator deaktiviert. Nur Screenshot.', + 'contextChip.tooltip.line2.domOffNoVision': + 'Die Textextraktion ist von Ihrem Administrator deaktiviert. Ihr Modell unterstützt keine Bildanalyse.', + 'contextChip.tooltip.line2.textOnlyNoVision': + 'Nur Textkontext – Ihr Modell unterstützt keine Bildanalyse.', + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': + 'Nur Textkontext – die Screenshot-Erfassung ist von Ihrem Administrator deaktiviert.', + 'contextChip.tooltip.line2.textOnlyCombined': + 'Nur Textkontext – Ihr Modell unterstützt keine Bildanalyse und die Screenshot-Erfassung ist von Ihrem Administrator deaktiviert.', + 'contextChip.aria.pause': 'Bildschirmkontext pausieren: {{label}}', + 'contextChip.aria.resume': 'Bildschirmkontext fortsetzen', 'savedPrompts.tab.title': 'Gespeicherte Prompts', 'savedPrompts.disabled.title': 'Gespeicherte Prompts sind deaktiviert', 'savedPrompts.disabled.body': diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/es.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/es.ts index 6da75520cdf..e792598d368 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/es.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/es.ts @@ -26,6 +26,11 @@ const intelligentAssistantTranslationEs = createTranslationMessages({ ref: intelligentAssistantTranslationRef, messages: { 'aria.chatHistoryMenu': 'Menú del historial de chat', + 'modelSelector.visionScreenshot.line1': 'Análisis de imágenes compatible.', + 'modelSelector.visionScreenshot.line2': + 'Se incluirá una captura de pantalla con su mensaje.', + 'modelSelector.visionScreenshot.ariaLabel': + 'Contexto de captura de pantalla del modelo de visión', 'modelSelector.disabledTooltip': 'Cada sesión de chat solo admite un modelo. Para cambiar de modelo, abra un nuevo chat.', 'aria.chatbotSelector': 'Selector de chatbot', @@ -348,6 +353,39 @@ const intelligentAssistantTranslationEs = createTranslationMessages({ 'settings.savedPrompts.enable': 'Habilitar prompts guardados', 'settings.savedPrompts.enabled.description': 'Los prompts guardados están habilitados actualmente', + 'settings.screenContext.enable': 'Habilitar contexto de pantalla', + 'settings.screenContext.disable': 'Deshabilitar contexto de pantalla', + 'settings.screenContext.enabled.description': + 'El uso compartido del contexto de pantalla está habilitado actualmente', + 'settings.screenContext.disabled.description': + 'El uso compartido del contexto de pantalla está deshabilitado actualmente', + 'contextChip.label.paused': 'Contexto: en pausa', + 'contextChip.label.unavailable': 'Contexto: no disponible', + 'contextChip.label.softwareTemplates': 'Plantillas de software', + 'contextChip.tooltip.askAbout': 'Pregunte sobre {{label}}.', + 'contextChip.tooltip.template': + 'Pregunte cómo completar la plantilla {{label}}.', + 'contextChip.tooltip.search': 'Pregunte sobre su búsqueda: {{label}}.', + 'contextChip.tooltip.paused': + 'El contexto de pantalla está en pausa. Haga clic para reanudar el uso compartido de su pantalla actual con el asistente inteligente.', + 'contextChip.tooltip.unavailable': + 'El contexto de pantalla no está disponible en modo de pantalla completa. Cambie al modo Superposición o Anclar a la ventana para habilitarlo.', + 'contextChip.tooltip.line2.fullContext': + 'El texto de la página y una captura de pantalla se enviarán con su mensaje.', + 'contextChip.tooltip.line2.adminLimited': + 'El uso compartido del contexto de pantalla está limitado por la configuración del administrador.', + 'contextChip.tooltip.line2.screenshotOnly': + 'La extracción de texto está deshabilitada por su administrador. Solo captura de pantalla.', + 'contextChip.tooltip.line2.domOffNoVision': + 'La extracción de texto está deshabilitada por su administrador. Su modelo no admite el análisis de imágenes.', + 'contextChip.tooltip.line2.textOnlyNoVision': + 'Solo contexto de texto: su modelo no admite el análisis de imágenes.', + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': + 'Solo contexto de texto: la captura de pantalla está deshabilitada por su administrador.', + 'contextChip.tooltip.line2.textOnlyCombined': + 'Solo contexto de texto: su modelo no admite el análisis de imágenes y la captura de pantalla está deshabilitada por su administrador.', + 'contextChip.aria.pause': 'Pausar contexto de pantalla: {{label}}', + 'contextChip.aria.resume': 'Reanudar contexto de pantalla', 'savedPrompts.tab.title': 'Prompts guardados', 'savedPrompts.disabled.title': 'Los prompts guardados están deshabilitados', 'savedPrompts.disabled.body': diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/fr.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/fr.ts index 7c0c085db75..5bfc199bdff 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/fr.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/fr.ts @@ -26,6 +26,11 @@ const intelligentAssistantTranslationFr = createTranslationMessages({ ref: intelligentAssistantTranslationRef, messages: { 'aria.chatHistoryMenu': 'Menu de l’historique de conversations', + 'modelSelector.visionScreenshot.line1': "Analyse d'images prise en charge.", + 'modelSelector.visionScreenshot.line2': + "Une capture d'écran sera jointe à votre message.", + 'modelSelector.visionScreenshot.ariaLabel': + "Contexte de capture d'écran du modèle vision", 'modelSelector.disabledTooltip': "Chaque session de chat ne prend en charge qu'un seul modèle. Pour changer de modèle, ouvrez un nouveau chat.", 'aria.chatbotSelector': 'Sélecteur Chatbot', @@ -354,6 +359,40 @@ const intelligentAssistantTranslationFr = createTranslationMessages({ 'settings.savedPrompts.enable': 'Activer les prompts sauvegardés', 'settings.savedPrompts.enabled.description': 'Les prompts sauvegardés sont actuellement activés', + 'settings.screenContext.enable': "Activer le contexte d'écran", + 'settings.screenContext.disable': "Désactiver le contexte d'écran", + 'settings.screenContext.enabled.description': + "Le partage du contexte d'écran est actuellement activé", + 'settings.screenContext.disabled.description': + "Le partage du contexte d'écran est actuellement désactivé", + 'contextChip.label.paused': 'Contexte : en pause', + 'contextChip.label.unavailable': 'Contexte : indisponible', + 'contextChip.label.softwareTemplates': 'Modèles logiciels', + 'contextChip.tooltip.askAbout': 'Posez une question sur {{label}}.', + 'contextChip.tooltip.template': + 'Demandez comment remplir le modèle {{label}}.', + 'contextChip.tooltip.search': + 'Posez une question sur votre recherche : {{label}}.', + 'contextChip.tooltip.paused': + "Le contexte d'écran est en pause. Cliquez pour reprendre le partage de votre écran actuel avec l'assistant intelligent.", + 'contextChip.tooltip.unavailable': + "Le contexte d'écran n'est pas disponible en mode plein écran. Passez en mode Superposition ou Ancrer à la fenêtre pour l'activer.", + 'contextChip.tooltip.line2.fullContext': + "Le texte de la page et une capture d'écran seront envoyés avec votre message.", + 'contextChip.tooltip.line2.adminLimited': + "Le partage du contexte d'écran est limité par les paramètres de l'administrateur.", + 'contextChip.tooltip.line2.screenshotOnly': + "L'extraction de texte est désactivée par votre administrateur. Capture d'écran uniquement.", + 'contextChip.tooltip.line2.domOffNoVision': + "L'extraction de texte est désactivée par votre administrateur. Votre modèle ne prend pas en charge l'analyse d'images.", + 'contextChip.tooltip.line2.textOnlyNoVision': + "Contexte texte uniquement – votre modèle ne prend pas en charge l'analyse d'images.", + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': + "Contexte texte uniquement – la capture d'écran est désactivée par votre administrateur.", + 'contextChip.tooltip.line2.textOnlyCombined': + "Contexte texte uniquement – votre modèle ne prend pas en charge l'analyse d'images et la capture d'écran est désactivée par votre administrateur.", + 'contextChip.aria.pause': "Mettre en pause le contexte d'écran : {{label}}", + 'contextChip.aria.resume': "Reprendre le contexte d'écran", 'savedPrompts.tab.title': 'Prompts enregistrés', 'savedPrompts.disabled.title': 'Les prompts enregistrés sont désactivés', 'savedPrompts.disabled.body': diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/it.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/it.ts index 9d45f8c6023..e7f67700e7d 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/it.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/it.ts @@ -26,6 +26,12 @@ const intelligentAssistantTranslationIt = createTranslationMessages({ ref: intelligentAssistantTranslationRef, messages: { 'aria.chatHistoryMenu': 'Menu cronologia chat', + 'modelSelector.visionScreenshot.line1': + 'Analisi delle immagini supportata.', + 'modelSelector.visionScreenshot.line2': + 'Uno screenshot verrà incluso nel messaggio.', + 'modelSelector.visionScreenshot.ariaLabel': + 'Contesto screenshot del modello vision', 'modelSelector.disabledTooltip': 'Ogni sessione di chat supporta un solo modello. Per cambiare modello, apri una nuova chat.', 'aria.chatbotSelector': 'Selettore di chatbot', @@ -353,6 +359,41 @@ const intelligentAssistantTranslationIt = createTranslationMessages({ 'settings.savedPrompts.enable': 'Abilita prompt salvati', 'settings.savedPrompts.enabled.description': 'I prompt salvati sono attualmente abilitati', + 'settings.screenContext.enable': 'Abilita contesto dello schermo', + 'settings.screenContext.disable': 'Disabilita contesto dello schermo', + 'settings.screenContext.enabled.description': + 'La condivisione del contesto dello schermo è attualmente abilitata', + 'settings.screenContext.disabled.description': + 'La condivisione del contesto dello schermo è attualmente disabilitata', + 'contextChip.label.paused': 'Contesto: in pausa', + 'contextChip.label.unavailable': 'Contesto: non disponibile', + 'contextChip.label.softwareTemplates': 'Template software', + 'contextChip.tooltip.askAbout': 'Chiedi informazioni su {{label}}.', + 'contextChip.tooltip.template': + 'Chiedi come compilare il template {{label}}.', + 'contextChip.tooltip.search': + 'Chiedi informazioni sulla ricerca: {{label}}.', + 'contextChip.tooltip.paused': + "Il contesto dello schermo è in pausa. Fai clic per riprendere la condivisione dello schermo corrente con l'assistente intelligente.", + 'contextChip.tooltip.unavailable': + 'Il contesto dello schermo non è disponibile in modalità schermo intero. Passa alla modalità Sovrapposizione o Aggancia alla finestra per abilitarlo.', + 'contextChip.tooltip.line2.fullContext': + 'Il testo della pagina e uno screenshot verranno inviati con il messaggio.', + 'contextChip.tooltip.line2.adminLimited': + "La condivisione del contesto dello schermo è limitata dalle impostazioni dell'amministratore.", + 'contextChip.tooltip.line2.screenshotOnly': + "L'estrazione del testo è disabilitata dall'amministratore. Solo screenshot.", + 'contextChip.tooltip.line2.domOffNoVision': + "L'estrazione del testo è disabilitata dall'amministratore. Il modello non supporta l'analisi delle immagini.", + 'contextChip.tooltip.line2.textOnlyNoVision': + "Solo contesto testuale: il modello non supporta l'analisi delle immagini.", + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': + "Solo contesto testuale: l'acquisizione di screenshot è disabilitata dall'amministratore.", + 'contextChip.tooltip.line2.textOnlyCombined': + "Solo contesto testuale: il modello non supporta l'analisi delle immagini e l'acquisizione di screenshot è disabilitata dall'amministratore.", + 'contextChip.aria.pause': + 'Metti in pausa il contesto dello schermo: {{label}}', + 'contextChip.aria.resume': 'Riprendi il contesto dello schermo', 'savedPrompts.tab.title': 'Prompt salvati', 'savedPrompts.disabled.title': 'I prompt salvati sono disabilitati', 'savedPrompts.disabled.body': diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ja.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ja.ts index f415167a481..24b27f464a7 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ja.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ja.ts @@ -26,6 +26,11 @@ const intelligentAssistantTranslationJa = createTranslationMessages({ ref: intelligentAssistantTranslationRef, messages: { 'aria.chatHistoryMenu': 'チャット履歴メニュー', + 'modelSelector.visionScreenshot.line1': '画像分析に対応しています。', + 'modelSelector.visionScreenshot.line2': + 'メッセージにスクリーンショットが含まれます。', + 'modelSelector.visionScreenshot.ariaLabel': + 'ビジョンモデルのスクリーンショットコンテキスト', 'modelSelector.disabledTooltip': '各チャットセッションは1つのモデルのみをサポートしています。モデルを切り替えるには、新しいチャットを開いてください。', 'aria.chatbotSelector': 'チャットボットセレクター', @@ -342,6 +347,39 @@ const intelligentAssistantTranslationJa = createTranslationMessages({ 'settings.savedPrompts.enable': '保存済みプロンプトを有効にする', 'settings.savedPrompts.enabled.description': '保存済みプロンプトは現在有効です', + 'settings.screenContext.enable': '画面コンテキストを有効にする', + 'settings.screenContext.disable': '画面コンテキストを無効にする', + 'settings.screenContext.enabled.description': + '画面コンテキストの共有は現在有効です', + 'settings.screenContext.disabled.description': + '画面コンテキストの共有は現在無効です', + 'contextChip.label.paused': 'コンテキスト: 一時停止中', + 'contextChip.label.unavailable': 'コンテキスト: 利用不可', + 'contextChip.label.softwareTemplates': 'ソフトウェアテンプレート', + 'contextChip.tooltip.askAbout': '{{label}}について質問する。', + 'contextChip.tooltip.template': + '{{label}}テンプレートの記入方法を質問する。', + 'contextChip.tooltip.search': '検索内容について質問する: {{label}}。', + 'contextChip.tooltip.paused': + '画面コンテキストは一時停止中です。クリックすると、現在の画面の共有をインテリジェントアシスタントに再開します。', + 'contextChip.tooltip.unavailable': + 'フルスクリーンモードでは画面コンテキストを利用できません。有効にするには、オーバーレイまたはウィンドウにドッキングモードに切り替えてください。', + 'contextChip.tooltip.line2.fullContext': + 'ページのテキストとスクリーンショットがメッセージと一緒に送信されます。', + 'contextChip.tooltip.line2.adminLimited': + '画面コンテキストの共有は管理者の設定により制限されています。', + 'contextChip.tooltip.line2.screenshotOnly': + 'テキスト抽出は管理者によって無効になっています。スクリーンショットのみ。', + 'contextChip.tooltip.line2.domOffNoVision': + 'テキスト抽出は管理者によって無効になっています。ご利用のモデルは画像分析に対応していません。', + 'contextChip.tooltip.line2.textOnlyNoVision': + 'テキストコンテキストのみ - ご利用のモデルは画像分析に対応していません。', + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': + 'テキストコンテキストのみ - スクリーンショットの取得は管理者によって無効になっています。', + 'contextChip.tooltip.line2.textOnlyCombined': + 'テキストコンテキストのみ - ご利用のモデルは画像分析に対応しておらず、スクリーンショットの取得も管理者によって無効になっています。', + 'contextChip.aria.pause': '画面コンテキストを一時停止: {{label}}', + 'contextChip.aria.resume': '画面コンテキストを再開', 'savedPrompts.tab.title': '保存済みプロンプト', 'savedPrompts.disabled.title': '保存済みプロンプトは無効です', 'savedPrompts.disabled.body': diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ref.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ref.ts index 2fbbf1def97..576437c500b 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ref.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/translations/ref.ts @@ -226,6 +226,10 @@ export const intelligentAssistantMessages = { // Model selector 'modelSelector.disabledTooltip': 'Each chat session supports only one model. To switch models, open a new chat.', + 'modelSelector.visionScreenshot.line1': 'Image analysis supported.', + 'modelSelector.visionScreenshot.line2': + 'A screenshot will be included with your message.', + 'modelSelector.visionScreenshot.ariaLabel': 'Vision model screenshot context', // Accessibility and ARIA labels 'aria.chatbotSelector': 'Chatbot selector', @@ -340,6 +344,39 @@ export const intelligentAssistantMessages = { 'Saved prompts are currently enabled', 'settings.savedPrompts.disabled.description': 'Saved prompts are currently disabled', + 'settings.screenContext.enable': 'Enable screen context', + 'settings.screenContext.disable': 'Disable screen context', + 'settings.screenContext.enabled.description': + 'Screen context sharing is currently enabled', + 'settings.screenContext.disabled.description': + 'Screen context sharing is currently disabled', + + 'contextChip.label.paused': 'Context: paused', + 'contextChip.label.unavailable': 'Context: unavailable', + 'contextChip.label.softwareTemplates': 'Software templates', + 'contextChip.tooltip.askAbout': 'Ask about {{label}}.', + 'contextChip.tooltip.template': 'Ask how to fill in the {{label}} template.', + 'contextChip.tooltip.search': 'Ask about your search: {{label}}.', + 'contextChip.tooltip.paused': + 'Screen context is paused. Click to resume sharing your current screen with the intelligent assistant.', + 'contextChip.tooltip.unavailable': + 'Screen context is not available in fullscreen mode. Switch to Overlay or Docked mode to enable it.', + 'contextChip.tooltip.line2.fullContext': + 'Page text and a screenshot will be sent with your message.', + 'contextChip.tooltip.line2.adminLimited': + 'Screen context sharing is limited by administrator settings.', + 'contextChip.tooltip.line2.screenshotOnly': + 'Text extraction is disabled by your administrator. Screenshot only.', + 'contextChip.tooltip.line2.domOffNoVision': + 'Text extraction is disabled by your administrator. Your model does not support image analysis.', + 'contextChip.tooltip.line2.textOnlyNoVision': + 'Text context only - your model does not support image analysis.', + 'contextChip.tooltip.line2.textOnlyAdminScreenshotsOff': + 'Text context only - screenshot capture is disabled by your administrator.', + 'contextChip.tooltip.line2.textOnlyCombined': + 'Text context only - your model does not support image analysis and screenshot capture is disabled by your administrator.', + 'contextChip.aria.pause': 'Pause screen context: {{label}}', + 'contextChip.aria.resume': 'Resume screen context', 'settings.panel.title': 'Settings', 'settings.mcp.label': 'MCP and Prompt Settings', 'settings.prompt.label': 'Prompt Settings', diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-context-utils.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-context-utils.test.ts new file mode 100644 index 00000000000..c957aca9fe6 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-context-utils.test.ts @@ -0,0 +1,658 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { extractPageContext } from '../dom-extractor'; +import { captureScreenshot } from '../screen-capture'; +import { + buildScreenContextAttachments, + getPageTitleFromDom, + getScreenContextChipLabel, + getScreenContextRouteKind, + getScreenContextTooltipLine1Text, + getScreenContextTooltipLine2Key, + getSearchQueryFromLocation, + isElementVisible, + isSoftwareTemplatesListPath, + normalizeDocumentTitle, + parseSoftwareTemplateDetailSlug, + resolveScreenContextChipLabel, + shouldAttachScreenContext, + truncateChipLabel, +} from '../screen-context-utils'; + +jest.mock('../dom-extractor', () => ({ + extractPageContext: jest.fn(), +})); + +jest.mock('../screen-capture', () => ({ + captureScreenshot: jest.fn(), +})); + +const mockExtractPageContext = extractPageContext as jest.MockedFunction< + typeof extractPageContext +>; +const mockCaptureScreenshot = captureScreenshot as jest.MockedFunction< + typeof captureScreenshot +>; + +describe('screen-context-utils', () => { + describe('truncateChipLabel', () => { + it('truncates long labels with ellipsis', () => { + const long = 'A'.repeat(50); + expect(truncateChipLabel(long, 40)).toHaveLength(40); + expect(truncateChipLabel(long, 40).endsWith('…')).toBe(true); + }); + }); + + describe('normalizeDocumentTitle', () => { + it('strips app suffix after pipe', () => { + expect(normalizeDocumentTitle('My Service | Red Hat Developer Hub')).toBe( + 'My Service', + ); + }); + }); + + describe('getScreenContextRouteKind', () => { + it('detects template routes', () => { + expect(getScreenContextRouteKind('/catalog/create/template', '')).toBe( + 'template', + ); + expect( + getScreenContextRouteKind('/catalog/default/template/foo', ''), + ).toBe('template'); + }); + + it('detects search routes', () => { + expect(getScreenContextRouteKind('/search', '?query=foo')).toBe('search'); + expect( + getScreenContextRouteKind('/catalog', '?filters[kind]=component'), + ).toBe('search'); + }); + + it('does not treat substring param names as search keys', () => { + expect( + getScreenContextRouteKind('/catalog/default/component/foo', '?freq=5'), + ).toBe('default'); + expect( + getScreenContextRouteKind( + '/catalog/default/component/foo', + '?longterm=1', + ), + ).toBe('default'); + }); + }); + + describe('getPageTitleFromDom', () => { + beforeEach(() => { + document.body.innerHTML = ''; + document.title = ''; + }); + + it('prefers visible BUI title inside main over chatbot title', () => { + document.body.innerHTML = ` +
+
+

Prototype — Sample documentation

+
+
+
+

Lightspeed

+
+ `; + expect(getPageTitleFromDom()).toBe('Prototype — Sample documentation'); + }); + + it('falls back to legacy header title', () => { + document.body.innerHTML = ` +
+

Search

+
+ `; + expect(getPageTitleFromDom()).toBe('Search'); + }); + + it('uses dialog title when modal is open', () => { + document.body.innerHTML = ` +

Background Page

+
+

Confirm delete

+
+ `; + expect(getPageTitleFromDom()).toBe('Confirm delete'); + }); + }); + + describe('getSearchQueryFromLocation', () => { + beforeEach(() => { + document.body.innerHTML = ''; + }); + + it('reads query from URL params', () => { + expect(getSearchQueryFromLocation('?query=my-service')).toBe( + 'my-service', + ); + expect(getSearchQueryFromLocation('?term=backend')).toBe('backend'); + }); + + it('reads query from visible search input', () => { + document.body.innerHTML = ` + + `; + expect(getSearchQueryFromLocation('')).toBe('catalog entity'); + }); + }); + + describe('resolveScreenContextChipLabel', () => { + beforeEach(() => { + document.body.innerHTML = ''; + document.title = ''; + }); + + it('uses search query as chip label on search routes', () => { + const result = resolveScreenContextChipLabel({ + pathname: '/search', + search: '?query=payments-api', + }); + expect(result.routeKind).toBe('search'); + expect(result.label).toBe('payments-api'); + expect(result.searchQuery).toBe('payments-api'); + }); + + it('uses scoped page title for default routes', () => { + document.body.innerHTML = ` +
+
+

backstage

+
+
+ `; + const result = resolveScreenContextChipLabel({ + pathname: '/catalog/default/component/backstage', + search: '', + }); + expect(result.routeKind).toBe('default'); + expect(result.label).toBe('backstage'); + }); + + it('uses plugin toolbar name when settings breadcrumbs are present', () => { + document.body.innerHTML = ` +
+
+
+
+
+

Settings

+
+ +
+
+
+
+ `; + const result = resolveScreenContextChipLabel({ + pathname: '/settings/general', + search: '', + }); + expect(result.label).toBe('Settings'); + }); + + it('uses Software templates on template gallery with filters', () => { + const result = resolveScreenContextChipLabel({ + pathname: '/create/templates', + search: '?filters%5Bkind%5D=template&filters%5Buser%5D=all', + }); + expect(result.routeKind).toBe('template'); + expect(result.label).toBe('Software templates'); + }); + + it('uses URL slug on software template detail when card title is absent', () => { + const result = resolveScreenContextChipLabel({ + pathname: '/create/templates/default/argocd-template', + search: '', + }); + expect(result.routeKind).toBe('template'); + expect(result.label).toBe('argocd-template'); + }); + + it('prefers info card title on software template detail page', () => { + document.body.innerHTML = ` +
+
+
+

+ Ansible Job Template +

+
+
+
+ `; + const result = resolveScreenContextChipLabel({ + pathname: '/create/templates/default/argocd-template', + search: '', + }); + expect(result.label).toBe('Ansible Job Template'); + }); + + it('uses template header title on template routes', () => { + document.body.innerHTML = ` +
+
+

Node.js template

+
+
+ `; + const result = resolveScreenContextChipLabel({ + pathname: '/create/tasks/default/template/nodejs', + search: '', + }); + expect(result.routeKind).toBe('template'); + expect(result.label).toBe('Node.js template'); + }); + }); + + describe('getScreenContextTooltipLine1Text', () => { + it('uses search tooltip copy on search routes', () => { + const text = getScreenContextTooltipLine1Text( + { chipLabel: 'api', routeKind: 'search' }, + (key, options) => `${key}:${options?.label}`, + ); + expect(text).toBe('contextChip.tooltip.search:api'); + }); + + it('uses template tooltip on software template detail', () => { + const text = getScreenContextTooltipLine1Text( + { + chipLabel: 'argocd-template', + routeKind: 'template', + pathname: '/create/templates/default/argocd-template', + }, + (key, options) => `${key}:${options?.label}`, + ); + expect(text).toBe('contextChip.tooltip.template:argocd-template'); + }); + + it('uses askAbout tooltip on software template gallery', () => { + const text = getScreenContextTooltipLine1Text( + { + chipLabel: 'Software templates', + routeKind: 'template', + pathname: '/create/templates', + }, + (key, options) => `${key}:${options?.label}`, + ); + expect(text).toBe('contextChip.tooltip.askAbout:Software templates'); + }); + }); + + describe('getScreenContextTooltipLine2Key', () => { + it('returns fullContext when dom and screenshot will be sent', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toBe('fullContext'); + }); + + it('returns textOnlyNoVision when model lacks vision', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: true, + screenshotsEnabled: true, + supportsVision: false, + }), + ).toBe('textOnlyNoVision'); + }); + + it('returns screenshotOnly when dom disabled but vision on', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: false, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toBe('screenshotOnly'); + }); + + it('returns adminLimited when both dom and screenshots are disabled', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: false, + screenshotsEnabled: false, + supportsVision: true, + }), + ).toBe('adminLimited'); + }); + + it('returns textOnlyAdminScreenshotsOff when screenshots disabled by admin', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: true, + screenshotsEnabled: false, + supportsVision: true, + }), + ).toBe('textOnlyAdminScreenshotsOff'); + }); + + it('returns textOnlyCombined when screenshots off and model lacks vision', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: true, + screenshotsEnabled: false, + supportsVision: false, + }), + ).toBe('textOnlyCombined'); + }); + + it('returns domOffNoVision when dom off, screenshots on, but model lacks vision', () => { + expect( + getScreenContextTooltipLine2Key({ + domEnabled: false, + screenshotsEnabled: true, + supportsVision: false, + }), + ).toBe('domOffNoVision'); + }); + }); + + describe('shouldAttachScreenContext', () => { + it('attaches nothing when paused', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: true, + paused: true, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toEqual({ attachDom: false, attachScreenshot: false }); + }); + + it('attaches nothing when admin disabled', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: false, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toEqual({ attachDom: false, attachScreenshot: false }); + }); + + it('attaches nothing when sharing disabled', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: false, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toEqual({ attachDom: false, attachScreenshot: false }); + }); + + it('attaches nothing when fullscreen', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: true, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toEqual({ attachDom: false, attachScreenshot: false }); + }); + + it('attaches dom and screenshot when all gates pass', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toEqual({ attachDom: true, attachScreenshot: true }); + }); + + it('attaches screenshot without dom when dom disabled', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: false, + screenshotsEnabled: true, + supportsVision: true, + }), + ).toEqual({ attachDom: false, attachScreenshot: true }); + }); + + it('attaches dom only when model lacks vision', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: false, + }), + ).toEqual({ attachDom: true, attachScreenshot: false }); + }); + + it('attaches dom only when screenshots disabled by admin', () => { + expect( + shouldAttachScreenContext({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: false, + supportsVision: true, + }), + ).toEqual({ attachDom: true, attachScreenshot: false }); + }); + }); + + describe('isElementVisible', () => { + it('returns false for non-HTMLElement nodes', () => { + const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + expect(isElementVisible(svg)).toBe(false); + }); + + it('returns false for elements inside screen-capture exclude', () => { + const wrap = document.createElement('div'); + wrap.setAttribute('data-screen-capture-exclude', 'true'); + const child = document.createElement('span'); + child.textContent = 'secret'; + wrap.appendChild(child); + document.body.appendChild(wrap); + expect(isElementVisible(child)).toBe(false); + wrap.remove(); + }); + }); + + describe('software template path helpers', () => { + it('detects gallery vs detail paths', () => { + expect(isSoftwareTemplatesListPath('/create/templates')).toBe(true); + expect(isSoftwareTemplatesListPath('/create/templates/')).toBe(true); + expect(isSoftwareTemplatesListPath('/create/templates/default/foo')).toBe( + false, + ); + expect(parseSoftwareTemplateDetailSlug('/create/templates')).toBe( + undefined, + ); + expect( + parseSoftwareTemplateDetailSlug( + '/create/templates/default/argocd-template', + ), + ).toBe('argocd-template'); + }); + }); + + describe('getScreenContextChipLabel', () => { + it('reads the current window location', () => { + window.history.pushState({}, '', '/search?query=widgets'); + document.body.innerHTML = ''; + expect(getScreenContextChipLabel()).toBe('widgets'); + }); + }); + + describe('buildScreenContextAttachments', () => { + beforeEach(() => { + mockExtractPageContext.mockReset(); + mockCaptureScreenshot.mockReset(); + }); + + it('returns no attachments when gates fail', async () => { + await expect( + buildScreenContextAttachments({ + adminEnabled: true, + sharingEnabled: true, + paused: true, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + domExtractionMaxChars: 1000, + }), + ).resolves.toEqual([]); + expect(mockExtractPageContext).not.toHaveBeenCalled(); + expect(mockCaptureScreenshot).not.toHaveBeenCalled(); + }); + + it('attaches DOM and screenshot when both gates pass', async () => { + mockExtractPageContext.mockReturnValue('Page: /catalog'); + mockCaptureScreenshot.mockResolvedValue({ + success: true, + contentType: 'image/jpeg', + base64: 'abc123', + width: 1280, + height: 720, + captureTimeMs: 10, + }); + + await expect( + buildScreenContextAttachments({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + domExtractionMaxChars: 2000, + }), + ).resolves.toEqual([ + { + attachment_type: 'configuration', + content_type: 'text/plain', + content: 'Page: /catalog', + }, + { + attachment_type: 'image', + content_type: 'image/jpeg', + content: 'abc123', + }, + ]); + }); + + it('degrades to DOM only when screenshot capture fails', async () => { + mockExtractPageContext.mockReturnValue('Page: /catalog'); + mockCaptureScreenshot.mockResolvedValue({ + success: false, + error: 'capture failed', + }); + + await expect( + buildScreenContextAttachments({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + domExtractionMaxChars: 2000, + }), + ).resolves.toEqual([ + { + attachment_type: 'configuration', + content_type: 'text/plain', + content: 'Page: /catalog', + }, + ]); + }); + + it('skips DOM attachment when extractPageContext throws', async () => { + mockExtractPageContext.mockImplementation(() => { + throw new Error('dom boom'); + }); + mockCaptureScreenshot.mockResolvedValue({ + success: true, + contentType: 'image/jpeg', + base64: 'img', + width: 800, + height: 600, + captureTimeMs: 5, + }); + + await expect( + buildScreenContextAttachments({ + adminEnabled: true, + sharingEnabled: true, + paused: false, + isFullscreen: false, + domEnabled: true, + screenshotsEnabled: true, + supportsVision: true, + domExtractionMaxChars: 2000, + }), + ).resolves.toEqual([ + { + attachment_type: 'image', + content_type: 'image/jpeg', + content: 'img', + }, + ]); + }); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-selectors.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-selectors.ts index dfcac416f0e..9c3c9e3997d 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-selectors.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-selectors.ts @@ -48,10 +48,42 @@ export const NOISE_SELECTOR = NOISE_SELECTORS.join(', '); // ── Header ── export const PLUGIN_NAME_SELECTOR = '.bui-PluginHeaderToolbarName'; +/** BUI NFS plugin header name area (hashed class names). */ +export const PLUGIN_HEADER_TOOLBAR_NAME_SELECTOR = + '.bui-PluginHeaderToolbarName, [class*="bui-PluginHeaderToolbarName"]'; +/** Breadcrumbs inside BUI plugin header toolbar (Settings → General, etc.). */ +export const PLUGIN_HEADER_BREADCRUMBS_SELECTOR = + '#Breadcrumbs, nav[aria-label="Breadcrumbs"], nav[aria-label="breadcrumb" i], [class*="PluginHeaderBreadcrumbs"]'; export const PAGE_TITLE_SELECTOR = '.bui-HeaderTitle'; + +/** BUI / NFS header title (class may be hashed). */ +export const PAGE_TITLE_SCOPED_SELECTORS = [ + PAGE_TITLE_SELECTOR, + '[class*="bui-HeaderTitle"]', +] as const; + +/** Software template entity page (scaffolder) info card title. */ +export const TEMPLATE_INFO_CARD_TITLE_SELECTORS = [ + '[class*="BackstageInfoCard-headerTitle"]', + '[class*="MuiCardHeader-title"]', +] as const; + +/** Legacy Backstage core-components Header title. */ +export const LEGACY_PAGE_TITLE_SELECTORS = [ + '[data-testid="header-title"]', + '[class*="Header-title"]', + 'header h1', +] as const; + +/** Roots to search for page chrome (exclude Lightspeed overlay). */ +export const PAGE_CHROME_ROOT_SELECTORS = [ + '#root main', + 'main[class*="BackstagePage-root"]', + '#root', +] as const; export const ACTIVE_TAB_SELECTOR = '[role="tab"][aria-selected="true"]'; export const BREADCRUMB_SELECTOR = - '[aria-label="breadcrumb"], [class*="MuiBreadcrumbs-root"], [class*="bui-HeaderBreadcrumb"]'; + '[aria-label="Breadcrumbs"], [aria-label="breadcrumb" i], [class*="MuiBreadcrumbs-root"], [class*="bui-HeaderBreadcrumb"], [class*="PluginHeaderBreadcrumbs"], #Breadcrumbs'; export const ACTIVE_NAV_SELECTOR = '[aria-current="page"]'; // ── Tabs ── diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-context-utils.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-context-utils.ts new file mode 100644 index 00000000000..e42b6dc6956 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-context-utils.ts @@ -0,0 +1,550 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Attachment } from '../types'; +import { extractPageContext } from './dom-extractor'; +import { + DIALOG_TITLE_SELECTOR, + LEGACY_PAGE_TITLE_SELECTORS, + MODAL_DIALOG_SELECTOR, + PAGE_CHROME_ROOT_SELECTORS, + PAGE_TITLE_SCOPED_SELECTORS, + PLUGIN_HEADER_BREADCRUMBS_SELECTOR, + PLUGIN_HEADER_TOOLBAR_NAME_SELECTOR, + SEARCH_INPUT_SELECTOR, + TEMPLATE_INFO_CARD_TITLE_SELECTORS, +} from './dom-selectors'; +import { captureScreenshot } from './screen-capture'; + +export const SCREEN_CONTEXT_CHIP_MAX_LABEL_LENGTH = 40; + +const SCREEN_CONTEXT_EXCLUDE_SELECTOR = + '[data-screen-capture-exclude], .pf-chatbot'; + +const SEARCH_QUERY_PARAM_KEYS = ['query', 'term', 'q'] as const; + +export type ScreenContextRouteKind = 'default' | 'template' | 'search'; + +export type ScreenContextTooltipLine2Key = + | 'fullContext' + | 'adminLimited' + | 'screenshotOnly' + | 'domOffNoVision' + | 'textOnlyNoVision' + | 'textOnlyAdminScreenshotsOff' + | 'textOnlyCombined'; + +export type ShouldAttachScreenContextInput = { + adminEnabled: boolean; + sharingEnabled: boolean; + paused: boolean; + isFullscreen: boolean; + domEnabled: boolean; + screenshotsEnabled: boolean; + supportsVision: boolean; +}; + +export type ShouldAttachScreenContextResult = { + attachDom: boolean; + attachScreenshot: boolean; +}; + +export type ResolvedScreenContextLabel = { + label: string; + routeKind: ScreenContextRouteKind; + searchQuery?: string; +}; + +export function truncateChipLabel( + label: string, + maxLength = SCREEN_CONTEXT_CHIP_MAX_LABEL_LENGTH, +): string { + const trimmed = label.trim(); + if (trimmed.length <= maxLength) { + return trimmed; + } + return `${trimmed.slice(0, maxLength - 1).trimEnd()}…`; +} + +export function normalizeDocumentTitle(title: string): string { + const trimmed = title.trim(); + const pipeIndex = trimmed.indexOf(' | '); + if (pipeIndex > 0) { + return trimmed.slice(0, pipeIndex).trim(); + } + return trimmed; +} + +export function isElementVisible(el: Element): boolean { + if (!(el instanceof HTMLElement)) { + return false; + } + if (el.closest(SCREEN_CONTEXT_EXCLUDE_SELECTOR)) { + return false; + } + const style = window.getComputedStyle(el); + if (el.hasAttribute('hidden')) { + return false; + } + if ( + style.display === 'none' || + style.visibility === 'hidden' || + Number.parseFloat(style.opacity) === 0 + ) { + return false; + } + return true; +} + +function queryFirstVisibleTitle( + root: ParentNode, + selectors: readonly string[], +): string | undefined { + for (const selector of selectors) { + const elements = root.querySelectorAll(selector); + for (const el of elements) { + if (!isElementVisible(el)) { + continue; + } + const text = el.textContent?.trim(); + if (text) { + return text; + } + } + } + return undefined; +} + +function getChromeRoots(doc: Document): ParentNode[] { + const roots: ParentNode[] = []; + for (const selector of PAGE_CHROME_ROOT_SELECTORS) { + const node = doc.querySelector(selector); + if (node && !roots.includes(node)) { + roots.push(node); + } + } + return roots.length > 0 ? roots : [doc]; +} + +/** + * BUI plugin pages (e.g. Settings → General) use toolbar breadcrumbs instead of + * HeaderTitle. Prefer the plugin name (visually hidden h1 or first crumb link). + */ +export function getPluginHeaderToolbarChipLabel( + root: ParentNode = document, +): string | undefined { + const toolbarName = root.querySelector(PLUGIN_HEADER_TOOLBAR_NAME_SELECTOR); + if (!(toolbarName instanceof Element)) { + return undefined; + } + if (toolbarName.closest(SCREEN_CONTEXT_EXCLUDE_SELECTOR)) { + return undefined; + } + + const breadcrumbs = toolbarName.querySelector( + PLUGIN_HEADER_BREADCRUMBS_SELECTOR, + ); + if (!breadcrumbs) { + return undefined; + } + + const breadcrumbItems = breadcrumbs.querySelectorAll( + 'li.react-aria-Breadcrumb, li[class*="Breadcrumb"]', + ); + const hasNestedCrumbs = breadcrumbItems.length >= 2; + + const h1Text = toolbarName.querySelector('h1')?.textContent?.trim(); + if (h1Text) { + return h1Text; + } + + if (hasNestedCrumbs) { + const firstLink = breadcrumbs.querySelector('a[href]'); + const linkText = firstLink?.textContent?.trim(); + if (linkText) { + return linkText; + } + } + + const currentCrumb = breadcrumbs.querySelector( + '[data-current="true"], [aria-current="page"]', + ); + const currentText = currentCrumb?.textContent?.trim(); + if (currentText) { + return currentText; + } + + return undefined; +} + +export function getPageTitleFromDom(doc: Document = document): string { + const dialog = doc.querySelector(MODAL_DIALOG_SELECTOR); + if (dialog) { + const dialogTitle = dialog.querySelector(DIALOG_TITLE_SELECTOR); + if (dialogTitle && isElementVisible(dialogTitle)) { + const text = dialogTitle.textContent?.trim(); + if (text) { + return text; + } + } + } + + const roots = getChromeRoots(doc); + for (const root of roots) { + const buiTitle = queryFirstVisibleTitle(root, PAGE_TITLE_SCOPED_SELECTORS); + if (buiTitle) { + return buiTitle; + } + const pluginToolbarLabel = getPluginHeaderToolbarChipLabel(root); + if (pluginToolbarLabel) { + return pluginToolbarLabel; + } + const legacyTitle = queryFirstVisibleTitle( + root, + LEGACY_PAGE_TITLE_SELECTORS, + ); + if (legacyTitle) { + return legacyTitle; + } + } + + const normalized = normalizeDocumentTitle(doc.title); + return normalized || 'Current page'; +} + +export function getScreenContextRouteKind( + pathname: string, + search: string, +): ScreenContextRouteKind { + if ( + pathname.includes('/create/') || + pathname.includes('/template/') || + /\/template\/[^/]+/.test(pathname) || + (pathname.includes('/edit/') && pathname.includes('template')) + ) { + return 'template'; + } + if ( + pathname.includes('/search') || + SEARCH_QUERY_PARAM_KEYS.some(key => new URLSearchParams(search).has(key)) || + search.includes('filters[') || + (pathname.includes('/catalog') && /[?&](query|term|filters\[)/.test(search)) + ) { + return 'search'; + } + return 'default'; +} + +export function getSearchQueryFromLocation( + search: string, + doc: Document = document, +): string | undefined { + const params = new URLSearchParams( + search.startsWith('?') ? search : `?${search}`, + ); + for (const key of SEARCH_QUERY_PARAM_KEYS) { + const fromUrl = params.get(key)?.trim(); + if (fromUrl) { + return fromUrl; + } + } + + const inputs = doc.querySelectorAll(SEARCH_INPUT_SELECTOR); + for (const input of inputs) { + if (!(input instanceof HTMLInputElement)) { + continue; + } + if (!isElementVisible(input)) { + continue; + } + const fromInput = input.value?.trim(); + if (fromInput) { + return fromInput; + } + } + + return undefined; +} + +/** @deprecated Prefer resolveScreenContextChipLabel */ +export function getSearchQueryFromPage(): string | undefined { + return getSearchQueryFromLocation(window.location.search); +} + +/** Chip label for `/create/templates` gallery (filters in query string). */ +export const SOFTWARE_TEMPLATES_LIST_CHIP_LABEL = 'Software templates'; + +const SOFTWARE_TEMPLATES_GALLERY_PATH = '/create/templates'; + +function normalizePathname(pathname: string): string { + const trimmed = pathname.trim(); + if (!trimmed || trimmed === '/') { + return '/'; + } + return trimmed.replace(/\/+$/, '') || '/'; +} + +export function isSoftwareTemplatesListPath(pathname: string): boolean { + return normalizePathname(pathname) === SOFTWARE_TEMPLATES_GALLERY_PATH; +} + +export function parseSoftwareTemplateDetailSlug( + pathname: string, +): string | undefined { + const normalized = normalizePathname(pathname); + const prefix = `${SOFTWARE_TEMPLATES_GALLERY_PATH}/`; + if (!normalized.startsWith(prefix)) { + return undefined; + } + const segments = normalized.slice(prefix.length).split('/').filter(Boolean); + if (segments.length < 2) { + return undefined; + } + return segments[segments.length - 1]; +} + +export function getSoftwareTemplateTitleFromDom( + doc: Document = document, +): string | undefined { + const roots = getChromeRoots(doc); + for (const root of roots) { + const title = queryFirstVisibleTitle( + root, + TEMPLATE_INFO_CARD_TITLE_SELECTORS, + ); + if (title) { + return title; + } + } + return undefined; +} + +/** + * Resolves chip text for software template gallery and template entity pages. + */ +export function resolveSoftwareTemplateChipLabel( + pathname: string, + doc: Document = document, +): string | undefined { + if (isSoftwareTemplatesListPath(pathname)) { + return SOFTWARE_TEMPLATES_LIST_CHIP_LABEL; + } + const slug = parseSoftwareTemplateDetailSlug(pathname); + if (!slug) { + return undefined; + } + return getSoftwareTemplateTitleFromDom(doc) ?? slug; +} + +export function resolveScreenContextChipLabel(options: { + pathname: string; + search: string; + document?: Document; +}): ResolvedScreenContextLabel { + const doc = options.document ?? document; + const routeKind = getScreenContextRouteKind(options.pathname, options.search); + + if (routeKind === 'search') { + const searchQuery = getSearchQueryFromLocation(options.search, doc); + if (searchQuery) { + return { + label: truncateChipLabel(searchQuery), + routeKind, + searchQuery, + }; + } + } + + if (routeKind === 'template') { + const softwareTemplateLabel = resolveSoftwareTemplateChipLabel( + options.pathname, + doc, + ); + if (softwareTemplateLabel) { + return { + label: truncateChipLabel(softwareTemplateLabel), + routeKind, + }; + } + } + + const pageTitle = getPageTitleFromDom(doc); + return { + label: truncateChipLabel(pageTitle), + routeKind, + searchQuery: + routeKind === 'search' + ? getSearchQueryFromLocation(options.search, doc) + : undefined, + }; +} + +export function getScreenContextChipLabel(): string { + return resolveScreenContextChipLabel({ + pathname: window.location.pathname, + search: window.location.search, + }).label; +} + +export type TooltipLine1Params = { + chipLabel: string; + routeKind: ScreenContextRouteKind; + pathname?: string; + searchQuery?: string; +}; + +export function getScreenContextTooltipLine1Key( + routeKind: ScreenContextRouteKind, + pathname?: string, +): + | 'contextChip.tooltip.askAbout' + | 'contextChip.tooltip.template' + | 'contextChip.tooltip.search' { + switch (routeKind) { + case 'template': + if (pathname && isSoftwareTemplatesListPath(pathname)) { + return 'contextChip.tooltip.askAbout'; + } + return 'contextChip.tooltip.template'; + case 'search': + return 'contextChip.tooltip.search'; + default: + return 'contextChip.tooltip.askAbout'; + } +} + +export function getScreenContextTooltipLine1Text( + params: TooltipLine1Params, + t: (key: string, options?: Record) => string, +): string { + const label = params.chipLabel; + const key = getScreenContextTooltipLine1Key( + params.routeKind, + params.pathname, + ); + return t(key, { label }); +} + +export type TooltipLine2Input = { + domEnabled: boolean; + screenshotsEnabled: boolean; + supportsVision: boolean; +}; + +export function getScreenContextTooltipLine2Key( + input: TooltipLine2Input, +): ScreenContextTooltipLine2Key { + const { domEnabled, screenshotsEnabled, supportsVision } = input; + + const willSendDom = domEnabled; + const willSendScreenshot = screenshotsEnabled && supportsVision; + + if (willSendDom && willSendScreenshot) { + return 'fullContext'; + } + + if (!domEnabled && !screenshotsEnabled) { + return 'adminLimited'; + } + + if (!domEnabled && screenshotsEnabled && supportsVision) { + return 'screenshotOnly'; + } + + if (willSendDom) { + const adminOff = !screenshotsEnabled; + const noVision = !supportsVision; + if (adminOff && noVision) { + return 'textOnlyCombined'; + } + if (noVision) { + return 'textOnlyNoVision'; + } + if (adminOff) { + return 'textOnlyAdminScreenshotsOff'; + } + } + + // DOM off, screenshots on, but model lacks vision — nothing attaches. + if (!domEnabled && screenshotsEnabled && !supportsVision) { + return 'domOffNoVision'; + } + + return 'adminLimited'; +} + +export function shouldAttachScreenContext( + input: ShouldAttachScreenContextInput, +): ShouldAttachScreenContextResult { + const baseGate = + input.adminEnabled && + input.sharingEnabled && + !input.paused && + !input.isFullscreen; + + if (!baseGate) { + return { attachDom: false, attachScreenshot: false }; + } + + return { + attachDom: input.domEnabled, + attachScreenshot: input.screenshotsEnabled && input.supportsVision, + }; +} + +export type BuildScreenContextAttachmentsInput = + ShouldAttachScreenContextInput & { + domExtractionMaxChars: number; + }; + +export async function buildScreenContextAttachments( + input: BuildScreenContextAttachmentsInput, +): Promise { + const { attachDom, attachScreenshot } = shouldAttachScreenContext(input); + const attachments: Attachment[] = []; + + if (attachDom) { + try { + const domContext = extractPageContext({ + maxChars: input.domExtractionMaxChars, + }); + if (domContext) { + attachments.push({ + attachment_type: 'configuration', + content_type: 'text/plain', + content: domContext, + }); + } + } catch { + // non-fatal + } + } + + if (attachScreenshot) { + const result = await captureScreenshot({ + excludeSelector: '[data-screen-capture-exclude]', + }); + if (result.success) { + attachments.push({ + attachment_type: 'image', + content_type: result.contentType, + content: result.base64, + }); + } + } + + return attachments; +}