test(intelligent-assistant): screen context QE coverage (stacks on #4862)

workspaces/intelligent-assistant/.changeset/screen-context-chip-ux.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor
+---
+
+Add screen context UX: kebab opt-in, context chip (recording/paused/unavailable), and gated DOM and screenshot attachments on send per RHIDP-14319.

workspaces/intelligent-assistant/e2e-tests/fixtures/responses.ts

@@ -27,6 +27,7 @@ export const models = [
     type: 'model',
     owned_by: 'library',
     provider_id: 'mock-provider-1',
+    supportsVision: false,
   },
   {
     identifier: 'mock-provider-1/mock-model-2',
@@ -36,6 +37,7 @@ export const models = [
     type: 'model',
     owned_by: 'library',
     provider_id: 'mock-provider-1',
+    supportsVision: true,
   },
 ];
 

workspaces/intelligent-assistant/e2e-tests/lightspeed.conversation.test.ts

@@ -411,6 +411,37 @@ test.describe('Intelligent assistant conversation', () => {
         await sharedPage.keyboard.press('Escape');
       });
 
+      test('Vision models show screenshot indicator in the dropdown list only', async () => {
+        const visionAriaLabel =
+          translations['modelSelector.visionScreenshot.ariaLabel'];
+        const toggle = sharedPage.locator(
+          `button[aria-label="${translations['aria.chatbotSelector']}"]`,
+        );
+
+        await expect(toggle.getByLabel(visionAriaLabel)).toHaveCount(0);
+
+        await toggle.click();
+
+        const visionItem = sharedPage.getByRole('menuitem', {
+          name: /mock-model-2/,
+        });
+        await expect(visionItem.getByLabel(visionAriaLabel)).toBeVisible();
+
+        const textItem = sharedPage.getByRole('menuitem', {
+          name: 'mock-model-1',
+        });
+        await expect(textItem.getByLabel(visionAriaLabel)).toHaveCount(0);
+
+        await expect(
+          visionItem.locator('.lightspeed-model-tick-slot'),
+        ).toBeVisible();
+        await expect(
+          textItem.locator('.lightspeed-model-tick-slot'),
+        ).toBeVisible();
+
+        await sharedPage.keyboard.press('Escape');
+      });
+
       test('Model selector becomes disabled after sending a message', async () => {
         await sendMessage(
           LIGHTSPEED_E2E_DEFAULT_BOT_QUERY,

workspaces/intelligent-assistant/e2e-tests/lightspeed.screen-context.test.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, type Page } from '@playwright/test';
+import {
+  conversations,
+  generateQueryResponse,
+  modelBaseUrl,
+} from './fixtures/responses';
+import {
+  expectConversationArea,
+  openChatbot,
+  selectDisplayMode,
+  waitForBackstageCatalogReady,
+} from './pages/LightspeedPage';
+import {
+  bootstrapLightspeedE2ePage,
+  LIGHTSPEED_E2E_DEFAULT_BOT_QUERY,
+} from './utils/lightspeedE2eSetup';
+import { sendMessage } from './utils/testHelper';
+import { openChatbotSettings } from './utils/chatManagement';
+import { mockQuery } from './utils/devMode';
+import {
+  disableScreenContextViaKebab,
+  enableScreenContextViaKebab,
+  expectScreenContextChipHidden,
+  expectScreenContextPausedVisible,
+  expectScreenContextRecordingVisible,
+  expectScreenContextUnavailableVisible,
+  pauseScreenContextChip,
+  resumeScreenContextChip,
+  selectEnableScreenContext,
+  verifyEnableScreenContextOption,
+} from './utils/screenContext';
+import type { LightspeedMessages } from './utils/translations';
+
+test.describe('Intelligent assistant screen context', () => {
+  let translations: LightspeedMessages;
+  let sharedPage: Page;
+
+  test.beforeAll(async ({ browser }) => {
+    const boot = await bootstrapLightspeedE2ePage(browser);
+    sharedPage = boot.page;
+    translations = boot.translations;
+  });
+
+  test.beforeEach(async () => {
+    await sharedPage.goto('/catalog');
+    await waitForBackstageCatalogReady(sharedPage);
+    await openChatbot(sharedPage, translations);
+  });
+
+  test('kebab Enable shows recording chip; Disable hides it', async () => {
+    await expectScreenContextChipHidden(sharedPage);
+
+    // Verify the enable item, then click it in the same open menu.
+    // Escape closes the chatbot panel (not just the menu), so do not dismiss
+    // and reopen Options between verify and enable.
+    await openChatbotSettings(sharedPage, translations);
+    await verifyEnableScreenContextOption(sharedPage, translations);
+    await selectEnableScreenContext(sharedPage, translations);
+    await expectScreenContextRecordingVisible(sharedPage);
+
+    await disableScreenContextViaKebab(sharedPage, translations);
+    await expectScreenContextChipHidden(sharedPage);
+  });
+
+  test('chip pause/resume toggles Context: paused label', async () => {
+    await enableScreenContextViaKebab(sharedPage, translations);
+    await pauseScreenContextChip(sharedPage);
+    await expectScreenContextPausedVisible(sharedPage, translations);
+
+    await resumeScreenContextChip(sharedPage, translations);
+    await expectScreenContextRecordingVisible(sharedPage);
+
+    await disableScreenContextViaKebab(sharedPage, translations);
+  });
+
+  test('fullscreen shows Context: unavailable', async () => {
+    await enableScreenContextViaKebab(sharedPage, translations);
+    await selectDisplayMode(sharedPage, translations, 'Fullscreen');
+    await expectConversationArea(sharedPage, translations, 'Fullscreen');
+    // Guest sessions do not persist sharing across the fullscreen remount;
+    // re-enable so the unavailable chip can render on the new surface.
+    // Do not use enableScreenContextViaKebab — it asserts the recording chip.
+    await openChatbotSettings(sharedPage, translations);
+    await verifyEnableScreenContextOption(sharedPage, translations);
+    await selectEnableScreenContext(sharedPage, translations);
+    await expectScreenContextUnavailableVisible(sharedPage, translations);
+  });
+
+  test('paused send omits screen-context attachments', async () => {
+    await enableScreenContextViaKebab(sharedPage, translations);
+    await pauseScreenContextChip(sharedPage);
+    await expectScreenContextPausedVisible(sharedPage, translations);
+
+    let capturedAttachments: Array<{ attachment_type?: string }> | undefined;
+
+    await sharedPage.unroute(`${modelBaseUrl}/v1/query`);
+    await sharedPage.route(`${modelBaseUrl}/v1/query`, async route => {
+      const payload = route.request().postDataJSON();
+      capturedAttachments = payload.attachments;
+      if (payload.conversation_id) {
+        conversations[1].conversation_id = payload.conversation_id;
+      }
+      const conversationId =
+        conversations[1].conversation_id ?? conversations[0].conversation_id;
+      await route.fulfill({
+        body: generateQueryResponse(conversationId),
+      });
+    });
+
+    await sendMessage(
+      LIGHTSPEED_E2E_DEFAULT_BOT_QUERY,
+      sharedPage,
+      translations,
+    );
+
+    const attachments = capturedAttachments ?? [];
+    expect(
+      attachments.some(
+        a =>
+          a.attachment_type === 'image' ||
+          a.attachment_type === 'configuration',
+      ),
+    ).toBe(false);
+
+    await sharedPage.unroute(`${modelBaseUrl}/v1/query`);
+    await mockQuery(
+      sharedPage,
+      LIGHTSPEED_E2E_DEFAULT_BOT_QUERY,
+      conversations,
+    );
+
+    await disableScreenContextViaKebab(sharedPage, translations);
+  });
+});

workspaces/intelligent-assistant/e2e-tests/utils/chatManagement.ts

@@ -307,9 +307,11 @@ export const openChatbotSettings = async (
   page: Page,
   translations: LightspeedMessages,
 ) => {
-  await page
-    .getByRole('button', { name: translations['aria.options.label'] })
-    .click();
+  const options = page
+    .locator('.pf-chatbot__header')
+    .getByRole('button', { name: translations['aria.options.label'] });
+  await expect(options).toBeVisible({ timeout: 15_000 });
+  await options.click();
 };
 
 export const verifyChatbotSettingsVisible = async (

workspaces/intelligent-assistant/e2e-tests/utils/screenContext.ts

@@ -0,0 +1,125 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect, type Page } from '@playwright/test';
+import type { LightspeedMessages } from './translations';
+import { openChatbotSettings } from './chatManagement';
+
+export const screenContextChip = (page: Page) =>
+  page.locator('.lightspeed-page-context-label');
+
+export const expectScreenContextChipHidden = async (page: Page) => {
+  await expect(screenContextChip(page)).toHaveCount(0);
+};
+
+export const expectScreenContextRecordingVisible = async (page: Page) => {
+  await expect(
+    page.locator('.lightspeed-page-context-label-recording'),
+  ).toBeVisible();
+};
+
+export const expectScreenContextPausedVisible = async (
+  page: Page,
+  _translations: LightspeedMessages,
+) => {
+  await expect(
+    page.locator('.lightspeed-page-context-label-paused'),
+  ).toBeVisible();
+};
+
+export const expectScreenContextUnavailableVisible = async (
+  page: Page,
+  _translations: LightspeedMessages,
+) => {
+  await expect(
+    page.locator('.lightspeed-page-context-label-unavailable'),
+  ).toBeVisible({ timeout: 15_000 });
+};
+
+export const verifyEnableScreenContextOption = async (
+  page: Page,
+  translations: LightspeedMessages,
+) => {
+  await expect(
+    page.getByRole('menuitem', {
+      name: `${translations['settings.screenContext.enable']} ${translations['settings.screenContext.disabled.description']}`,
+    }),
+  ).toBeVisible();
+};
+
+export const verifyDisableScreenContextOption = async (
+  page: Page,
+  translations: LightspeedMessages,
+) => {
+  await expect(
+    page.getByRole('menuitem', {
+      name: `${translations['settings.screenContext.disable']} ${translations['settings.screenContext.enabled.description']}`,
+    }),
+  ).toBeVisible();
+};
+
+export const selectEnableScreenContext = async (
+  page: Page,
+  translations: LightspeedMessages,
+) => {
+  await page
+    .getByRole('menuitem', {
+      name: translations['settings.screenContext.enable'],
+    })
+    .click();
+};
+
+export const selectDisableScreenContext = async (
+  page: Page,
+  translations: LightspeedMessages,
+) => {
+  await page
+    .getByRole('menuitem', {
+      name: translations['settings.screenContext.disable'],
+    })
+    .click();
+};
+
+export const enableScreenContextViaKebab = async (
+  page: Page,
+  translations: LightspeedMessages,
+) => {
+  await openChatbotSettings(page, translations);
+  await verifyEnableScreenContextOption(page, translations);
+  await selectEnableScreenContext(page, translations);
+  await expectScreenContextRecordingVisible(page);
+};
+
+export const disableScreenContextViaKebab = async (
+  page: Page,
+  translations: LightspeedMessages,
+) => {
+  await openChatbotSettings(page, translations);
+  await verifyDisableScreenContextOption(page, translations);
+  await selectDisableScreenContext(page, translations);
+  await expectScreenContextChipHidden(page);
+};
+
+export const pauseScreenContextChip = async (page: Page) => {
+  await page.locator('.lightspeed-page-context-label-recording').click();
+};
+
+export const resumeScreenContextChip = async (
+  page: Page,
+  _translations: LightspeedMessages,
+) => {
+  await page.locator('.lightspeed-page-context-label-paused').click();
+};

workspaces/intelligent-assistant/plugins/intelligent-assistant/README.md

@@ -161,7 +161,18 @@ Intelligent assistant supports multiple **display modes** from Settings (for exa
 
 ### Screen Context
 
-When enabled, the assistant automatically captures structured page context from the current RHDH viewport and attaches it to each message. This provides the LLM with headings, tables, alerts, form fields, filters, and other on-screen content for more accurate, context-aware responses.
+Screen context attaches structured page content from the current RHDH viewport to chat messages (headings, tables, alerts, form fields, filters, and optional screenshots) so the LLM can give more accurate, context-aware responses.
+
+Enablement has two levels:
+
+1. **Administrator** — set `screen-context.enabled: true` in `app-config.yaml` (default: `false`).
+2. **User** — opt in from the chatbot options (kebab) menu with **Enable screen context**. Sharing is off until the user opts in.
+
+When sharing is on, a **context chip** in the message bar shows the current page label and state:
+
+- **Recording** — context will be attached on send; click the chip to pause.
+- **Paused** — context is not attached; click to resume.
+- **Unavailable** — fullscreen mode; switch to Overlay or Docked to use screen context.
 
 Configure in `app-config.yaml`:
 
@@ -176,7 +187,8 @@ intelligent-assistant:
       maxChars: 8000 # Max characters extracted per page (default: 8000)
 ```
 
-- `screen-context.enabled` — enables the full screen-context feature (DOM extraction and optional screenshots).
+- `screen-context.enabled` — enables the full screen-context feature (DOM extraction and optional screenshots). Users must still opt in via the kebab menu.
+- `screen-context.screenshots.enabled` — toggles screenshot capture. Screenshots are only attached when the selected model supports vision (`supportsVision`).
 - `screen-context.dom-extraction.enabled` — toggles DOM text extraction independently of screenshots.
 - `screen-context.dom-extraction.maxChars` — caps the extracted text size to control LLM token usage.