diff --git a/.github/workflows/.reusable-docker-e2e-tests.yml b/.github/workflows/.reusable-docker-e2e-tests.yml index ddbfa0913665..2f9debaf0bc8 100644 --- a/.github/workflows/.reusable-docker-e2e-tests.yml +++ b/.github/workflows/.reusable-docker-e2e-tests.yml @@ -27,6 +27,16 @@ on: description: The runner label to use. Defaults to `depot-ubuntu-latest` required: false default: depot-ubuntu-latest + visual-regression: + type: boolean + description: Enable visual regression screenshot comparison + required: false + default: false + visual-regression-update: + type: boolean + description: Update visual regression baselines (use on main branch) + required: false + default: false secrets: GCR_TOKEN: description: A token to use for logging into Github Container Registry. If not provided, login does not occur. @@ -78,6 +88,24 @@ jobs: - name: Login to Depot Registry run: depot pull-token | docker login -u x-token --password-stdin registry.depot.dev + - name: Prepare visual regression snapshots directory + if: inputs.visual-regression + working-directory: frontend + run: mkdir -p e2e/visual-regression-snapshots + + - name: Download visual regression baselines + if: inputs.visual-regression + id: download-baseline + continue-on-error: true + uses: dawidd6/action-download-artifact@v6 + with: + github_token: ${{ secrets.GCR_TOKEN }} + workflow: platform-docker-build-test-publish.yml + branch: main + name: visual-regression-baselines + path: frontend/e2e/visual-regression-snapshots/ + if_no_artifact_found: warn + - name: Run tests on dockerised frontend working-directory: frontend run: make test @@ -87,6 +115,8 @@ jobs: E2E_IMAGE: ${{ inputs.e2e-image }} E2E_CONCURRENCY: ${{ inputs.concurrency }} E2E_RETRIES: 2 + VISUAL_REGRESSION: ${{ inputs.visual-regression && '1' || '' }} + VISUAL_REGRESSION_ARGS: ${{ inputs.visual-regression-update && '--update-snapshots' || '' }} SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} timeout-minutes: 20 @@ -165,3 +195,48 @@ jobs: header: playwright-e2e-results append: true message: ${{ steps.report-summary-success.outputs.summary || steps.report-summary-failure.outputs.summary }} + + # Visual regression: after all E2E retries, run comparison and upload results + - name: Upload visual regression baselines (main branch) + if: always() && inputs.visual-regression-update + uses: actions/upload-artifact@v4 + with: + name: visual-regression-baselines + path: frontend/e2e/visual-regression-screenshots/ + retention-days: 90 + overwrite: true + + - name: Upload visual regression report + if: always() && inputs.visual-regression && !inputs.visual-regression-update + uses: actions/upload-artifact@v4 + with: + name: visual-regression-report-${{ github.run_id }}-${{ strategy.job-index }} + path: frontend/e2e/visual-regression-report/ + retention-days: 30 + + - name: Generate visual regression summary + if: always() && inputs.visual-regression && !inputs.visual-regression-update && github.event_name == 'pull_request' + id: visual-regression-summary + shell: bash + run: | + if [ "${{ steps.download-baseline.outcome }}" != "success" ]; then + echo "message=No baseline found — first run. Baselines will be generated after merge to main." >> $GITHUB_OUTPUT + else + SCREENSHOT_COUNT=$(find frontend/e2e/visual-regression-screenshots -name "*.png" 2>/dev/null | wc -l | tr -d ' ') + REPORT_EXISTS=$(test -d frontend/e2e/visual-regression-report && echo "true" || echo "false") + if [ "$REPORT_EXISTS" = "true" ]; then + echo "message=$SCREENSHOT_COUNT screenshots compared. See report for details." >> $GITHUB_OUTPUT + else + echo "message=$SCREENSHOT_COUNT screenshots captured but comparison did not run." >> $GITHUB_OUTPUT + fi + fi + + - name: Comment PR with visual regression results + if: always() && inputs.visual-regression && !inputs.visual-regression-update && github.event_name == 'pull_request' && steps.visual-regression-summary.outputs.message + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: visual-regression-results + message: | + ## Visual Regression + ${{ steps.visual-regression-summary.outputs.message }} + [View full report](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) diff --git a/.github/workflows/platform-docker-build-test-publish.yml b/.github/workflows/platform-docker-build-test-publish.yml index 273708e60818..4a603f1ba3fa 100644 --- a/.github/workflows/platform-docker-build-test-publish.yml +++ b/.github/workflows/platform-docker-build-test-publish.yml @@ -85,6 +85,10 @@ jobs: e2e-image: ${{ needs.docker-build-e2e.outputs.image }} api-image: ${{ matrix.args.api-image }} args: ${{ matrix.args.args }} + # Run visual regression on the enterprise E2E job (which runs all OSS + enterprise tests) + # on a single architecture only, and update baselines since this is the main branch. + visual-regression: ${{ matrix.runs-on == 'depot-ubuntu-latest-16' && contains(matrix.args.args, '@enterprise') }} + visual-regression-update: ${{ matrix.runs-on == 'depot-ubuntu-latest-16' && contains(matrix.args.args, '@enterprise') }} secrets: GCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} diff --git a/.github/workflows/platform-pull-request.yml b/.github/workflows/platform-pull-request.yml index 255462500541..d2f8c6fa5504 100644 --- a/.github/workflows/platform-pull-request.yml +++ b/.github/workflows/platform-pull-request.yml @@ -168,6 +168,7 @@ jobs: e2e-image: ${{ needs.docker-build-e2e.outputs.image }} api-image: ${{ needs.docker-build-private-cloud.outputs.image }} args: --grep "@oss|@enterprise" + visual-regression: ${{ matrix.runs-on == 'depot-ubuntu-latest-16' }} secrets: GCR_TOKEN: ${{ needs.permissions-check.outputs.can-write == 'true' && secrets.GITHUB_TOKEN || '' }} SLACK_TOKEN: ${{ needs.permissions-check.outputs.can-write == 'true' && secrets.SLACK_TOKEN || '' }} diff --git a/frontend/.gitignore b/frontend/.gitignore index 29f737c9d966..e29afcc8f3d4 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -33,3 +33,10 @@ common/project.js # Playwright e2e/playwright-report/ e2e/test-results/ + +# Visual regression (baselines stored as CI artifacts, not in git) +e2e/visual-regression-snapshots/ +e2e/visual-regression-screenshots/ +e2e/visual-regression-report/ +e2e/tests/_visual-regression-compare.pw.ts +e2e/tests/_visual-regression-compare.pw.ts-snapshots/ diff --git a/frontend/Makefile b/frontend/Makefile index 463342fa0d8c..c081b6990191 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -31,11 +31,16 @@ serve: test: @echo "Running E2E tests..." @docker compose run --name e2e-test-run frontend \ - sh -c 'npx cross-env E2E_CONCURRENCY=${E2E_CONCURRENCY} E2E_RETRIES=${E2E_RETRIES} npm run test -- $(opts)' \ + sh -c 'npx cross-env E2E_CONCURRENCY=${E2E_CONCURRENCY} E2E_RETRIES=${E2E_RETRIES} npm run test -- $(opts); \ + EXIT=$$?; \ + if [ "$${VISUAL_REGRESSION}" = "1" ]; then npm run test:visual:compare -- $${VISUAL_REGRESSION_ARGS} || true; fi; \ + exit $$EXIT' \ || TEST_FAILED=1; \ echo "Copying test results from container..."; \ docker cp e2e-test-run:/srv/flagsmith/e2e/test-results ./e2e/test-results 2>/dev/null || echo "No test results to copy"; \ docker cp e2e-test-run:/srv/flagsmith/e2e/playwright-report ./e2e/playwright-report 2>/dev/null || echo "No HTML report to copy"; \ + docker cp e2e-test-run:/srv/flagsmith/e2e/visual-regression-screenshots ./e2e/visual-regression-screenshots 2>/dev/null || echo "No visual regression screenshots to copy"; \ + docker cp e2e-test-run:/srv/flagsmith/e2e/visual-regression-report ./e2e/visual-regression-report 2>/dev/null || echo "No visual regression report to copy"; \ docker rm e2e-test-run 2>/dev/null || true; \ if [ "$$TEST_FAILED" = "1" ]; then \ echo "\n=== API logs ===" && docker compose logs flagsmith-api && \ @@ -50,3 +55,10 @@ test-oss: .PHONY: test-enterprise test-enterprise: @$(MAKE) test opts="--grep @enterprise" + +# Visual regression: run E2E tests with screenshot comparison enabled. +# Snapshots are shared via volume mount in docker-compose-e2e-tests.yml. +.PHONY: test-visual +test-visual: + @mkdir -p e2e/visual-regression-snapshots + @VISUAL_REGRESSION=1 $(MAKE) test opts="$(opts)" diff --git a/frontend/README.md b/frontend/README.md index 900f995126d8..6cd3e0f3e3df 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -145,6 +145,32 @@ E2E_RETRIES=0 SKIP_BUNDLE=1 E2E_CONCURRENCY=1 npm run test -- tests/flag-tests.p - `trace.zip` - Interactive trace viewer - Screenshots and videos +#### Visual Regression + +Visual regression screenshots are captured during E2E tests via `visualSnapshot()` calls. They are a no-op unless `VISUAL_REGRESSION=1` is set. Comparison runs as a separate step after all E2E retries complete, so flaky tests don't affect the report. + +```bash +# 1. Run E2E tests with screenshot capture (with retries) +VISUAL_REGRESSION=1 npm run test + +# 2a. Generate/update baselines from captured screenshots +npm run test:visual:compare -- --update-snapshots + +# 2b. Compare screenshots against baselines (generates Playwright report with diffs) +npm run test:visual:compare + +# 3. Open the report +npm run test:visual:report +``` + +Visual diffs never fail CI — they are reported via PR comment and the Playwright HTML report. + +Screenshots are saved to `e2e/visual-regression-screenshots/`, baselines to `e2e/visual-regression-snapshots/` (both git-ignored). In CI, the main branch uploads screenshots as baseline artifacts, and PRs download them for comparison. + +| Variable | Description | +|----------|-------------| +| `VISUAL_REGRESSION=1` | Enable screenshot capture during E2E tests | + #### Claude Code Commands When using Claude Code, these commands are available for e2e testing: diff --git a/frontend/docker-compose-e2e-tests.yml b/frontend/docker-compose-e2e-tests.yml index 914efd143759..1f881ece6303 100644 --- a/frontend/docker-compose-e2e-tests.yml +++ b/frontend/docker-compose-e2e-tests.yml @@ -49,12 +49,14 @@ services: FLAGSMITH_API_URL: http://flagsmith-api:8000/api/v1/ SLACK_TOKEN: ${SLACK_TOKEN} GITHUB_ACTION_URL: ${GITHUB_ACTION_URL} + VISUAL_REGRESSION: ${VISUAL_REGRESSION:-} ports: - 3000:3000 depends_on: flagsmith-api: condition: service_healthy - + volumes: + - ./e2e/visual-regression-snapshots:/srv/flagsmith/e2e/visual-regression-snapshots links: - flagsmith-api:flagsmith-api command: [npm, run, test] diff --git a/frontend/e2e/compare-visual-regression.ts b/frontend/e2e/compare-visual-regression.ts new file mode 100644 index 000000000000..19ccc3b95d55 --- /dev/null +++ b/frontend/e2e/compare-visual-regression.ts @@ -0,0 +1,74 @@ +import * as fs from 'fs' +import * as path from 'path' + +const BASELINES_DIR = path.resolve(__dirname, 'visual-regression-snapshots') +const SCREENSHOTS_DIR = path.resolve(__dirname, 'visual-regression-screenshots') +const COMPARE_TEST_FILE = path.resolve(__dirname, 'tests', '_visual-regression-compare.pw.ts') + +/** + * Generates a Playwright test file that compares each captured screenshot + * against its baseline using toMatchSnapshot(). Run this AFTER E2E tests + * complete to get a Playwright HTML report with diff viewer. + * + * Screenshots and baselines use the same flat naming convention: + * {testFileName}--{snapshotName}.png (dots replaced with dashes) + * e.g. flag-tests-pw-ts--features-list.png + */ + +if (!fs.existsSync(SCREENSHOTS_DIR)) { + console.log('No screenshots found — run E2E tests with VISUAL_REGRESSION=1 first.') + process.exit(0) +} + +// Collect screenshots +const screenshots = fs + .readdirSync(SCREENSHOTS_DIR) + .filter((f) => f.endsWith('.png')) + +if (screenshots.length === 0) { + console.log('No screenshots to compare.') + process.exit(0) +} + +if (!fs.existsSync(BASELINES_DIR)) { + fs.mkdirSync(BASELINES_DIR, { recursive: true }) +} + +// Build test entries from all screenshots +const pairs: { file: string; label: string }[] = [] +for (const png of screenshots) { + const label = png + .replace('.png', '') + .replace(/^(.+?)--(.+)$/, (_, testFile, name) => { + const restored = testFile.replace(/-pw-ts$/, '.pw.ts').replace(/-/g, '.') + return `${restored} / ${name.replace(/-/g, ' ')}` + }) + pairs.push({ file: png, label }) +} + +// Generate Playwright test file +const testCases = pairs + .map(({ file, label }) => { + const screenshotPath = path.join(SCREENSHOTS_DIR, file).replace(/\\/g, '\\\\').replace(/'/g, "\\'") + return ` + test('${label}', async () => { + const screenshot = fs.readFileSync('${screenshotPath}') + expect(screenshot).toMatchSnapshot('${file}', { + maxDiffPixels: 300, + threshold: 0.02, + }) + })` + }) + .join('\n') + +const testContent = `// Auto-generated by compare-visual-regression.ts — do not edit +import { test, expect } from '@playwright/test' +import * as fs from 'fs' + +test.describe('Visual Regression', () => { +${testCases} +}) +` + +fs.writeFileSync(COMPARE_TEST_FILE, testContent) +console.log(`Generated ${pairs.length} comparison tests → ${COMPARE_TEST_FILE}`) diff --git a/frontend/e2e/helpers/index.ts b/frontend/e2e/helpers/index.ts index 2b52a0994891..bf836787ddcb 100644 --- a/frontend/e2e/helpers/index.ts +++ b/frontend/e2e/helpers/index.ts @@ -1,3 +1,4 @@ export * from './utils.playwright'; export * from './browser-logging.playwright'; export * from './e2e-helpers.playwright'; +export * from './visual-regression'; diff --git a/frontend/e2e/helpers/visual-regression.ts b/frontend/e2e/helpers/visual-regression.ts new file mode 100644 index 000000000000..e9cc506d6efa --- /dev/null +++ b/frontend/e2e/helpers/visual-regression.ts @@ -0,0 +1,140 @@ +import { expect, Page, TestInfo } from '@playwright/test' +import * as fs from 'fs' +import * as path from 'path' + +/** + * CSS injected before every visual snapshot to hide dynamic content + * that changes between runs. Playwright's toHaveScreenshot() already + * handles animations (animations: 'disabled') and caret (caret: 'hide'), + * so we only target app-specific volatile elements here. + */ +const STABILISING_CSS = ` + /* Hide environment select (contains dynamic API key) */ + #environment-select { + visibility: hidden !important; + } + + /* Hide timestamps and relative dates */ + .ago, + time, + [data-test*="timestamp"], + [data-test*="ago"], + .text-muted:has(> .ago), + .relative-date { + visibility: hidden !important; + } + + /* Hide loading spinners */ + .spinner, + .loader, + [class*="spinner"], + [class*="loader"] { + display: none !important; + } + + /* Hide any live chat / support widgets */ + .intercom-launcher, + #intercom-container, + .drift-widget, + [class*="chatbot"], + iframe[title*="chat"], + iframe[title*="Chat"] { + display: none !important; + } + + /* Stabilise scrollbars across platforms */ + ::-webkit-scrollbar { + display: none !important; + } + * { + scrollbar-width: none !important; + } +` + +/** Directory where screenshots are captured during E2E runs */ +const SCREENSHOTS_DIR = path.resolve(process.cwd(), 'e2e', 'visual-regression-screenshots') + +/** Directory where baselines live (downloaded from main in CI) */ +const BASELINES_DIR = path.resolve(process.cwd(), 'e2e', 'visual-regression-snapshots') + +/** + * Whether visual regression snapshots are enabled for this run. + */ +export function isVisualRegressionEnabled(): boolean { + return process.env.VISUAL_REGRESSION === '1' +} + +/** + * Wait for the page to settle before taking a screenshot. + */ +async function preparePage(page: Page): Promise { + await page.addStyleTag({ content: STABILISING_CSS }) + + // Wait for images to finish loading + await page + .evaluate(() => { + return Promise.all( + Array.from(document.images) + .filter((img) => !img.complete) + .map( + (img) => + new Promise((resolve) => { + img.addEventListener('load', resolve) + img.addEventListener('error', resolve) + setTimeout(resolve, 5000) + }), + ), + ) + }) + .catch(() => {}) + + // Double rAF to ensure paint is complete + await page.evaluate(() => { + return new Promise((resolve) => { + requestAnimationFrame(() => { + requestAnimationFrame(() => { + resolve(undefined) + }) + }) + }) + }) + + // Small settle time for any final layout shifts + await page.waitForTimeout(500) +} + +/** + * Take a screenshot during E2E tests and save it to the screenshots directory. + * + * This ONLY captures the screenshot — it does NOT compare against baselines. + * Comparison happens as a separate step after all E2E retries have completed, + * via `npx tsx e2e/compare-visual-regression.ts`. + * + * @param page Playwright page + * @param name Descriptive snapshot name, e.g. "features-list" + * @param testInfo Playwright testInfo for resolving the test file name + */ +export async function visualSnapshot( + page: Page, + name: string, + testInfo: TestInfo, +): Promise { + if (!isVisualRegressionEnabled()) return + + await preparePage(page) + + // Save with the sanitised name Playwright's toMatchSnapshot expects: + // {testFileName}--{name} with dots replaced by dashes + const testFileName = path.basename(testInfo.file) + const sanitisedName = `${testFileName}--${name}`.replace(/\./g, '-') + fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true }) + + const screenshotPath = path.join(SCREENSHOTS_DIR, `${sanitisedName}.png`) + await page.screenshot({ + path: screenshotPath, + fullPage: true, + animations: 'disabled', + caret: 'hide', + scale: 'css', + }) +} diff --git a/frontend/e2e/tests/change-request-test.pw.ts b/frontend/e2e/tests/change-request-test.pw.ts index 90fd71589411..d58cc1af6e44 100644 --- a/frontend/e2e/tests/change-request-test.pw.ts +++ b/frontend/e2e/tests/change-request-test.pw.ts @@ -1,5 +1,5 @@ import { test, expect } from '../test-setup' -import { byId, getFlagsmith, log, createHelpers } from '../helpers' +import { byId, getFlagsmith, log, createHelpers, visualSnapshot } from '../helpers' import { E2E_NON_ADMIN_USER_WITH_PROJECT_PERMISSIONS, E2E_TEST_PROJECT, @@ -10,7 +10,7 @@ import { test.describe('Change Request Tests', () => { test('Change requests can be created, approved, and published with 4-eyes approval @enterprise', async ({ page, - }) => { + }, testInfo) => { const { assertChangeRequestCount, approveChangeRequest, @@ -107,6 +107,7 @@ test.describe('Change Request Tests', () => { log('Go to change requests') await gotoChangeRequests() + await visualSnapshot(page, 'change-requests-list', testInfo) log('Open change request') await openChangeRequest(0) diff --git a/frontend/e2e/tests/environment-test.pw.ts b/frontend/e2e/tests/environment-test.pw.ts index 881e48595d4b..7c168391627d 100644 --- a/frontend/e2e/tests/environment-test.pw.ts +++ b/frontend/e2e/tests/environment-test.pw.ts @@ -1,9 +1,9 @@ import { test, expect } from '../test-setup'; -import { byId, log, createHelpers } from '../helpers'; +import { byId, log, createHelpers, visualSnapshot } from '../helpers'; import { PASSWORD, E2E_USER, E2E_TEST_PROJECT } from '../config' test.describe('Environment Tests', () => { - test('Environments can be created, renamed, and deleted @oss', async ({ page }) => { + test('Environments can be created, renamed, and deleted @oss', async ({ page }, testInfo) => { const { click, createEnvironment, @@ -11,6 +11,7 @@ test.describe('Environment Tests', () => { login, setText, waitForElementVisible, + waitForToastsToClear, } = createHelpers(page); log('Login') @@ -20,6 +21,9 @@ test.describe('Environment Tests', () => { log('Create environment') await click('#create-env-link') await createEnvironment('Staging') + await waitForToastsToClear() + await visualSnapshot(page, 'environment-created', testInfo) + log('Edit Environment') await click('#env-settings-link') await setText("[name='env-name']", 'Internal') diff --git a/frontend/e2e/tests/flag-tests.pw.ts b/frontend/e2e/tests/flag-tests.pw.ts index a8ac0b4bcf6a..805d103680ca 100644 --- a/frontend/e2e/tests/flag-tests.pw.ts +++ b/frontend/e2e/tests/flag-tests.pw.ts @@ -1,9 +1,9 @@ import { test, expect } from '../test-setup'; -import { byId, log, createHelpers, LONG_TIMEOUT } from '../helpers'; +import { byId, log, createHelpers, LONG_TIMEOUT, visualSnapshot } from '../helpers'; import { E2E_USER, PASSWORD, E2E_TEST_PROJECT } from '../config'; test.describe('Flag Tests', () => { - test('Feature flags can be created, toggled, edited, and deleted across environments @oss', async ({ page }) => { + test('Feature flags can be created, toggled, edited, and deleted across environments @oss', async ({ page }, testInfo) => { const { click, createFeature, @@ -19,10 +19,17 @@ test.describe('Flag Tests', () => { waitForElementClickable, waitForElementVisible, waitForFeatureSwitch, + waitForToastsToClear, } = createHelpers(page); log('Login') + await page.goto('/login', { waitUntil: 'domcontentloaded' }) + await page.waitForSelector('[name="email"]', { state: 'visible' }) + await visualSnapshot(page, 'login', testInfo) + await login(E2E_USER, PASSWORD) + await visualSnapshot(page, 'project-list', testInfo) + await gotoProject(E2E_TEST_PROJECT) // Ensure we're on the Development environment @@ -45,6 +52,9 @@ test.describe('Flag Tests', () => { { value: 'small', weight: 0 }, ]}) + await waitForToastsToClear() + await visualSnapshot(page, 'features-list', testInfo) + log('Create Short Life Feature') await createFeature({ name: 'short_life_feature', value: false }) await scrollBy(0, 15000) @@ -92,6 +102,8 @@ test.describe('Flag Tests', () => { await waitForElementVisible(byId('switch-environment-production-active')) await waitForFeatureSwitch('header_enabled', 'off') + await visualSnapshot(page, 'features-list-production', testInfo) + log('Clear down features') // Ensure features list is fully loaded before attempting to delete await waitForFeatureSwitch('header_enabled', 'off') @@ -99,7 +111,7 @@ test.describe('Flag Tests', () => { await deleteFeature('header_enabled') }); - test('Feature flags can have tags added and be archived @oss', async ({ page }) => { + test('Feature flags can have tags added and be archived @oss', async ({ page }, testInfo) => { const { addTagToFeature, archiveFeature, @@ -179,5 +191,7 @@ test.describe('Flag Tests', () => { }).first() await archivedFeature.waitFor({ state: 'visible', timeout: 5000 }) + await visualSnapshot(page, 'features-archived-filter', testInfo) + }); }); diff --git a/frontend/e2e/tests/initialise-tests.pw.ts b/frontend/e2e/tests/initialise-tests.pw.ts index 1354f79c263a..1bc804b660d7 100644 --- a/frontend/e2e/tests/initialise-tests.pw.ts +++ b/frontend/e2e/tests/initialise-tests.pw.ts @@ -4,11 +4,12 @@ import { createHelpers, getFlagsmith, log, + visualSnapshot, } from '../helpers'; import { E2E_SIGN_UP_USER, PASSWORD } from '../config'; test.describe('Signup', () => { - test('Create Organisation and Project @oss', async ({ page }) => { + test('Create Organisation and Project @oss', async ({ page }, testInfo) => { const { addErrorLogging, click, logout, setText, waitForElementVisible } = createHelpers(page); const flagsmith = await getFlagsmith(); @@ -22,6 +23,8 @@ test.describe('Signup', () => { await click(byId('jsSignup')); // Wait for firstName field to be visible after modal opens await waitForElementVisible(byId('firstName')); + await visualSnapshot(page, 'signup-form', testInfo) + await setText(byId('firstName'), 'Bullet'); await setText(byId('lastName'), 'Train'); await setText(byId('email'), E2E_SIGN_UP_USER); @@ -30,6 +33,8 @@ test.describe('Signup', () => { // Wait for navigation and form to load after signup await page.waitForURL(/\/create/, { timeout: 20000 }); await waitForElementVisible('[name="orgName"]'); + await visualSnapshot(page, 'create-organisation', testInfo) + await setText('[name="orgName"]', 'Flagsmith Ltd 0'); await click('#create-org-btn'); diff --git a/frontend/e2e/tests/organisation-test.pw.ts b/frontend/e2e/tests/organisation-test.pw.ts index 16b6de4bc620..348eb2ca92b3 100644 --- a/frontend/e2e/tests/organisation-test.pw.ts +++ b/frontend/e2e/tests/organisation-test.pw.ts @@ -1,9 +1,9 @@ import { test, expect } from '../test-setup'; -import { byId, log, createHelpers, LONG_TIMEOUT } from '../helpers' +import { byId, log, createHelpers, LONG_TIMEOUT, visualSnapshot } from '../helpers' import { E2E_SEPARATE_TEST_USER, PASSWORD } from '../config' test.describe('Organisation Tests', () => { - test('Organisations can be created, renamed, and deleted with name validation @oss', async ({ page }) => { + test('Organisations can be created, renamed, and deleted with name validation @oss', async ({ page }, testInfo) => { const { assertTextContent, click, @@ -41,6 +41,8 @@ test.describe('Organisation Tests', () => { await click(byId('org-settings-link')) await waitForElementVisible("[data-test='organisation-name']") + await visualSnapshot(page, 'organisation-settings', testInfo) + log('Test 2: Create and Delete Organisation, Verify Next Org in Nav') log('Navigate to create organisation') await click(byId('home-link')) diff --git a/frontend/e2e/tests/project-test.pw.ts b/frontend/e2e/tests/project-test.pw.ts index 9c700cf4f7e6..ea32ff071e6f 100644 --- a/frontend/e2e/tests/project-test.pw.ts +++ b/frontend/e2e/tests/project-test.pw.ts @@ -1,9 +1,9 @@ import { test, expect } from '../test-setup'; -import { byId, getFlagsmith, log, createHelpers } from '../helpers'; +import { byId, getFlagsmith, log, createHelpers, visualSnapshot } from '../helpers'; import { E2E_USER, PASSWORD } from '../config' test.describe('Project Tests', () => { - test('Additional Projects can be created and renamed with configurable change request approvals @enterprise', async ({ page }) => { + test('Additional Projects can be created and renamed with configurable change request approvals @enterprise', async ({ page }, testInfo) => { const { assertInputValue, assertTextContent, @@ -27,6 +27,8 @@ test.describe('Project Tests', () => { await click(byId('create-project-btn')) await waitForElementVisible(byId('features-page')) + await visualSnapshot(page, 'new-project-features', testInfo) + log('Edit Project') await click('#project-link') await click('#project-settings-link') diff --git a/frontend/e2e/tests/roles-test.pw.ts b/frontend/e2e/tests/roles-test.pw.ts index 66e00a5f3976..d2c223aca780 100644 --- a/frontend/e2e/tests/roles-test.pw.ts +++ b/frontend/e2e/tests/roles-test.pw.ts @@ -1,5 +1,5 @@ import { test } from '../test-setup'; -import { byId, log, createHelpers } from '../helpers'; +import { byId, log, createHelpers, visualSnapshot } from '../helpers'; import { PASSWORD, E2E_NON_ADMIN_USER_WITH_A_ROLE, @@ -8,7 +8,7 @@ import { } from '../config' test.describe('Roles Tests', () => { - test('Roles can be created with project and environment permissions @enterprise', async ({ page }) => { + test('Roles can be created with project and environment permissions @enterprise', async ({ page }, testInfo) => { const { click, closeModal, @@ -30,6 +30,8 @@ test.describe('Roles Tests', () => { await click(byId('organisation-link')) await click(byId('users-and-permissions')) await waitForElementVisible(byId('tab-item-roles')) + await visualSnapshot(page, 'users-and-permissions', testInfo) + log('Create Role') await createRole('test_role', [4]) log('Add project permissions to the Role') diff --git a/frontend/e2e/tests/segment-test.pw.ts b/frontend/e2e/tests/segment-test.pw.ts index eefd5e61cfe3..3f0455741055 100644 --- a/frontend/e2e/tests/segment-test.pw.ts +++ b/frontend/e2e/tests/segment-test.pw.ts @@ -1,5 +1,5 @@ import { test, expect } from '../test-setup'; -import { byId, log, createHelpers } from '../helpers'; +import { byId, log, createHelpers, visualSnapshot } from '../helpers'; import { E2E_USER, PASSWORD, E2E_TEST_IDENTITY, E2E_SEGMENT_PROJECT_1, E2E_SEGMENT_PROJECT_2, E2E_SEGMENT_PROJECT_3 } from '../config' const REMOTE_CONFIG_FEATURE = 'remote_config' @@ -53,7 +53,7 @@ const segmentRules = [ }, ] -test('Segment test 1 - Create, update, and manage segments with multivariate flags @oss', async ({ page }) => { +test('Segment test 1 - Create, update, and manage segments with multivariate flags @oss', async ({ page }, testInfo) => { const { addSegmentOverride, assertInputValue, @@ -114,11 +114,16 @@ test('Segment test 1 - Create, update, and manage segments with multivariate fla await assertInputValue(byId(`rule-${0}-value-0`), `${lastRule.value + 1}`) await deleteSegmentFromPage('segment_to_update') + await waitForToastsToClear() + await visualSnapshot(page, 'segments-list', testInfo) + log('Create segment') await createSegment('18_or_19', segmentRules) log('Add segment trait for user') await gotoTraits(E2E_TEST_IDENTITY) + await visualSnapshot(page, 'identity-traits', testInfo) + await createTrait('age', 18) // Wait for trait to be applied and feature values to load @@ -276,7 +281,7 @@ test('Segment test 2 - Test segment priority and overrides @oss', async ({ page await deleteFeature('config') }) -test('Segment test 3 - Test user-specific feature overrides @oss', async ({ page }) => { +test('Segment test 3 - Test user-specific feature overrides @oss', async ({ page }, testInfo) => { const { assertUserFeatureValue, click, @@ -337,6 +342,8 @@ test('Segment test 3 - Test user-specific feature overrides @oss', async ({ page const valueInList = identityRow.locator('.table-column').filter({ hasText: 'small' }) await expect(valueInList).toBeVisible() + await visualSnapshot(page, 'feature-identity-overrides', testInfo) + log('Close modal') await closeModal() diff --git a/frontend/e2e/tests/versioning-tests.pw.ts b/frontend/e2e/tests/versioning-tests.pw.ts index ff91d3786338..dbf2248b894e 100644 --- a/frontend/e2e/tests/versioning-tests.pw.ts +++ b/frontend/e2e/tests/versioning-tests.pw.ts @@ -4,10 +4,11 @@ import { getFlagsmith, log, createHelpers, + visualSnapshot, } from '../helpers'; import { E2E_USER, PASSWORD } from '../config'; -test('Versioning tests - Create, edit, and compare feature versions @oss', async ({ page }) => { +test('Versioning tests - Create, edit, and compare feature versions @oss', async ({ page }, testInfo) => { const { assertNumberOfVersions, click, @@ -21,6 +22,7 @@ test('Versioning tests - Create, edit, and compare feature versions @oss', async tryItExpect, waitForElementVisible, waitForFeatureSwitch, + waitForToastsToClear, } = createHelpers(page) const flagsmith = await getFlagsmith() const hasFeature = flagsmith.hasFeature("feature_versioning") @@ -41,6 +43,10 @@ test('Versioning tests - Create, edit, and compare feature versions @oss', async await click('#confirm-btn-yes') // Feature versioning takes up to a minute to enable on the backend await waitForElementVisible(byId('feature-versioning-enabled')) + await waitForElementVisible('.toast-message') + await waitForToastsToClear() + + await visualSnapshot(page, 'versioning-enabled', testInfo) log('Create feature 1') await createRemoteConfig({ name: 'a', value: 'small' }) diff --git a/frontend/package.json b/frontend/package.json index 27106b6514ee..af97deba2c2c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,6 +17,9 @@ "test:unit": "jest", "test:unit:watch": "jest --watch", "test:unit:coverage": "jest --coverage", + "test:visual": "cross-env VISUAL_REGRESSION=1 npx playwright test", + "test:visual:compare": "npx tsx e2e/compare-visual-regression.ts && npx playwright test -c playwright.visual.config.ts", + "test:visual:report": "npx playwright show-report e2e/visual-regression-report", "test:report": "npx playwright show-report e2e/playwright-report", "env": "node ./bin/env.js", "lint": "eslint .", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index eeeaf1387fe0..d15c63c56d11 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ maxFailures: process.env.E2E_RETRIES === '0' ? 1 : undefined, /* Output directory for test results */ outputDir: './e2e/test-results', + /* Configure projects for major browsers */ projects: [ { @@ -51,6 +52,7 @@ export default defineConfig({ }, }, ], + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: [ [ @@ -65,9 +67,15 @@ export default defineConfig({ ['list', { printSteps: false }], // Only shows test names with pass/fail status ['./e2e/failed-tests-reporter.ts'], // Writes failed.json for CI ], + /* Retry on CI only */ retries: process.env.CI ? 2 : 0, + + /* Visual regression baselines directory — used by the comparison test */ + snapshotPathTemplate: './e2e/visual-regression-snapshots/{arg}{ext}', + testDir: './e2e', + testIgnore: /.*_visual-regression-compare.*/, testMatch: /.*\.pw\.ts$/, /* Test timeout */ timeout: 300000, diff --git a/frontend/playwright.visual.config.ts b/frontend/playwright.visual.config.ts new file mode 100644 index 000000000000..fed0f82ca74f --- /dev/null +++ b/frontend/playwright.visual.config.ts @@ -0,0 +1,23 @@ +import { defineConfig } from '@playwright/test' + +/** + * Minimal Playwright config for the visual regression comparison step. + * No global setup, no web server, no browser — just PNG comparison. + */ +export default defineConfig({ + reporter: [ + [ + 'html', + { + open: 'never', + outputFolder: + process.env.PLAYWRIGHT_HTML_REPORT || + './e2e/visual-regression-report', + }, + ], + ['list'], + ], + snapshotPathTemplate: './e2e/visual-regression-snapshots/{arg}{ext}', + testDir: './e2e', + testMatch: /.*_visual-regression-compare\.pw\.ts$/, +}) diff --git a/frontend/web/components/EnvironmentSelect.tsx b/frontend/web/components/EnvironmentSelect.tsx index 0872cf2729b8..bbf07bd8d2cb 100644 --- a/frontend/web/components/EnvironmentSelect.tsx +++ b/frontend/web/components/EnvironmentSelect.tsx @@ -41,6 +41,7 @@ const EnvironmentSelect: FC = ({ } return true }) + .sort((a, b) => a.label.localeCompare(b.label)) }, [data?.results, ignore, idField]) const foundValue = useMemo(