chore: Cypress visual regression + HTML report

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
 **Merge request checklist**
 
--   [ ] All lints and tests pass. If needed, new unit tests were added.
+-   [ ] All lints and tests pass. If needed, new tests were added.
 -   [ ] If needed, the [documentation](https://github.com/mistic100/Photo-Sphere-Viewer/tree/main/docs) has been updated.

.github/workflows/build.yml

@@ -24,10 +24,11 @@ jobs:
       - name: build
         run: yarn ci:build
 
-      - uses: dorny/test-reporter@1a288b62f8b75c0f433cbfdbc2e4800fbae50bd7
+      - name: test summary
+        uses: dorny/test-reporter@1a288b62f8b75c0f433cbfdbc2e4800fbae50bd7
         if: ${{ (success() || failure()) && github.repository == 'mistic100/Photo-Sphere-Viewer' }}
         with:
-          name: mocha
+          name: build
           use-actions-summary: true
           path: 'packages/*/reports/mocha.json'
           reporter: mocha-json
@@ -49,10 +50,18 @@ jobs:
           yarn ci:build
           yarn ci:e2e
 
-      - uses: dorny/test-reporter@1a288b62f8b75c0f433cbfdbc2e4800fbae50bd7
+      - name: test report
+        uses: actions/upload-artifact@v4
         if: ${{ (success() || failure()) && github.repository == 'mistic100/Photo-Sphere-Viewer' }}
         with:
-          name: cypress
+          name: e2e-report-${{ github.ref_name }}-$SHORT_SHA
+          path: cypress/reports/html
+
+      - name: test summary
+        uses: dorny/test-reporter@1a288b62f8b75c0f433cbfdbc2e4800fbae50bd7
+        if: ${{ (success() || failure()) && github.repository == 'mistic100/Photo-Sphere-Viewer' }}
+        with:
+          name: e2e
           use-actions-summary: true
           path: cypress/reports/e2e.json
           reporter: mocha-json
@@ -67,5 +76,5 @@ jobs:
       - name: setup
         uses: ./.github/workflows/shared/setup
 
-      - name: build
+      - name: build-doc
         run: yarn ci:build-doc

.github/workflows/pre-release.yml

@@ -7,6 +7,9 @@ on:
       version:
         description: Version
         required: true
+      tag:
+        description: NPM tag
+        required: true
 
 jobs:
   release:
@@ -17,25 +20,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v4
-        with:
-          node-version: '20'
-
-      - name: data
-        id: data
-        run: |
-          echo "NPM_TAG=$(echo ${{ github.event.inputs.version }} | cut -d '-' -f2 -s | cut -d '.' -f1 -s)" >> $GITHUB_OUTPUT
-          echo "YARN_CACHE_DIR=$(yarn cache dir)" >> $GITHUB_OUTPUT
-
-      - name: yarn cache ro
-        uses: actions/cache/restore@v4
+      - name: setup
+        uses: ./.github/workflows/shared/setup
         with:
-          key: yarn-${{ hashFiles('yarn.lock') }}
-          path: ${{ steps.data.outputs.YARN_CACHE_DIR }}
+          turbo-cache: false
 
       - name: build
         run: |
-          yarn install --frozen-lockfile
           yarn ci:version ${{ github.event.inputs.version }}
           yarn ci:build
 
@@ -44,4 +35,4 @@ jobs:
           yarn ci:publish
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
-          NPM_TAG: ${{ steps.data.outputs.NPM_TAG || 'alpha' }}
+          NPM_TAG: ${{ github.event.inputs.tag }}

.github/workflows/shared/setup/action.yml

@@ -12,25 +12,25 @@ runs:
       with:
         node-version: '20'
 
-    - name: data
-      id: data
+    - name: env
       shell: bash
       run: |
-        echo "YARN_CACHE_DIR=$(yarn cache dir)" >> $GITHUB_OUTPUT
+        echo "YARN_CACHE_DIR=$(yarn cache dir)" >> $GITHUB_ENV
+        echo "SHORT_SHA=`echo ${{ github.sha }} | cut -c1-8`" >> $GITHUB_ENV
 
     - name: yarn cache
       uses: actions/cache@v4
       if: ${{ !startsWith(github.ref_name, 'dependabot') }}
       with:
         key: yarn-${{ hashFiles('yarn.lock') }}
-        path: ${{ steps.data.outputs.YARN_CACHE_DIR }}
+        path: $YARN_CACHE_DIR
 
     - name: yarn cache ro
       uses: actions/cache/restore@v4
       if: ${{ startsWith(github.ref_name, 'dependabot') }}
       with:
         key: yarn-${{ hashFiles('yarn.lock') }}
-        path: ${{ steps.data.outputs.YARN_CACHE_DIR }}
+        path: $YARN_CACHE_DIR
 
     - name: turbo cache
       uses: actions/cache@v4

.gitignore

@@ -9,4 +9,5 @@ dist
 /public
 reports
 cypress/screenshots
+cypress/snapshots/diff
 .tmp

build/liveserver.mjs

@@ -1,9 +1,9 @@
 import liveServer from 'alive-server';
-import path from 'path';
-import os from 'os';
 import fs from 'fs';
-import { fileURLToPath } from 'url';
+import os from 'os';
+import path from 'path';
 import selfsigned from 'selfsigned';
+import { fileURLToPath } from 'url';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);

build/mocha-reporter.js

@@ -1,6 +1,5 @@
 const BaseReporter = require('mocha/lib/reporters/base');
 const SpecReporter = require('mocha/lib/reporters/spec');
-const JsonReporter = require('mocha/lib/reporters/json');
 const path = require('path');
 const fs = require('fs');
 
@@ -11,19 +10,31 @@ module.exports = class MultiReporter extends BaseReporter {
         new SpecReporter(runner, {});
 
         if (options.reporterOptions?.cypress) {
+            const MochawesomeReporter = require('cypress-mochawesome-reporter/lib/reporter');
+            this.mochawesome = new MochawesomeReporter(runner, {});
+
             new CypressJsonReporter(runner, {
                 reporterOption: {
                     output: 'cypress/reports/e2e.json',
                 },
             });
         } else {
+            const JsonReporter = require('mocha/lib/reporters/json');
             new JsonReporter(runner, {
                 reporterOption: {
                     output: 'reports/mocha.json',
                 },
             });
         }
     }
+
+    done(failures, exit) {
+        if (this.mochawesome) {
+            this.mochawesome.done(failures, exit);
+        } else {
+            exit(failures);
+        }
+    }
 };
 
 // custom json reporter for cypress (for some reason, calling JsonReporter does not work)

cypress.config.ts

@@ -1,4 +1,7 @@
 import { defineConfig } from 'cypress';
+import { configureVisualRegression } from 'cypress-visual-regression';
+// @ts-ignore
+import  cypressMochawesomeReporterPlugin from 'cypress-mochawesome-reporter/plugin';
 
 export default defineConfig({
     e2e: {
@@ -7,7 +10,13 @@ export default defineConfig({
         baseUrl: 'https://127.0.0.1:8080',
         scrollBehavior: false,
         screenshotOnRunFailure: false,
+        env: {
+          visualRegressionType: 'regression',
+        },
         setupNodeEvents(on) {
+            configureVisualRegression(on);
+            cypressMochawesomeReporterPlugin(on);
+
             on('before:browser:launch', (browser, launchOptions) => {
                 // should be bigger than the largest viewport used + browser UI elements
                 const width = 1600;

cypress/e2e/navbar.cy.ts

@@ -7,11 +7,18 @@ describe('navbar', () => {
         waitViewerReady();
     });
 
+    it('should have a navbar', () => {
+        cy.get('.psv-navbar')
+            .should('be.visible')
+            .compareSnapshot('base');
+    });
+
     it('should have a custom button', () => {
         const alertStub = cy.stub();
         cy.on('window:alert', alertStub);
 
-        cy.get('.custom-button:eq(0)').click()
+        cy.get('.custom-button:eq(0)')
+            .click()
             .then(() => {
                 expect(alertStub.getCall(0)).to.be.calledWith('Custom button clicked');
             });
@@ -24,6 +31,8 @@ describe('navbar', () => {
 
         cy.get('.psv-caption-content').should('have.text', 'Name: Lorem Ipsum');
 
+        cy.get('.psv-navbar').compareSnapshot('update-caption');
+
         callViewer('change caption via API', viewer => viewer.navbar.setCaption('Loading...'));
 
         cy.get('.psv-caption-content').should('have.text', 'Loading...');
@@ -37,7 +46,8 @@ describe('navbar', () => {
         cy.get('.psv-panel')
             .should('be.visible')
             .should('include.text', 'Parc national du Mercantour © Damien Sorel')
-            .should('include.text', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit');
+            .should('include.text', 'Lorem ipsum dolor sit amet, consectetur adipiscing elit')
+            .compareSnapshot('description');
     });
 
     // does not work in headless mode
@@ -60,11 +70,14 @@ describe('navbar', () => {
 
         cy.get('.psv-caption-content').should('not.be.visible');
 
+        cy.get('.psv-navbar').compareSnapshot('no-caption');
+
         cy.get('.psv-description-button').click();
 
-        cy.get('.psv-notification')
+        cy.get('.psv-notification-content')
             .should('be.visible')
-            .should('have.text', 'Parc national du Mercantour © Damien Sorel');
+            .should('have.text', 'Parc national du Mercantour © Damien Sorel')
+            .compareSnapshot('caption-notification');
 
         cy.get('.psv-description-button').click();
 
@@ -95,6 +108,8 @@ describe('navbar', () => {
             cy.get(visible).should('be.visible');
         });
 
+        cy.get('.psv-navbar').compareSnapshot('with-menu');
+
         cy.get('.psv-menu-button').click();
 
         cy.get('.psv-panel')
@@ -104,7 +119,8 @@ describe('navbar', () => {
 
                 cy.contains('Download').should('be.visible');
                 cy.contains('Click me').should('be.visible');
-            });
+            })
+            .compareSnapshot('menu-content');
 
         cy.get('.psv-panel-close-button').click();
 
@@ -167,20 +183,22 @@ describe('navbar', () => {
     });
 
     it('should update the buttons', () => {
-        function assertButtons(titles: string[]) {
+        function assertButtons(expected: string[]) {
             cy.get('.psv-button').then($buttons => {
                 const titles = $buttons
                     .filter(':visible')
                     .map((i, btn) => btn.getAttribute('title'))
                     .get();
-                expect(titles).to.have.members(titles);
+                expect(titles).to.have.members(expected);
             });
         }
 
         callViewer('change buttons via options', viewer => viewer.setOption('navbar', 'zoom move'));
 
         assertButtons(['Zoom out', 'Zoom in', 'Move left', 'Move right', 'Move up', 'Move down']);
 
+        cy.get('.psv-navbar').compareSnapshot('update-buttons');
+
         callViewer('change buttons via API', viewer => viewer.navbar.setButtons(['download', 'fullscreen']));
 
         assertButtons(['Download', 'Fullscreen']);
@@ -191,6 +209,8 @@ describe('navbar', () => {
 
         cy.get('.psv-fullscreen-button').should('not.be.visible');
 
+        cy.get('.psv-navbar').compareSnapshot('hide-button');
+
         callViewer('show fullscreen button', viewer => viewer.navbar.getButton('fullscreen').show());
 
         cy.get('.psv-fullscreen-button').should('be.visible');
@@ -201,6 +221,8 @@ describe('navbar', () => {
 
         cy.get('.psv-download-button').should('have.class', 'psv-button--disabled');
 
+        cy.get('.psv-navbar').compareSnapshot('disable-button');
+
         callViewer('enable download button', viewer => viewer.navbar.getButton('download').enable());
 
         cy.get('.psv-download-button').should('not.have.class', 'psv-button--disabled');
@@ -223,5 +245,7 @@ describe('navbar', () => {
                 cy.get('#title').should('have.text', 'Custom element');
                 cy.get('#value').should('have.text', '50');
             });
+
+        cy.get('.psv-custom-button').compareSnapshot('custom-element');
     });
 });

cypress/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "photo-sphere-viewer-e2e",
+  "version": "0.0.0",
+  "private": true,
+  "license": "MIT",
+  "scripts": {
+    "lint": "eslint . --fix"
+  }
+}

cypress/support/e2e.ts

@@ -1 +1,26 @@
 import 'cypress-real-events';
+import 'cypress-mochawesome-reporter/register';
+import { addCompareSnapshotCommand } from 'cypress-visual-regression/dist/command';
+
+addCompareSnapshotCommand({
+    errorThreshold: 0.1,
+    failSilently: !Cypress.config('isInteractive'),
+});
+
+if (!Cypress.config('isInteractive')) {
+    Cypress.Commands.overwrite('compareSnapshot', (originalFn, name, options = {}) => {
+        return originalFn(name, options)
+            .then(result => {
+                if (result.images.diff) {
+                    // @ts-ignore
+                    Cypress.Mochawesome.context.push({ title: 'Visual regression diff', value: 'data:image/png;base64,' + result.images.diff });
+                }
+
+                if (result.error) {
+                    throw new Error(result.error);
+                }
+
+                return result;
+            });
+    });
+}

cypress/tsconfig.json

@@ -3,7 +3,8 @@
         "types": [
             "cypress",
             "node",
-            "cypress-real-events"
+            "cypress-real-events",
+            "cypress-mochawesome-reporter"
         ]
     }
 }

cypress/utils/index.ts

@@ -14,3 +14,7 @@ export function callViewer(log: string, cb: (viewer: Viewer) => void) {
     cy.log(`Viewer: ${log}`);
     cy.window({ log: false }).its('viewer', { log: false }).then(cb);
 }
+
+export function createBaseSnapshot() {
+    Cypress.env('visualRegressionType', 'base');
+}