Convert suite settings (coins, custom-firmware) + refactors desktop p…

…ipeline (#15869)

* feat(e2e): Convert coins.test.ts from cy to pw

minor refactoring
some verification commented out and will be part of future refactoring

* feat(e2e): Convert cy test file custom-firmware.test.ts to PW

* fix(e2e): Stabilizing new pw tests for desktop

adds logic to use correct API url based on web/desktop/local/CI
removes flakyness of Coins test

* refactor(e2e): #15868 Refactor desktop e2e test pipeline

better matrix
move docker pull from tests step to improve readability
pull only what is need
Attach playwright report for devs who dont have currents
add --forbid-only to both pipelines

* fix(e2e): Fix start and name of electron-regtest

.github/workflows/test-suite-desktop-e2e.yml

@@ -27,78 +27,87 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - TEST_FILE: "spawn-bridge spawn-bridge-daemon suite-guide wallet-discovery"
-            CONTAINERS: "trezor-user-env-unix"
-          - TEST_FILE: "electrum"
+          # - TEST_GROUP: "@group=suite"
+          #   CONTAINERS: "trezor-user-env-unix"
+          # - TEST_GROUP: "@group=device-management"
+          #   CONTAINERS: "trezor-user-env-unix"
+          - TEST_GROUP: "@group=settings"
             CONTAINERS: "trezor-user-env-unix electrum-regtest"
+          # - TEST_GROUP: "@group=metadata"
+          #   CONTAINERS: "trezor-user-env-unix"
+          # - TEST_GROUP: "@group=passphrase"
+          #   CONTAINERS: "trezor-user-env-unix"
+          # - TEST_GROUP: "@group=other"
+          #   CONTAINERS: "trezor-user-env-unix"
+          - TEST_GROUP: "@group=wallet"
+            CONTAINERS: "trezor-user-env-unix"
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Run and store git values for Currents
-        run: |
-          echo "COMMIT_MESSAGE=$(git show -s --pretty=%s)" >> $GITHUB_ENV
-          echo "COMMIT_EMAIL=$(git show -s --pretty=%ae)" >> $GITHUB_ENV
-          echo "COMMIT_AUTHOR=$(git show -s --pretty=%an)" >> $GITHUB_ENV
-          echo "COMMIT_SHA=$(git show -s --pretty=%H)" >> $GITHUB_ENV
-          echo "COMMIT_TIMESTAMP=$(git show -s --pretty=%ct)" >> $GITHUB_ENV
-
       - name: Setup node
         uses: actions/setup-node@v4
         with:
           node-version-file: ".nvmrc"
           cache: yarn
 
-      - name: Install deps and build libs
+      - name: Install dependencies, build libs and pull docker images
+        env:
+          COMPOSE_FILE: ./docker/docker-compose.suite-desktop-ci.yml
         run: |
           yarn install --immutable
           yarn message-system-sign-config
           yarn workspace @trezor/suite-data build:lib
           yarn workspace @trezor/transport-bridge build:lib
+          docker compose pull  ${{ matrix.CONTAINERS }}
 
-      - name: Build app.js for tests
+      - name: Build electron app.js for tests
         run: |
           yarn workspace @trezor/suite-desktop build:app
           yarn workspace @trezor/suite-desktop build:ui
 
-      - name: Run e2e desktop tests
+      - name: Run Playwright e2e desktop tests
         env:
-          GITHUB_WORKFLOW: ${{github.workflow}}
+          COMPOSE_FILE: ./docker/docker-compose.suite-desktop-ci.yml
           GITHUB_ACTION: true
-          GITHUB_EVENT_NAME: ${{github.event_name}}
-          GITHUB_RUN_ID: ${{github.run_id}}
-          GITHUB_RUN_ATTEMPT: ${{github.run_attempt}}
-          GITHUB_REPOSITORY: ${{github.repository}}
-          COMMIT_INFO_BRANCH: ${{github.head_ref}}
-          COMMIT_INFO_MESSAGE: ${{env.COMMIT_MESSAGE}}
-          COMMIT_INFO_EMAIL: ${{env.COMMIT_EMAIL}}
-          COMMIT_INFO_AUTHOR: ${{env.COMMIT_AUTHOR}}
-          COMMIT_INFO_SHA: ${{env.COMMIT_SHA}}
-          COMMIT_INFO_TIMESTAMP: ${{env.COMMIT_TIMESTAMP}}
-          COMMIT_INFO_REMOTE: ${{github.repository}}
           CURRENTS_PROJECT_ID: 4ytF0E
           CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
           CURRENTS_CI_BUILD_ID: pr-run-${{github.run_id}}
-          COMPOSE_FILE: ./docker/docker-compose.suite-desktop-ci.yml
-          TEST_FILE: ${{ matrix.TEST_FILE }}
         run: |
-          docker compose pull
           docker compose up -d ${{ matrix.CONTAINERS }}
-          yarn workspace @trezor/suite-desktop-core test:e2e:desktop ${{ env.TEST_FILE }}
+          echo "Starting Playwright Desktop test group ${{ matrix.TEST_GROUP }}"
+          yarn workspace @trezor/suite-desktop-core test:e2e:desktop --forbid-only --grep=${{ matrix.TEST_GROUP }}
 
-      - name: cleanup
+      - name: Extract Trezor-user-env logs
+        if: ${{ ! cancelled() }}
+        run: |
+          docker cp docker-trezor-user-env-unix-1:/trezor-user-env/logs/debugging.log trezor-user-env-debugging.log || true
+          docker cp docker-trezor-user-env-unix-1:/trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log || true
+          docker cp docker-trezor-user-env-unix-1:/trezor-user-env/docker/version.txt trezor-user-env-version.txt || true
+
+      - name: Upload Trezor-user-env logs
+        if: ${{ ! cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: emulator-logs-${{ matrix.TEST_GROUP }}
+          path: |
+            trezor-user-env-debugging.log
+            tenv-emulator-bridge-debugging.log
+            trezor-user-env-version.txt
+          retention-days: 30
+
+      - name: Docker compose down
         env:
           COMPOSE_FILE: ./docker/docker-compose.suite-desktop-ci.yml
         run: docker compose down
 
-      # TODO: currently only uploads trace.zip, figure out why screens are not uploaded
-      - name: Upload artifacts
+      - name: Upload Playwright report
         if: ${{ ! cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-          name: test-artifacts-${{ matrix.TEST_FILE }}
-          path: |
-            ./packages/suite-desktop-core/e2e/test-results
+          name: playwright-report-${{ matrix.TEST_GROUP }}
+          path: ./packages/suite-desktop-core/playwright-report/
+          retention-days: 30

.github/workflows/test-suite-web-e2e-pw.yml

@@ -107,12 +107,14 @@ jobs:
           #   CONTAINERS: "trezor-user-env-unix"
           - TEST_GROUP: "@group=wallet"
             CONTAINERS: "trezor-user-env-unix bitcoin-regtest"
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{github.event.after}}
           fetch-depth: 2
+
       - name: Setup node
         uses: actions/setup-node@v4
         with:
@@ -124,16 +126,6 @@ jobs:
         run: |
           echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
 
-      - name: Extract commit message
-        id: extract_commit_message
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            git fetch origin +refs/pull/${{ github.event.pull_request.number }}/merge: --depth=1
-            echo "message=$(git log --no-merges -1 --pretty=format:"%s")" >> $GITHUB_OUTPUT
-          else
-            echo "message=$(git log --no-merges -1 --pretty=format:"%s")" >> $GITHUB_OUTPUT
-          fi
-
       - name: Install dependencies and pull docker images
         env:
           COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml
@@ -144,21 +136,18 @@ jobs:
           npx playwright install --with-deps chromium
           docker compose pull ${{ matrix.CONTAINERS }}
 
-      - name: Run Playwright e2e tests
+      - name: Run Playwright e2e web tests
         env:
           COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml
           BASE_URL: https://dev.suite.sldev.cz/suite-web/${{ steps.extract_branch.outputs.branch }}/web/
-          ## Keep retries enabled once stabilized
-          ALLOW_RETRY: true
-          TEST_GROUP: ${{ matrix.TEST_GROUP }}
           GITHUB_ACTION: true
           CURRENTS_PROJECT_ID: Og0NOQ
           CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
           CURRENTS_CI_BUILD_ID: pr-run-${{github.run_id}}
         run: |
           docker compose up -d ${{ matrix.CONTAINERS }}
-          echo "Starting Playwright Web test group ${TEST_GROUP}"
-          yarn workspace @trezor/suite-desktop-core test:e2e:web --grep=${TEST_GROUP}
+          echo "Starting Playwright Web test group ${{ matrix.TEST_GROUP }}"
+          yarn workspace @trezor/suite-desktop-core test:e2e:web --forbid-only --grep=${{ matrix.TEST_GROUP }}
 
       - name: Extract Trezor-user-env logs
         if: ${{ ! cancelled() }}
@@ -176,6 +165,12 @@ jobs:
             trezor-user-env-debugging.log
             tenv-emulator-bridge-debugging.log
             trezor-user-env-version.txt
+          retention-days: 30
+
+      - name: Docker compose down
+        env:
+          COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml
+        run: docker compose down
 
       - name: Upload Playwright report
         if: ${{ ! cancelled() }}

packages/suite-desktop-core/e2e/playwright.config.ts

@@ -5,14 +5,16 @@ export enum PlaywrightProjects {
     Web = 'web',
     Desktop = 'desktop',
 }
+const timeoutCIRun = 1000 * 60;
+const timeoutLocalRun = 1000 * 30;
 
 const config: PlaywrightTestConfig = {
     projects: [
         {
             name: PlaywrightProjects.Web,
             use: {
                 browserName: 'chromium',
-                baseURL: process.env.BASE_URL || 'http://localhost:8000',
+                baseURL: process.env.BASE_URL || 'http://localhost:8000/',
             },
             grepInvert: /@desktopOnly/,
         },
@@ -34,7 +36,7 @@ const config: PlaywrightTestConfig = {
     reporter: process.env.GITHUB_ACTION
         ? [['list'], ['@currents/playwright'], ['html', { open: 'never' }]]
         : [['list'], ['html', { open: 'never' }]],
-    timeout: 1000 * 60 * 5,
+    timeout: process.env.GITHUB_ACTION ? timeoutCIRun : timeoutLocalRun,
     outputDir: path.join(__dirname, 'test-results'),
 };
 

packages/suite-desktop-core/e2e/support/common.ts

@@ -1,11 +1,13 @@
 /* eslint-disable no-console */
 
-import { _electron as electron } from '@playwright/test';
+import { _electron as electron, TestInfo } from '@playwright/test';
 import path from 'path';
 import fse from 'fs-extra';
 
 import { TrezorUserEnvLink } from '@trezor/trezor-user-env-link';
 
+import { PlaywrightProjects } from '../playwright.config';
+
 // specific version of legacy bridge is requested & expected
 export const LEGACY_BRIDGE_VERSION = '2.0.33';
 const disableHashCheckArgument = '--state.suite.settings.isFirmwareHashCheckDisabled=true';
@@ -100,3 +102,14 @@ export const launchSuite = async (params: LaunchSuiteParams = {}) => {
 
     return { electronApp, window };
 };
+
+export const getApiUrl = (webBaseUrl: string | undefined, testInfo: TestInfo) => {
+    const electronApiURL = 'file:///';
+    const apiURL =
+        testInfo.project.name === PlaywrightProjects.Desktop ? electronApiURL : webBaseUrl;
+    if (!apiURL) {
+        throw new Error('apiURL is not defined');
+    }
+
+    return apiURL;
+};

packages/suite-desktop-core/e2e/support/customMatchers.ts

@@ -0,0 +1,26 @@
+import { Locator, expect as baseExpect } from '@playwright/test';
+
+export const expect = baseExpect.extend({
+    async toBeEnabledCoin(locator: Locator) {
+        const isActive = await locator.getAttribute('data-active');
+
+        return {
+            pass: isActive === 'true',
+            message: () =>
+                isActive === null
+                    ? `expected ${locator} to have attribute 'data-active', but it does not have this attribute at all`
+                    : `expected ${locator} to have attribute 'data-active' set to 'true', but got '${isActive}'`,
+        };
+    },
+    async toBeDisabledCoin(locator: Locator) {
+        const isActive = await locator.getAttribute('data-active');
+
+        return {
+            pass: isActive === 'false',
+            message: () =>
+                isActive === null
+                    ? `expected ${locator} to have attribute 'data-active', but it does not have this attribute at all`
+                    : `expected ${locator} to have attribute 'data-active' set to 'false', but got '${isActive}'`,
+        };
+    },
+});

packages/suite-desktop-core/e2e/support/fixtures.ts

@@ -1,5 +1,4 @@
 /* eslint-disable react-hooks/rules-of-hooks */
-
 import { test as base, ElectronApplication, Page } from '@playwright/test';
 
 import {
@@ -10,14 +9,15 @@ import {
 } from '@trezor/trezor-user-env-link';
 
 import { DashboardActions } from './pageActions/dashboardActions';
-import { launchSuite } from './common';
+import { getApiUrl, launchSuite } from './common';
 import { SettingsActions } from './pageActions/settingsActions';
 import { SuiteGuide } from './pageActions/suiteGuideActions';
 import { WalletActions } from './pageActions/walletActions';
 import { OnboardingActions } from './pageActions/onboardingActions';
 import { PlaywrightProjects } from '../playwright.config';
 
 type Fixtures = {
+    apiURL: string;
     startEmulator: boolean;
     emulatorStartConf: StartEmu;
     emulatorSetupConf: SetupEmu;
@@ -35,6 +35,9 @@ const test = base.extend<Fixtures>({
     startEmulator: true,
     emulatorStartConf: {},
     emulatorSetupConf: {},
+    apiURL: async ({ baseURL }, use, testInfo) => {
+        await use(getApiUrl(baseURL, testInfo));
+    },
     /* eslint-disable-next-line no-empty-pattern */
     trezorUserEnvLink: async ({}, use) => {
         await use(TrezorUserEnvLink);
@@ -115,4 +118,4 @@ const test = base.extend<Fixtures>({
 });
 
 export { test };
-export { expect } from '@playwright/test';
+export { expect } from './customMatchers';

packages/suite-desktop-core/e2e/support/pageActions/settingsActions.ts

@@ -1,7 +1,9 @@
-import { expect, Locator, Page } from '@playwright/test';
+import { Locator, Page } from '@playwright/test';
 
 import { BackendType, NetworkSymbol } from '@suite-common/wallet-config';
 
+import { expect } from '../customMatchers';
+
 export class SettingsActions {
     private readonly window: Page;
     private readonly TIMES_CLICK_TO_SET_DEBUG_MODE = 5;
@@ -56,7 +58,7 @@ export class SettingsActions {
 
     async navigateTo() {
         await this.settingsMenuButton.click();
-        await expect(this.settingsHeader).toHaveText('Settings');
+        await expect(this.settingsHeader).toHaveText('Settings', { timeout: 10000 });
     }
 
     async toggleDebugModeInSettings() {
@@ -79,12 +81,12 @@ export class SettingsActions {
 
     async enableCoin(coin: NetworkSymbol) {
         await this.coinNetworkButton(coin).click();
-        await expect(this.coinNetworkButton(coin)).toHaveAttribute('data-active', 'true');
+        await expect(this.coinNetworkButton(coin)).toBeEnabledCoin();
     }
 
     async disableCoin(coin: NetworkSymbol) {
         await this.coinNetworkButton(coin).click();
-        await expect(this.coinNetworkButton(coin)).toHaveAttribute('data-active', 'false');
+        await expect(this.coinNetworkButton(coin)).toBeDisabledCoin();
     }
 
     async changeCoinBackend(backend: BackendType, backendUrl: string) {