From c9c84cfd4a459485068834ab61e830b86535bbd5 Mon Sep 17 00:00:00 2001 From: Martin Vere Cihlar Date: Tue, 3 Dec 2024 08:49:35 +0100 Subject: [PATCH] feat(e2e): Debuging PW web tests on CI --- .github/workflows/test-suite-web-e2e-pw.yml | 82 ++++++++----------- .gitignore | 1 + docker/docker-compose.suite-ci-pw.yml | 13 +++ docker/docker-compose.suite-ci.yml | 35 ++++++-- .../e2e/playwright.config.ts | 8 +- .../e2e/support/fixtures.ts | 3 +- .../tests/general/cardano-discovery.test.ts | 26 +++--- .../e2e/tests/general/eap-modal.test.ts | 2 +- .../e2e/tests/general/suite-guide.test.ts | 60 +++++++------- packages/suite-desktop-core/eslint.config.mjs | 3 + 10 files changed, 135 insertions(+), 98 deletions(-) create mode 100644 docker/docker-compose.suite-ci-pw.yml diff --git a/.github/workflows/test-suite-web-e2e-pw.yml b/.github/workflows/test-suite-web-e2e-pw.yml index c6bd02a73d2f..0c93da0c83c4 100644 --- a/.github/workflows/test-suite-web-e2e-pw.yml +++ b/.github/workflows/test-suite-web-e2e-pw.yml @@ -95,25 +95,18 @@ jobs: include: # - TEST_GROUP: "@group=suite" # CONTAINERS: "trezor-user-env-unix" - # CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" # - TEST_GROUP: "@group=device-management" # CONTAINERS: "trezor-user-env-unix" - # CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" - TEST_GROUP: "@group=settings" CONTAINERS: "trezor-user-env-unix" - CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" # - TEST_GROUP: "@group=metadata" # CONTAINERS: "trezor-user-env-unix" - # CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" # - TEST_GROUP: "@group=passphrase" # CONTAINERS: "trezor-user-env-unix" - # CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" # - TEST_GROUP: "@group=other" # CONTAINERS: "trezor-user-env-unix" - # CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" - TEST_GROUP: "@group=wallet" CONTAINERS: "trezor-user-env-unix bitcoin-regtest" - CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1" steps: - name: Checkout uses: actions/checkout@v4 @@ -141,56 +134,53 @@ jobs: echo "message=$(git log --no-merges -1 --pretty=format:"%s")" >> $GITHUB_OUTPUT fi - - name: Install dependencies + - name: Install dependencies and pull docker images + env: + COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml run: | echo -e "\nenableScripts: false" >> .yarnrc.yml echo -e "\nenableHardenedMode: false" >> .yarnrc.yml yarn workspaces focus @trezor/suite-desktop-core - - - name: Install Playwright browsers - run: npx playwright install + npx playwright install --with-deps chromium + docker compose pull ${{ matrix.CONTAINERS }} - name: Run Playwright e2e tests env: - COMPOSE_FILE: ./docker/docker-compose.suite-ci.yml - BASE_URL: https://dev.suite.sldev.cz/suite-web/feat-parametrize-pw-suite/web/ - - ## reporter url - TRACK_SUITE_URL: https://track-suite-ff9ad9f5b4f6.herokuapp.com - ## when debugging or developing tests it does not make sense to have retries, - ## in other cases retries are useful to avoid occasional failures due to flaky tests - ALLOW_RETRY: false + COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml + BASE_URL: https://dev.suite.sldev.cz/suite-web/${{ steps.extract_branch.outputs.branch }}/web/ + ## Keep retries enabled once stabilized + ALLOW_RETRY: true TEST_GROUP: ${{ matrix.TEST_GROUP }} - CI_JOB_ID: ${{ github.run_id }} - CI_COMMIT_SHA: ${{ github.sha }} - CI_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - CI_COMMIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - CI_COMMIT_MESSAGE: ${{ steps.extract_commit_message.outputs.message }} + GITHUB_ACTION: true CURRENTS_PROJECT_ID: Og0NOQ CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }} CURRENTS_CI_BUILD_ID: pr-run-${{github.run_id}} run: | - docker compose pull docker compose up -d ${{ matrix.CONTAINERS }} - echo "group ${TEST_GROUP}" + echo "Starting Playwright Web test group ${TEST_GROUP}" yarn workspace @trezor/suite-desktop-core test:e2e:web --grep=${TEST_GROUP} - # - name: Upload logs - # run: | - # docker cp docker_trezor-user-env-unix_1:/trezor-user-env/logs/debugging.log trezor-user-env-debugging.log || true - # docker cp docker_trezor-user-env-unix_1:/trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log || true - # docker cp docker_trezor-user-env-unix_1:/trezor-user-env/docker/version.txt trezor-user-env-version.txt || true - - # - name: Upload artifacts - # # this will run the upload artifacts even if the previous steps failed (e.g. tests failed). It wont run if the workflow was cancelled. - # if: ${{ ! cancelled() }} - # uses: actions/upload-artifact@v4 - # with: - # name: test-artifacts-${{ matrix.TEST_GROUP }} - # path: | - # ./packages/suite-web/e2e/snapshots - # ./packages/suite-web/e2e/screenshots - # ./packages/suite-web/e2e/videos - # download-snapshots.sh - # trezor-user-env-debugging.log - # tenv-emulator-bridge-debugging.log - # trezor-user-env-version.txt + + - name: Upload logs + if: ${{ ! cancelled() }} + run: | + docker cp docker-trezor-user-env-unix-1:/trezor-user-env/logs/debugging.log trezor-user-env-debugging.log || true + docker cp docker-trezor-user-env-unix-1:/trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log || true + docker cp docker-trezor-user-env-unix-1:/trezor-user-env/docker/version.txt trezor-user-env-version.txt || true + + - name: Upload Trezor-user-env logs + if: ${{ ! cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: emulator-logs-${{ matrix.TEST_GROUP }} + path: | + trezor-user-env-debugging.log + tenv-emulator-bridge-debugging.log + trezor-user-env-version.txt + + - name: Upload Playwright report + if: ${{ ! cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-report-${{ matrix.TEST_GROUP }} + path: ./packages/suite-desktop-core/e2e/playwright-report/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index 47a92d5492df..85be5053d422 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ __pycache__/ # test outputs screenshots test-results +playwright-report # build outputs lib diff --git a/docker/docker-compose.suite-ci-pw.yml b/docker/docker-compose.suite-ci-pw.yml new file mode 100644 index 000000000000..8ee275baad34 --- /dev/null +++ b/docker/docker-compose.suite-ci-pw.yml @@ -0,0 +1,13 @@ +version: "3.9" +services: + trezor-user-env-unix: + image: ghcr.io/trezor/trezor-user-env:c637c2f58e799284f6019481fa0de30457bd6b60 + environment: + - SDL_VIDEODRIVER=dummy + - XDG_RUNTIME_DIR=/var/tmp + network_mode: host + bitcoin-regtest: + image: ghcr.io/trezor/trezor-user-env-regtest # this is a special image that runs regtest and blockbook + depends_on: + - trezor-user-env-unix + network_mode: service:trezor-user-env-unix diff --git a/docker/docker-compose.suite-ci.yml b/docker/docker-compose.suite-ci.yml index 423caeda351d..a8a759ad41e2 100644 --- a/docker/docker-compose.suite-ci.yml +++ b/docker/docker-compose.suite-ci.yml @@ -1,19 +1,38 @@ -version: "4.0" +version: "3.9" services: trezor-user-env-unix: image: ghcr.io/trezor/trezor-user-env:c637c2f58e799284f6019481fa0de30457bd6b60 environment: - SDL_VIDEODRIVER=dummy - XDG_RUNTIME_DIR=/var/tmp - ports: - - "9001:9001" - - "21325:21325" - - "21326:21326" - + network_mode: bridge # this makes docker reuse existing networks + test-run: + image: cypress/included:13.6.4 + entrypoint: [] + environment: + - CYPRESS_SNAPSHOT=$CYPRESS_SNAPSHOT + - CYPRESS_updateSnapshots=$CYPRESS_updateSnapshots + - CYPRESS_baseUrl=$CYPRESS_baseUrl + - CYPRESS_ASSET_PREFIX=$CYPRESS_ASSET_PREFIX + - CYPRESS_TEST_URLS=$CYPRESS_TEST_URLS + - CYPRESS_USE_TREZOR_USER_ENV_BRIDGE=$CYPRESS_USE_TREZOR_USER_ENV_BRIDGE + - LOCAL_USER_ID=$LOCAL_USER_ID + - TEST_GROUP=$TEST_GROUP + - TRACK_SUITE_URL=$TRACK_SUITE_URL + - ALLOW_RETRY=$ALLOW_RETRY + - CI_JOB_URL=$CI_JOB_URL + - CI_JOB_ID=$CI_JOB_ID + - CI_COMMIT_BRANCH=$CI_COMMIT_BRANCH + - CI_COMMIT_MESSAGE=$CI_COMMIT_MESSAGE + - CI_COMMIT_SHA=$CI_COMMIT_SHA + - FIRMWARE=$FIRMWARE + network_mode: service:trezor-user-env-unix + working_dir: /trezor-suite + command: bash -c "cd packages/suite-web && yarn tsx ./e2e/run_tests.ts --group=$TEST_GROUP" + volumes: + - ../:/trezor-suite bitcoin-regtest: image: ghcr.io/trezor/trezor-user-env-regtest # this is a special image that runs regtest and blockbook depends_on: - trezor-user-env-unix - ports: - - "50001:50001" network_mode: service:trezor-user-env-unix diff --git a/packages/suite-desktop-core/e2e/playwright.config.ts b/packages/suite-desktop-core/e2e/playwright.config.ts index eb91c693e83a..1acaf3b7e228 100644 --- a/packages/suite-desktop-core/e2e/playwright.config.ts +++ b/packages/suite-desktop-core/e2e/playwright.config.ts @@ -31,7 +31,13 @@ const config: PlaywrightTestConfig = { testIdAttribute: 'data-testid', }, reportSlowTests: null, - reporter: process.env.GITHUB_ACTION ? [['list'], ['@currents/playwright']] : [['list']], + reporter: process.env.GITHUB_ACTION + ? [ + ['list'], + ['@currents/playwright'], + ['html', { open: 'never', outputFolder: './playwright-report' }], + ] + : [['list'], ['html', { open: 'never', outputFolder: './playwright-report' }]], timeout: 1000 * 60 * 5, outputDir: path.join(__dirname, 'test-results'), }; diff --git a/packages/suite-desktop-core/e2e/support/fixtures.ts b/packages/suite-desktop-core/e2e/support/fixtures.ts index a49fb9ebb916..e8b26915b6f8 100644 --- a/packages/suite-desktop-core/e2e/support/fixtures.ts +++ b/packages/suite-desktop-core/e2e/support/fixtures.ts @@ -66,13 +66,12 @@ const test = base.extend({ window: async ({ appContext, page }, use, testInfo) => { if (appContext) { const window = await appContext.firstWindow(); - await window.context().tracing.start({ screenshots: true, snapshots: true }); await use(window); const tracePath = `${testInfo.outputDir}/trace.electron.zip`; await window.context().tracing.stop({ path: tracePath }); } else { - await page.goto('/'); + await page.goto('./'); await use(page); } }, diff --git a/packages/suite-desktop-core/e2e/tests/general/cardano-discovery.test.ts b/packages/suite-desktop-core/e2e/tests/general/cardano-discovery.test.ts index 16b3d95a5ec2..fd71e74334c1 100644 --- a/packages/suite-desktop-core/e2e/tests/general/cardano-discovery.test.ts +++ b/packages/suite-desktop-core/e2e/tests/general/cardano-discovery.test.ts @@ -19,17 +19,21 @@ test.beforeEach(async ({ onboardingPage, dashboardPage }) => { * 2. Check that all types of Cardano accounts are discovered * 3. Check that Staking section is available */ -test('Discover all Cardano account types', async ({ dashboardPage, settingsPage, walletPage }) => { - await settingsPage.navigateTo(); - await settingsPage.coinsTabButton.click(); - await settingsPage.enableCoin('ada'); - await settingsPage.disableCoin('btc'); +test( + 'Discover all Cardano account types', + { tag: ['@group=wallet'] }, + async ({ dashboardPage, settingsPage, walletPage }) => { + await settingsPage.navigateTo(); + await settingsPage.coinsTabButton.click(); + await settingsPage.enableCoin('ada'); + await settingsPage.disableCoin('btc'); - await dashboardPage.navigateTo(); - await dashboardPage.discoveryShouldFinish(); + await dashboardPage.navigateTo(); + await dashboardPage.discoveryShouldFinish(); - await walletPage.expandAllAccountsInMenu(); - await walletPage.checkStakesOfCardanoAccounts(); + await walletPage.expandAllAccountsInMenu(); + await walletPage.checkStakesOfCardanoAccounts(); - expect(await walletPage.getAccountsCount('ada')).toEqual(3); -}); + expect(await walletPage.getAccountsCount('ada')).toEqual(3); + }, +); diff --git a/packages/suite-desktop-core/e2e/tests/general/eap-modal.test.ts b/packages/suite-desktop-core/e2e/tests/general/eap-modal.test.ts index ec6ad4012ea9..2ea20b7c82e3 100644 --- a/packages/suite-desktop-core/e2e/tests/general/eap-modal.test.ts +++ b/packages/suite-desktop-core/e2e/tests/general/eap-modal.test.ts @@ -15,7 +15,7 @@ test.beforeAll(async ({ onboardingPage }) => { await onboardingPage.completeOnboarding(); }); -test('Join early access button', async ({ settingsPage }) => { +test('Join early access button', { tag: '@settings' }, async ({ settingsPage }) => { const buttonText = 'Leave'; await settingsPage.navigateTo(); await settingsPage.applicationTabButton.click(); diff --git a/packages/suite-desktop-core/e2e/tests/general/suite-guide.test.ts b/packages/suite-desktop-core/e2e/tests/general/suite-guide.test.ts index 4d54f5b65a1b..a2b3c4e29e73 100644 --- a/packages/suite-desktop-core/e2e/tests/general/suite-guide.test.ts +++ b/packages/suite-desktop-core/e2e/tests/general/suite-guide.test.ts @@ -1,34 +1,36 @@ import { test, expect } from '../../support/fixtures'; -test.use({ startEmulator: false }); -/** - * Test case: - * 1. Go to Bug section in Suite Guide - * 2. Select Dashboard - * 3. Write into feedback field - * 4. Submit bug report (reporttext) - */ -test('Send a bug report', async ({ suiteGuidePage }) => { - await suiteGuidePage.openPanel(); - await suiteGuidePage.supportAndFeedbackButton.click(); - await suiteGuidePage.sendBugReport({ - location: 'account', - report: 'Henlo this is testy test writing hangry test user report', +test.describe('Suite Guide', { tag: '@suite' }, () => { + test.use({ startEmulator: false }); + /** + * Test case: + * 1. Go to Bug section in Suite Guide + * 2. Select Dashboard + * 3. Write into feedback field + * 4. Submit bug report (reporttext) + */ + test('Send a bug report', async ({ suiteGuidePage }) => { + await suiteGuidePage.openPanel(); + await suiteGuidePage.supportAndFeedbackButton.click(); + await suiteGuidePage.sendBugReport({ + location: 'account', + report: 'Henlo this is testy test writing hangry test user report', + }); + await expect(suiteGuidePage.feedbackSuccessToast).toBeVisible(); + await suiteGuidePage.closeGuide(); }); - await expect(suiteGuidePage.feedbackSuccessToast).toBeVisible(); - await suiteGuidePage.closeGuide(); -}); -/** - * Test case: - * 1. Go to Suggestion section in Suite Guide - * 2. Look up an article - * 3. Verify that the article is displayed - */ -test('Look up an article', async ({ suiteGuidePage }) => { - const article = 'Install firmware'; - await suiteGuidePage.openPanel(); - await suiteGuidePage.lookupArticle(article); - await expect(suiteGuidePage.articleHeader).toHaveText(article); - await suiteGuidePage.closeGuide(); + /** + * Test case: + * 1. Go to Suggestion section in Suite Guide + * 2. Look up an article + * 3. Verify that the article is displayed + */ + test('Look up an article', async ({ suiteGuidePage }) => { + const article = 'Install firmware'; + await suiteGuidePage.openPanel(); + await suiteGuidePage.lookupArticle(article); + await expect(suiteGuidePage.articleHeader).toHaveText(article); + await suiteGuidePage.closeGuide(); + }); }); diff --git a/packages/suite-desktop-core/eslint.config.mjs b/packages/suite-desktop-core/eslint.config.mjs index 37bf46504016..716b17d4eecb 100644 --- a/packages/suite-desktop-core/eslint.config.mjs +++ b/packages/suite-desktop-core/eslint.config.mjs @@ -17,4 +17,7 @@ export default [ ], }, }, + { + ignores: ['**/playwright-report/'], + }, ];