Skip to content

Commit

Permalink
feat(e2e): Debuging PW web tests on CI
Browse files Browse the repository at this point in the history
  • Loading branch information
Vere-Grey committed Dec 3, 2024
1 parent 6e1039b commit af55f74
Show file tree
Hide file tree
Showing 10 changed files with 134 additions and 97 deletions.
80 changes: 35 additions & 45 deletions .github/workflows/test-suite-web-e2e-pw.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,25 +95,18 @@ jobs:
include:
# - TEST_GROUP: "@group=suite"
# CONTAINERS: "trezor-user-env-unix"
# CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
# - TEST_GROUP: "@group=device-management"
# CONTAINERS: "trezor-user-env-unix"
# CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
- TEST_GROUP: "@group=settings"
CONTAINERS: "trezor-user-env-unix"
CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
# - TEST_GROUP: "@group=metadata"
# CONTAINERS: "trezor-user-env-unix"
# CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
# - TEST_GROUP: "@group=passphrase"
# CONTAINERS: "trezor-user-env-unix"
# CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
# - TEST_GROUP: "@group=other"
# CONTAINERS: "trezor-user-env-unix"
# CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
- TEST_GROUP: "@group=wallet"
CONTAINERS: "trezor-user-env-unix bitcoin-regtest"
CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
steps:
- name: Checkout
uses: actions/checkout@v4
Expand Down Expand Up @@ -141,56 +134,53 @@ jobs:
echo "message=$(git log --no-merges -1 --pretty=format:"%s")" >> $GITHUB_OUTPUT
fi
- name: Install dependencies
- name: Install dependencies and pull docker images
env:
COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml
run: |
echo -e "\nenableScripts: false" >> .yarnrc.yml
echo -e "\nenableHardenedMode: false" >> .yarnrc.yml
yarn workspaces focus @trezor/suite-desktop-core
- name: Install Playwright browsers
run: npx playwright install
docker compose pull ${{ matrix.CONTAINERS }}
## npx playwright install --with-deps

- name: Run Playwright e2e tests
env:
COMPOSE_FILE: ./docker/docker-compose.suite-ci.yml
BASE_URL: https://dev.suite.sldev.cz/suite-web/feat-parametrize-pw-suite/web/

## reporter url
TRACK_SUITE_URL: https://track-suite-ff9ad9f5b4f6.herokuapp.com
## when debugging or developing tests it does not make sense to have retries,
## in other cases retries are useful to avoid occasional failures due to flaky tests
ALLOW_RETRY: false
COMPOSE_FILE: ./docker/docker-compose.suite-ci-pw.yml
BASE_URL: https://dev.suite.sldev.cz/suite-web/${{ steps.extract_branch.outputs.branch }}/web/
## Keep retries enabled once stabilized
ALLOW_RETRY: true
TEST_GROUP: ${{ matrix.TEST_GROUP }}
CI_JOB_ID: ${{ github.run_id }}
CI_COMMIT_SHA: ${{ github.sha }}
CI_JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
CI_COMMIT_BRANCH: ${{ steps.extract_branch.outputs.branch }}
CI_COMMIT_MESSAGE: ${{ steps.extract_commit_message.outputs.message }}
GITHUB_ACTION: true
CURRENTS_PROJECT_ID: Og0NOQ
CURRENTS_RECORD_KEY: ${{ secrets.CURRENTS_RECORD_KEY }}
CURRENTS_CI_BUILD_ID: pr-run-${{github.run_id}}
run: |
docker compose pull
docker compose up -d ${{ matrix.CONTAINERS }}
echo "group ${TEST_GROUP}"
yarn workspace @trezor/suite-desktop-core test:e2e:web --grep=${TEST_GROUP}
# - name: Upload logs
# run: |
# docker cp docker_trezor-user-env-unix_1:/trezor-user-env/logs/debugging.log trezor-user-env-debugging.log || true
# docker cp docker_trezor-user-env-unix_1:/trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log || true
# docker cp docker_trezor-user-env-unix_1:/trezor-user-env/docker/version.txt trezor-user-env-version.txt || true

# - name: Upload artifacts
# # this will run the upload artifacts even if the previous steps failed (e.g. tests failed). It wont run if the workflow was cancelled.
# if: ${{ ! cancelled() }}
# uses: actions/upload-artifact@v4
# with:
# name: test-artifacts-${{ matrix.TEST_GROUP }}
# path: |
# ./packages/suite-web/e2e/snapshots
# ./packages/suite-web/e2e/screenshots
# ./packages/suite-web/e2e/videos
# download-snapshots.sh
# trezor-user-env-debugging.log
# tenv-emulator-bridge-debugging.log
# trezor-user-env-version.txt
- name: Upload logs
if: ${{ ! cancelled() }}
run: |
docker cp docker-trezor-user-env-unix-1:/trezor-user-env/logs/debugging.log trezor-user-env-debugging.log || true
docker cp docker-trezor-user-env-unix-1:/trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log || true
docker cp docker-trezor-user-env-unix-1:/trezor-user-env/docker/version.txt trezor-user-env-version.txt || true
- name: Upload Trezor-user-env logs
if: ${{ ! cancelled() }}
uses: actions/upload-artifact@v4
with:
name: emulator-logs-${{ matrix.TEST_GROUP }}
path: |
trezor-user-env-debugging.log
tenv-emulator-bridge-debugging.log
trezor-user-env-version.txt
- name: Upload Playwright report
if: ${{ ! cancelled() }}
uses: actions/upload-artifact@v4
with:
name: playwright-report-${{ matrix.TEST_GROUP }}
path: ./packages/suite-desktop-core/e2e/playwright-report/
retention-days: 30
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ __pycache__/
# test outputs
screenshots
test-results
playwright-report

# build outputs
lib
Expand Down
13 changes: 13 additions & 0 deletions docker/docker-compose.suite-ci-pw.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
version: "3.9"
services:
trezor-user-env-unix:
image: ghcr.io/trezor/trezor-user-env:c637c2f58e799284f6019481fa0de30457bd6b60
environment:
- SDL_VIDEODRIVER=dummy
- XDG_RUNTIME_DIR=/var/tmp
network_mode: host
bitcoin-regtest:
image: ghcr.io/trezor/trezor-user-env-regtest # this is a special image that runs regtest and blockbook
depends_on:
- trezor-user-env-unix
network_mode: service:trezor-user-env-unix
35 changes: 27 additions & 8 deletions docker/docker-compose.suite-ci.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,38 @@
version: "4.0"
version: "3.9"
services:
trezor-user-env-unix:
image: ghcr.io/trezor/trezor-user-env:c637c2f58e799284f6019481fa0de30457bd6b60
environment:
- SDL_VIDEODRIVER=dummy
- XDG_RUNTIME_DIR=/var/tmp
ports:
- "9001:9001"
- "21325:21325"
- "21326:21326"

network_mode: bridge # this makes docker reuse existing networks
test-run:
image: cypress/included:13.6.4
entrypoint: []
environment:
- CYPRESS_SNAPSHOT=$CYPRESS_SNAPSHOT
- CYPRESS_updateSnapshots=$CYPRESS_updateSnapshots
- CYPRESS_baseUrl=$CYPRESS_baseUrl
- CYPRESS_ASSET_PREFIX=$CYPRESS_ASSET_PREFIX
- CYPRESS_TEST_URLS=$CYPRESS_TEST_URLS
- CYPRESS_USE_TREZOR_USER_ENV_BRIDGE=$CYPRESS_USE_TREZOR_USER_ENV_BRIDGE
- LOCAL_USER_ID=$LOCAL_USER_ID
- TEST_GROUP=$TEST_GROUP
- TRACK_SUITE_URL=$TRACK_SUITE_URL
- ALLOW_RETRY=$ALLOW_RETRY
- CI_JOB_URL=$CI_JOB_URL
- CI_JOB_ID=$CI_JOB_ID
- CI_COMMIT_BRANCH=$CI_COMMIT_BRANCH
- CI_COMMIT_MESSAGE=$CI_COMMIT_MESSAGE
- CI_COMMIT_SHA=$CI_COMMIT_SHA
- FIRMWARE=$FIRMWARE
network_mode: service:trezor-user-env-unix
working_dir: /trezor-suite
command: bash -c "cd packages/suite-web && yarn tsx ./e2e/run_tests.ts --group=$TEST_GROUP"
volumes:
- ../:/trezor-suite
bitcoin-regtest:
image: ghcr.io/trezor/trezor-user-env-regtest # this is a special image that runs regtest and blockbook
depends_on:
- trezor-user-env-unix
ports:
- "50001:50001"
network_mode: service:trezor-user-env-unix
8 changes: 7 additions & 1 deletion packages/suite-desktop-core/e2e/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,13 @@ const config: PlaywrightTestConfig = {
testIdAttribute: 'data-testid',
},
reportSlowTests: null,
reporter: process.env.GITHUB_ACTION ? [['list'], ['@currents/playwright']] : [['list']],
reporter: process.env.GITHUB_ACTION
? [
['list'],
['@currents/playwright'],
['html', { open: 'never', outputFolder: './playwright-report' }],
]
: [['list'], ['html', { open: 'never', outputFolder: './playwright-report' }]],
timeout: 1000 * 60 * 5,
outputDir: path.join(__dirname, 'test-results'),
};
Expand Down
3 changes: 1 addition & 2 deletions packages/suite-desktop-core/e2e/support/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,12 @@ const test = base.extend<Fixtures>({
window: async ({ appContext, page }, use, testInfo) => {
if (appContext) {
const window = await appContext.firstWindow();

await window.context().tracing.start({ screenshots: true, snapshots: true });
await use(window);
const tracePath = `${testInfo.outputDir}/trace.electron.zip`;
await window.context().tracing.stop({ path: tracePath });
} else {
await page.goto('/');
await page.goto('./');
await use(page);
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,21 @@ test.beforeEach(async ({ onboardingPage, dashboardPage }) => {
* 2. Check that all types of Cardano accounts are discovered
* 3. Check that Staking section is available
*/
test('Discover all Cardano account types', async ({ dashboardPage, settingsPage, walletPage }) => {
await settingsPage.navigateTo();
await settingsPage.coinsTabButton.click();
await settingsPage.enableCoin('ada');
await settingsPage.disableCoin('btc');
test(
'Discover all Cardano account types',
{ tag: ['@group=wallet'] },
async ({ dashboardPage, settingsPage, walletPage }) => {
await settingsPage.navigateTo();
await settingsPage.coinsTabButton.click();
await settingsPage.enableCoin('ada');
await settingsPage.disableCoin('btc');

await dashboardPage.navigateTo();
await dashboardPage.discoveryShouldFinish();
await dashboardPage.navigateTo();
await dashboardPage.discoveryShouldFinish();

await walletPage.expandAllAccountsInMenu();
await walletPage.checkStakesOfCardanoAccounts();
await walletPage.expandAllAccountsInMenu();
await walletPage.checkStakesOfCardanoAccounts();

expect(await walletPage.getAccountsCount('ada')).toEqual(3);
});
expect(await walletPage.getAccountsCount('ada')).toEqual(3);
},
);
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ test.beforeAll(async ({ onboardingPage }) => {
await onboardingPage.completeOnboarding();
});

test('Join early access button', async ({ settingsPage }) => {
test('Join early access button', { tag: '@settings' }, async ({ settingsPage }) => {
const buttonText = 'Leave';
await settingsPage.navigateTo();
await settingsPage.applicationTabButton.click();
Expand Down
60 changes: 31 additions & 29 deletions packages/suite-desktop-core/e2e/tests/general/suite-guide.test.ts
Original file line number Diff line number Diff line change
@@ -1,34 +1,36 @@
import { test, expect } from '../../support/fixtures';

test.use({ startEmulator: false });
/**
* Test case:
* 1. Go to Bug section in Suite Guide
* 2. Select Dashboard
* 3. Write into feedback field
* 4. Submit bug report (reporttext)
*/
test('Send a bug report', async ({ suiteGuidePage }) => {
await suiteGuidePage.openPanel();
await suiteGuidePage.supportAndFeedbackButton.click();
await suiteGuidePage.sendBugReport({
location: 'account',
report: 'Henlo this is testy test writing hangry test user report',
test.describe('Suite Guide', { tag: '@suite' }, () => {
test.use({ startEmulator: false });
/**
* Test case:
* 1. Go to Bug section in Suite Guide
* 2. Select Dashboard
* 3. Write into feedback field
* 4. Submit bug report (reporttext)
*/
test('Send a bug report', async ({ suiteGuidePage }) => {
await suiteGuidePage.openPanel();
await suiteGuidePage.supportAndFeedbackButton.click();
await suiteGuidePage.sendBugReport({
location: 'account',
report: 'Henlo this is testy test writing hangry test user report',
});
await expect(suiteGuidePage.feedbackSuccessToast).toBeVisible();
await suiteGuidePage.closeGuide();
});
await expect(suiteGuidePage.feedbackSuccessToast).toBeVisible();
await suiteGuidePage.closeGuide();
});

/**
* Test case:
* 1. Go to Suggestion section in Suite Guide
* 2. Look up an article
* 3. Verify that the article is displayed
*/
test('Look up an article', async ({ suiteGuidePage }) => {
const article = 'Install firmware';
await suiteGuidePage.openPanel();
await suiteGuidePage.lookupArticle(article);
await expect(suiteGuidePage.articleHeader).toHaveText(article);
await suiteGuidePage.closeGuide();
/**
* Test case:
* 1. Go to Suggestion section in Suite Guide
* 2. Look up an article
* 3. Verify that the article is displayed
*/
test('Look up an article', async ({ suiteGuidePage }) => {
const article = 'Install firmware';
await suiteGuidePage.openPanel();
await suiteGuidePage.lookupArticle(article);
await expect(suiteGuidePage.articleHeader).toHaveText(article);
await suiteGuidePage.closeGuide();
});
});
3 changes: 3 additions & 0 deletions packages/suite-desktop-core/eslint.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,7 @@ export default [
],
},
},
{
ignores: ['**/playwright-report/'],
},
];

0 comments on commit af55f74

Please sign in to comment.