diff --git a/.github/actions/testSetup/action.yml b/.github/actions/testSetup/action.yml index 3a56663c5c..d99ec44fa5 100644 --- a/.github/actions/testSetup/action.yml +++ b/.github/actions/testSetup/action.yml @@ -43,3 +43,6 @@ runs: - name: Copy dotenv shell: 'bash' run: cat tmp/dotenv >> .env && rm -rf tmp + - name: Clear old screenshots + shell: 'bash' + run: rm -rf screenshots/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8c5e4d669..f55e28bcfa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -67,7 +67,7 @@ jobs: with: name: rainbowbx-${{ github.sha }}.zip path: build/ - # FIREFOX TESTS - Disabled for now till all tests are passing + # FIREFOX TESTS firefox-e2e-parallel: runs-on: ubuntu-latest timeout-minutes: 16 @@ -114,12 +114,23 @@ jobs: shell: 'bash' run: cat tmp/dotenv >> .env && rm -rf tmp - name: Run e2e parallel (Firefox) + id: FFE2eParallel + continue-on-error: true run: | export BROWSER=firefox export OS=linux export FIREFOX_BIN=/opt/hostedtoolcache/firefox/latest-devedition/x64/firefox yarn firefox:manifest && yarn firefox:zip yarn vitest:parallel --retry=5 + - name: Upload deps artifacts + if: steps.FFE2eParallel.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: screenshots/ + - name: Fail if any tests failed + if: steps.FFE2eParallel.outcome == 'failure' + run: exit 1 # CHROME TESTS chrome-e2e-parallel: runs-on: ubuntu-latest @@ -134,11 +145,22 @@ jobs: with: gh-access-token: ${{ secrets.DOTENV_GITHUB_ACCESS_TOKEN }} - name: Run e2e parallel (Chrome) + id: ChromeE2EParallel + continue-on-error: true run: | export BROWSER=chrome export OS=linux export CHROMIUM_BIN=$(find chrome -type f -name 'chrome') yarn vitest:parallel --retry=5 + - name: Upload deps artifacts + if: steps.ChromeE2EParallel.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: screenshots/ + - name: Fail if any tests failed + if: steps.ChromeE2EParallel.outcome == 'failure' + run: exit 1 chrome-e2e-swap: runs-on: beefy-runner-bx timeout-minutes: 25 @@ -152,6 +174,8 @@ jobs: with: gh-access-token: ${{ secrets.DOTENV_GITHUB_ACCESS_TOKEN }} - name: Run e2e swap (Chrome) + id: ChromeE2ESwaps + continue-on-error: true uses: nick-fields/retry@v2 with: timeout_minutes: 25 @@ -161,6 +185,15 @@ jobs: export OS=linux export CHROMIUM_BIN=$(find chrome -type f -name 'chrome') yarn vitest:swap + - name: Upload deps artifacts + if: steps.ChromeE2ESwaps.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: screenshots/ + - name: Fail if any tests failed + if: steps.ChromeE2ESwaps.outcome == 'failure' + run: exit 1 chrome-e2e-send: runs-on: beefy-runner-bx timeout-minutes: 25 @@ -174,11 +207,22 @@ jobs: with: gh-access-token: ${{ secrets.DOTENV_GITHUB_ACCESS_TOKEN }} - name: Run e2e send (Chrome) + id: ChromeE2ESend + continue-on-error: true run: | export BROWSER=chrome export OS=linux export CHROMIUM_BIN=$(find chrome -type f -name 'chrome') yarn vitest:send --retry=5 + - name: Upload deps artifacts + if: steps.ChromeE2ESend.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: screenshots/ + - name: Fail if any tests failed + if: steps.ChromeE2ESend.outcome == 'failure' + run: exit 1 chrome-e2e-dappInteractions: runs-on: beefy-runner-bx timeout-minutes: 25 @@ -192,11 +236,22 @@ jobs: with: gh-access-token: ${{ secrets.DOTENV_GITHUB_ACCESS_TOKEN }} - name: Run e2e dappInteractions (Chrome) + id: ChromeE2EDappInteractions + continue-on-error: true run: | export BROWSER=chrome export OS=linux export CHROMIUM_BIN=$(find chrome -type f -name 'chrome') yarn vitest:dappInteractions --retry=5 + - name: Upload deps artifacts + if: steps.ChromeE2EDappInteractions.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: screenshots + path: screenshots/ + - name: Fail if any tests failed + if: steps.ChromeE2EDappInteractions.outcome == 'failure' + run: exit 1 # BRAVE TESTS # brave-e2e: # needs: [build] @@ -328,3 +383,6 @@ jobs: - uses: geekyeggo/delete-artifact@v2 with: name: node_modules.tar.gz + - uses: geekyeggo/delete-artifact@v2 + with: + name: screenshots diff --git a/e2e/helpers.ts b/e2e/helpers.ts index d8c4f46eb9..fa1996fe2f 100644 --- a/e2e/helpers.ts +++ b/e2e/helpers.ts @@ -184,12 +184,17 @@ export async function getExtensionIdByName( // search functions export async function querySelector(driver: WebDriver, selector: string) { - await driver.wait(untilDocumentLoaded(), waitUntilTime); - const el = await driver.wait( - until.elementLocated(By.css(selector)), - waitUntilTime, - ); - return await driver.wait(until.elementIsVisible(el), waitUntilTime); + try { + await driver.wait(untilDocumentLoaded(), waitUntilTime); + const el = await driver.wait( + until.elementLocated(By.css(selector)), + waitUntilTime, + ); + return await driver.wait(until.elementIsVisible(el), waitUntilTime); + } catch (error) { + await takeScreenshot(driver, selector); + throw error; + } } export async function findElementByText(driver: WebDriver, text: string) { @@ -318,9 +323,13 @@ export async function waitAndClick(element: WebElement, driver: WebDriver) { await driver.wait(until.elementIsEnabled(element), waitUntilTime); return element.click(); } catch (error) { - throw new Error( - `Failed to click element ${await element.getAttribute('data-testid')}`, - ); + const testId = await element.getAttribute('data-testid'); + if (testId) { + await takeScreenshot(driver, testId); + } else { + console.log("couldn't take screenshot because element has no test id"); + } + throw new Error(`Failed to click element ${testId}`); } } @@ -719,3 +728,14 @@ export async function delayTime( return await delay(5000); } } + +export async function takeScreenshot(driver: WebDriver, name: string) { + try { + const image = await driver.takeScreenshot(); + const safeName = name.replace('[data-testid="', '').replace('"]', ''); + const filename = `${new Date().getTime()}-${safeName}`; + require('fs').writeFileSync(`screenshots/${filename}.png`, image, 'base64'); + } catch (error) { + console.error('Error occurred while taking screenshot:', error); + } +} diff --git a/e2e/serial/dappInteractions/1_appInteractionsFlow.test.ts b/e2e/serial/dappInteractions/1_appInteractionsFlow.test.ts index dffb1247bd..a1cd209dd8 100644 --- a/e2e/serial/dappInteractions/1_appInteractionsFlow.test.ts +++ b/e2e/serial/dappInteractions/1_appInteractionsFlow.test.ts @@ -356,7 +356,7 @@ describe('App interactions flow', () => { const { popupHandler } = await getAllWindowHandles({ driver, dappHandler }); await driver.switchTo().window(popupHandler); - await delayTime('long'); + await delayTime('very-long'); await findElementByTestIdAndClick({ id: 'accept-request-button', driver }); await delayTime('long'); await driver.switchTo().window(dappHandler); diff --git a/package.json b/package.json index 249974ac5d..0a50af720d 100644 --- a/package.json +++ b/package.json @@ -171,7 +171,7 @@ "@vanilla-extract/webpack-plugin": "2.2.0", "audit-ci": "6.3.0", "browserify": "17.0.0", - "chromedriver": "115.0.0", + "chromedriver": "116.0.0", "circular-dependency-plugin": "5.2.2", "copy-webpack-plugin": "11.0.0", "css-loader": "6.7.1", @@ -186,7 +186,7 @@ "eslint-plugin-prettier": "4.2.1", "file-loader": "6.2.0", "fs-extra": "10.1.0", - "geckodriver": "4.1.3", + "geckodriver": "4.2.0", "globby": "11.0.4", "happy-dom": "8.9.0", "html-webpack-plugin": "5.5.0", @@ -205,7 +205,7 @@ "svg-path-commander": "1.0.5", "ts-loader": "9.4.1", "typescript": "4.9.4", - "vitest": "0.33.0", + "vitest": "0.34.2", "web-ext": "7.6.2", "webpack": "5.76.0", "webpack-bundle-analyzer": "4.8.0", @@ -289,4 +289,4 @@ "yarn lint --fix --ignore-pattern '!*.d.ts'" ] } -} \ No newline at end of file +} diff --git a/screenshots/.dummy b/screenshots/.dummy new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/screenshots/.dummy @@ -0,0 +1 @@ + diff --git a/src/entries/popup/App.test.tsx b/src/entries/popup/App.test.disabled.tsx similarity index 100% rename from src/entries/popup/App.test.tsx rename to src/entries/popup/App.test.disabled.tsx diff --git a/yarn.lock b/yarn.lock index 89d8a1ee21..9389757f33 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4139,44 +4139,44 @@ debug "^4.3.1" loader-utils "^2.0.0" -"@vitest/expect@0.33.0": - version "0.33.0" - resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-0.33.0.tgz#f48652591f3573ad6c2db828ad358d5c078845d3" - integrity sha512-sVNf+Gla3mhTCxNJx+wJLDPp/WcstOe0Ksqz4Vec51MmgMth/ia0MGFEkIZmVGeTL5HtjYR4Wl/ZxBxBXZJTzQ== +"@vitest/expect@0.34.2": + version "0.34.2" + resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-0.34.2.tgz#2bd09c20b828f8f9da625c203d88003a2b69e314" + integrity sha512-EZm2dMNlLyIfDMha17QHSQcg2KjeAZaXd65fpPzXY5bvnfx10Lcaz3N55uEe8PhF+w4pw+hmrlHLLlRn9vkBJg== dependencies: - "@vitest/spy" "0.33.0" - "@vitest/utils" "0.33.0" + "@vitest/spy" "0.34.2" + "@vitest/utils" "0.34.2" chai "^4.3.7" -"@vitest/runner@0.33.0": - version "0.33.0" - resolved "https://registry.yarnpkg.com/@vitest/runner/-/runner-0.33.0.tgz#0b1a4d04ff8bc5cdad73920eac019d99550edf9d" - integrity sha512-UPfACnmCB6HKRHTlcgCoBh6ppl6fDn+J/xR8dTufWiKt/74Y9bHci5CKB8tESSV82zKYtkBJo9whU3mNvfaisg== +"@vitest/runner@0.34.2": + version "0.34.2" + resolved "https://registry.yarnpkg.com/@vitest/runner/-/runner-0.34.2.tgz#3408682cd68475e733a3f151d27792be75d2f07d" + integrity sha512-8ydGPACVX5tK3Dl0SUwxfdg02h+togDNeQX3iXVFYgzF5odxvaou7HnquALFZkyVuYskoaHUOqOyOLpOEj5XTA== dependencies: - "@vitest/utils" "0.33.0" + "@vitest/utils" "0.34.2" p-limit "^4.0.0" pathe "^1.1.1" -"@vitest/snapshot@0.33.0": - version "0.33.0" - resolved "https://registry.yarnpkg.com/@vitest/snapshot/-/snapshot-0.33.0.tgz#4400a90c48907808122b573053a2112a832b3698" - integrity sha512-tJjrl//qAHbyHajpFvr8Wsk8DIOODEebTu7pgBrP07iOepR5jYkLFiqLq2Ltxv+r0uptUb4izv1J8XBOwKkVYA== +"@vitest/snapshot@0.34.2": + version "0.34.2" + resolved "https://registry.yarnpkg.com/@vitest/snapshot/-/snapshot-0.34.2.tgz#fce1b89aa1e836e3fd0229c03ef4ef2f69cb1409" + integrity sha512-qhQ+xy3u4mwwLxltS4Pd4SR+XHv4EajiTPNY3jkIBLUApE6/ce72neJPSUQZ7bL3EBuKI+NhvzhGj3n5baRQUQ== dependencies: magic-string "^0.30.1" pathe "^1.1.1" pretty-format "^29.5.0" -"@vitest/spy@0.33.0": - version "0.33.0" - resolved "https://registry.yarnpkg.com/@vitest/spy/-/spy-0.33.0.tgz#366074d3cf9cf1ed8aeaa76e50e78c799fb242eb" - integrity sha512-Kv+yZ4hnH1WdiAkPUQTpRxW8kGtH8VRTnus7ZTGovFYM1ZezJpvGtb9nPIjPnptHbsyIAxYZsEpVPYgtpjGnrg== +"@vitest/spy@0.34.2": + version "0.34.2" + resolved "https://registry.yarnpkg.com/@vitest/spy/-/spy-0.34.2.tgz#5c483d71e4c2d42f90bef29cdf6e5f5559c52827" + integrity sha512-yd4L9OhfH6l0Av7iK3sPb3MykhtcRN5c5K5vm1nTbuN7gYn+yvUVVsyvzpHrjqS7EWqn9WsPJb7+0c3iuY60tA== dependencies: tinyspy "^2.1.1" -"@vitest/utils@0.33.0": - version "0.33.0" - resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-0.33.0.tgz#6b9820cb8f128d649da6f78ecaa9b73d6222b277" - integrity sha512-pF1w22ic965sv+EN6uoePkAOTkAPWM03Ri/jXNyMIKBb/XHLDPfhLvf/Fa9g0YECevAIz56oVYXhodLvLQ/awA== +"@vitest/utils@0.34.2": + version "0.34.2" + resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-0.34.2.tgz#5d291a1b0f5d01be99fd1801d212b837a610c53b" + integrity sha512-Lzw+kAsTPubhoQDp1uVAOP6DhNia1GMDsI9jgB0yMn+/nDaPieYQ88lKqz/gGjSHL4zwOItvpehec9OY+rS73w== dependencies: diff-sequences "^29.4.3" loupe "^2.3.6" @@ -5983,10 +5983,10 @@ chrome-trace-event@^1.0.2: resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== -chromedriver@115.0.0: - version "115.0.0" - resolved "https://registry.yarnpkg.com/chromedriver/-/chromedriver-115.0.0.tgz#2d50dd1d12d553c1c6cad22d329dc5cf63487ada" - integrity sha512-mkPL+sXMLMUenoXCiKREw+cBl3ibfhiWxkiv9ByIPpqtrrInCt9zKdOolAsbmN/ndlH51WtT5ukUKbeRdrpikg== +chromedriver@116.0.0: + version "116.0.0" + resolved "https://registry.yarnpkg.com/chromedriver/-/chromedriver-116.0.0.tgz#3f5d07b5427953270461791651d7b68cb6afe9fe" + integrity sha512-/TQaRn+RUAYnVqy5Vx8VtU8DvtWosU8QLM2u7BoNM5h55PRQPXF/onHAehEi8Sj/CehdKqH50NFdiumQAUr0DQ== dependencies: "@testim/chrome-version" "^1.1.3" axios "^1.4.0" @@ -8512,10 +8512,10 @@ gauge@^4.0.3: strip-ansi "^6.0.1" wide-align "^1.1.5" -geckodriver@4.1.3: - version "4.1.3" - resolved "https://registry.yarnpkg.com/geckodriver/-/geckodriver-4.1.3.tgz#4a9e25a301e53a6fb5a9e1d72e3d29b7b1815727" - integrity sha512-w3vZvibJnDLui3ZcyWz/CjiWET31br7ALakbfEqssSzwBFzolOj6/FnXPgcmxSkdH1iMBOc83dewLEqR8xoilA== +geckodriver@4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/geckodriver/-/geckodriver-4.2.0.tgz#a385370011ae29fa8c68351b01d054a92b92d945" + integrity sha512-I2BlybeMFMzpxHRrh8X4VwP4ys74JHszyUgfezOrbTR7PEybFneDcuEjKIQxKV6vFPmsdwhwF1x8AshdQo56xA== dependencies: "@wdio/logger" "^8.11.0" decamelize "^6.0.0" @@ -14227,10 +14227,10 @@ tinycolor2@^1.4.1: resolved "https://registry.yarnpkg.com/tinycolor2/-/tinycolor2-1.4.2.tgz#3f6a4d1071ad07676d7fa472e1fac40a719d8803" integrity sha512-vJhccZPs965sV/L2sU4oRQVAos0pQXwsvTLkWYdqJ+a8Q5kPFzJTuOFwy7UniPli44NKQGAglksjvOcpo95aZA== -tinypool@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-0.6.0.tgz#c3640b851940abe2168497bb6e43b49fafb3ba7b" - integrity sha512-FdswUUo5SxRizcBc6b1GSuLpLjisa8N8qMyYoP3rl+bym+QauhtJP5bvZY1ytt8krKGmMLYIRl36HBZfeAoqhQ== +tinypool@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/tinypool/-/tinypool-0.7.0.tgz#88053cc99b4a594382af23190c609d93fddf8021" + integrity sha512-zSYNUlYSMhJ6Zdou4cJwo/p7w5nmAH17GRfU/ui3ctvjXFErXXkruT4MWW6poDeXgCaIBlGLrfU6TbTXxyGMww== tinyspy@^2.1.1: version "2.1.1" @@ -14723,10 +14723,10 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -vite-node@0.33.0: - version "0.33.0" - resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-0.33.0.tgz#c6a3a527e0b8090da7436241bc875760ae0eef28" - integrity sha512-19FpHYbwWWxDr73ruNahC+vtEdza52kA90Qb3La98yZ0xULqV8A5JLNPUff0f5zID4984tW7l3DH2przTJUZSw== +vite-node@0.34.2: + version "0.34.2" + resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-0.34.2.tgz#1daa025f8cee8a141c9b4d051e979cf61adaba2c" + integrity sha512-JtW249Zm3FB+F7pQfH56uWSdlltCo1IOkZW5oHBzeQo0iX4jtC7o1t9aILMGd9kVekXBP2lfJBEQt9rBh07ebA== dependencies: cac "^6.7.14" debug "^4.3.4" @@ -14746,19 +14746,19 @@ vite-node@0.33.0: optionalDependencies: fsevents "~2.3.2" -vitest@0.33.0: - version "0.33.0" - resolved "https://registry.yarnpkg.com/vitest/-/vitest-0.33.0.tgz#e2be6153aec1d30e3460ac6d64265bf72da2551c" - integrity sha512-1CxaugJ50xskkQ0e969R/hW47za4YXDUfWJDxip1hwbnhUjYolpfUn2AMOulqG/Dtd9WYAtkHmM/m3yKVrEejQ== +vitest@0.34.2: + version "0.34.2" + resolved "https://registry.yarnpkg.com/vitest/-/vitest-0.34.2.tgz#c90d563df18383f1749b8a4544adda1871bbc859" + integrity sha512-WgaIvBbjsSYMq/oiMlXUI7KflELmzM43BEvkdC/8b5CAod4ryAiY2z8uR6Crbi5Pjnu5oOmhKa9sy7uk6paBxQ== dependencies: "@types/chai" "^4.3.5" "@types/chai-subset" "^1.3.3" "@types/node" "*" - "@vitest/expect" "0.33.0" - "@vitest/runner" "0.33.0" - "@vitest/snapshot" "0.33.0" - "@vitest/spy" "0.33.0" - "@vitest/utils" "0.33.0" + "@vitest/expect" "0.34.2" + "@vitest/runner" "0.34.2" + "@vitest/snapshot" "0.34.2" + "@vitest/spy" "0.34.2" + "@vitest/utils" "0.34.2" acorn "^8.9.0" acorn-walk "^8.2.0" cac "^6.7.14" @@ -14771,9 +14771,9 @@ vitest@0.33.0: std-env "^3.3.3" strip-literal "^1.0.1" tinybench "^2.5.0" - tinypool "^0.6.0" + tinypool "^0.7.0" vite "^3.0.0 || ^4.0.0" - vite-node "0.33.0" + vite-node "0.34.2" why-is-node-running "^2.2.2" vm-browserify@^1.0.0: