diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 228026a40a..fc7fae47c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,13 +3,16 @@ name: CI on: pull_request: branches: [main] - paths-ignore: - - "**.md" - - .gitignore - - .editorconfig - - LICENSE - - "**.iml" - - .idea/** + ## Ideally we could include the following `paths-ignore` option to avoid unnecessary runs, but + ## it conflicts with branch protection rules that always require successful CI job status checks + ## See: https://github.com/orgs/community/discussions/13690 + # paths-ignore: + # - "**.md" + # - .gitignore + # - .editorconfig + # - LICENSE + # - "**.iml" + # - .idea/** push: branches: - release/** diff --git a/.secrets.baseline b/.secrets.baseline index d4695b20ce..42448f353a 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -149,7 +149,7 @@ "filename": ".github/workflows/ci.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 193 + "line_number": 196 } ], ".github/workflows/e2e-test-pre-release-browsers.yml": [ @@ -273,5 +273,5 @@ } ] }, - "generated_at": "2024-09-13T21:02:21Z" + "generated_at": "2024-10-07T19:58:30Z" } diff --git a/end-to-end-tests/README.md b/end-to-end-tests/README.md index 9d5b8056a4..27a712f7a5 100644 --- a/end-to-end-tests/README.md +++ b/end-to-end-tests/README.md @@ -65,7 +65,8 @@ in the cli. Focus on testing high-level user behavior and integration points, avoiding duplication of unit test coverage. Each test should represent one full feature flow, which may include multiple steps and assertions. Avoid splitting -a single feature flow across multiple tests, preferring longer tests if necessary. +a single feature flow across multiple tests, preferring longer tests if necessary. Ensure that new tests are not flaky by running them +multiple times locally (you can use the `--repeat-each` option for convenience: `npm run test:e2e -- ``` +You can also pass in the run id instead if you want to view a specific run's report. + +```bash +./scripts/show-pr-e2e-report.sh -r +``` + You will need to have `7zz` and `gh` installed to run this script (install using brew). ```bash diff --git a/end-to-end-tests/fixtures/authentication.ts b/end-to-end-tests/fixtures/authentication.ts index 8362cd9d03..a9785810b0 100644 --- a/end-to-end-tests/fixtures/authentication.ts +++ b/end-to-end-tests/fixtures/authentication.ts @@ -85,7 +85,7 @@ export const test = mergeTests( ); // The admin console automatically opens a new tab to log in and link the newly installed extension to the user's account. - const page = await context.waitForEvent("page", { timeout: 10_000 }); + const page = await context.waitForEvent("page"); await use({ context, page }); diff --git a/end-to-end-tests/fixtures/utils.ts b/end-to-end-tests/fixtures/utils.ts index 58c58456b1..166169491d 100644 --- a/end-to-end-tests/fixtures/utils.ts +++ b/end-to-end-tests/fixtures/utils.ts @@ -48,9 +48,7 @@ export const launchPersistentContextWithExtension = async ( export const getExtensionId = async (context: BrowserContext) => { const background = context.serviceWorkers()[0] || - (await context.waitForEvent("serviceworker", { - timeout: 3000, - })); + (await context.waitForEvent("serviceworker")); const extensionId = background.url().split("/")[2]; diff --git a/end-to-end-tests/pageObjects/extensionConsole/modsPage.ts b/end-to-end-tests/pageObjects/extensionConsole/modsPage.ts index 7508bfd481..286d497e6e 100644 --- a/end-to-end-tests/pageObjects/extensionConsole/modsPage.ts +++ b/end-to-end-tests/pageObjects/extensionConsole/modsPage.ts @@ -21,6 +21,7 @@ import { BasePageObject } from "../basePageObject"; import { ensureVisibility } from "../../utils"; import { validateRegistryId } from "@/types/helpers"; import { API_PATHS, UI_PATHS } from "@/data/service/urlPaths"; +import { DEFAULT_TIMEOUT } from "../../../playwright.config"; export class ModTableItem extends BasePageObject { dropdownButton = this.getByTestId("ellipsis-menu-button"); @@ -28,18 +29,29 @@ export class ModTableItem extends BasePageObject { statusCell = this.getByTestId("status-cell"); async clickAction(actionName: string) { - // Wrapped in `toPass` due to flakiness with dropdown visibility - // TODO: https://github.com/pixiebrix/pixiebrix-extension/issues/8458 + // Wrapped in `toPass` due to flakiness with dropdown visibility due to component remounting await expect(async () => { if (!(await this.dropdownMenu.isVisible())) { + await this.dropdownButton.click({ + timeout: 5000, + }); + } + + try { + await this.getByRole("menuitem", { name: actionName }).waitFor({ + timeout: 5000, + }); + } catch (error) { + // Sometimes the action is not visible because the permissions network request has not completed. + // Close the dropdown menu if the action is not visible, and try opening again. await this.dropdownButton.click(); + throw error; } await this.getByRole("menuitem", { name: actionName }).click({ - // Short timeout in order to handle retrying in the `toPass` block. - timeout: 500, + timeout: 5000, }); - }).toPass({ timeout: 5000 }); + }).toPass({ timeout: DEFAULT_TIMEOUT }); } } @@ -74,7 +86,7 @@ export class ModsPage extends BasePageObject { const contentLoadedLocator = this.getByText("Welcome to PixieBrix!").or( this.modTableItems.nth(0), ); - await expect(contentLoadedLocator).toBeVisible({ timeout: 15_000 }); + await expect(contentLoadedLocator).toBeVisible(); } async viewAllMods() { @@ -162,9 +174,7 @@ export class ActivateModPage extends BasePageObject { this.getByRole("heading", { name: "Activate " }), ).toBeVisible(); // Loading the mod details may take a long time. Using ensureVisibility because the modId may be attached and hidden - await ensureVisibility(this.getByText(this.modId), { - timeout: 10_000, - }); + await ensureVisibility(this.getByText(this.modId)); } async getIntegrationConfigField(index: number) { @@ -190,9 +200,7 @@ export class ActivateModPage extends BasePageObject { const modsPage = new ModsPage(this.page, this.extensionId); await modsPage.viewActiveMods(); // Loading mods sometimes takes upwards of 10s - await expect(modsPage.modTableItems.getByText(this.modId)).toBeVisible({ - timeout: 10_000, - }); + await expect(modsPage.modTableItems.getByText(this.modId)).toBeVisible(); return modsPage; } diff --git a/end-to-end-tests/pageObjects/extensionConsole/workshop/workshopPage.ts b/end-to-end-tests/pageObjects/extensionConsole/workshop/workshopPage.ts index 6b418db4f7..975ca1b36a 100644 --- a/end-to-end-tests/pageObjects/extensionConsole/workshop/workshopPage.ts +++ b/end-to-end-tests/pageObjects/extensionConsole/workshop/workshopPage.ts @@ -42,6 +42,7 @@ export class WorkshopPage extends BasePageObject { async findAndSelectMod(modId: string) { await this.getByPlaceholder("Start typing to find results").fill(modId); + await this.getByRole("cell", { name: modId }).waitFor(); await this.getByRole("cell", { name: modId }).click(); const editPage = new EditWorkshopModPage(this.page); @@ -56,9 +57,7 @@ export class WorkshopPage extends BasePageObject { const modMedata = await createPage.editor.replaceWithModDefinition(modDefinitionName); await createPage.createBrickButton.click(); - await expect(this.getByRole("status").getByText("Created ")).toBeVisible({ - timeout: 8000, - }); + await expect(this.getByRole("status").getByText("Created ")).toBeVisible(); return modMedata; } diff --git a/end-to-end-tests/pageObjects/pageEditor/pageEditorPage.ts b/end-to-end-tests/pageObjects/pageEditor/pageEditorPage.ts index 50ee50f44b..6b334a306c 100644 --- a/end-to-end-tests/pageObjects/pageEditor/pageEditorPage.ts +++ b/end-to-end-tests/pageObjects/pageEditor/pageEditorPage.ts @@ -30,6 +30,7 @@ import { ModifiesModFormState } from "./utils"; import { CreateModModal } from "./createModModal"; import { DeactivateModModal } from "end-to-end-tests/pageObjects/pageEditor/deactivateModModal"; import { uuidv4 } from "@/types/helpers"; +import { DEFAULT_TIMEOUT } from "../../../playwright.config"; class EditorPane extends BasePageObject { editTab = this.getByRole("tab", { name: "Edit" }); @@ -160,15 +161,16 @@ export class PageEditorPage extends BasePageObject { "Save the mod to assign a Mod ID", ), ).toBeVisible(); - // eslint-disable-next-line playwright/no-wait-for-timeout -- The save button re-mounts several times so we need a slight delay here before playwright clicks - await this.page.waitForTimeout(2000); - await modListItem.saveButton.click(); - // Handle the "Save new mod" modal const saveNewModModal = this.page.locator(".modal-content"); - await expect(saveNewModModal).toBeVisible(); - await expect(saveNewModModal.getByText("Save new mod")).toBeVisible(); + // The save button re-mounts several times so we need to retry clicking the saveButton until the modal is visible + // See: https://github.com/pixiebrix/pixiebrix-extension/issues/9266 + await expect(async () => { + await modListItem.saveButton.click(); + await expect(saveNewModModal).toBeVisible({ timeout: 5000 }); + }).toPass({ timeout: DEFAULT_TIMEOUT }); + await expect(saveNewModModal.getByText("Save new mod")).toBeVisible(); // // Can't use getByLabel to target because the field is composed of multiple widgets const registryIdInput = saveNewModModal.getByTestId("registryId-id-id"); const currentId = await registryIdInput.inputValue(); diff --git a/end-to-end-tests/setup/affiliated.setup.ts b/end-to-end-tests/setup/affiliated.setup.ts index 053d831903..68e18ac367 100644 --- a/end-to-end-tests/setup/affiliated.setup.ts +++ b/end-to-end-tests/setup/affiliated.setup.ts @@ -45,7 +45,6 @@ test("authenticate with affiliated user", async ({ page.getByText( "Successfully linked the Browser Extension to your PixieBrix account", ), - { timeout: 12_000 }, ); await expect(page.getByText(E2E_TEST_USER_EMAIL_AFFILIATED)).toBeVisible(); await expect(page.getByText("Admin Console")).toBeVisible(); @@ -54,7 +53,7 @@ test("authenticate with affiliated user", async ({ // We need to wait for a couple of seconds to ensure that the deployment is activated in the bg script to avoid flakiness // when loading the extension console. If the deployment is not activated, then a modal will pop up prompting for activation. // eslint-disable-next-line playwright/no-wait-for-timeout -- no easy way to detect when the bg script is done activating the deployment - await page.waitForTimeout(3000); + await page.waitForTimeout(5000); let extensionConsolePage: Page; await test.step("Open Extension Console", async () => { diff --git a/end-to-end-tests/setup/unaffiliated.setup.ts b/end-to-end-tests/setup/unaffiliated.setup.ts index ca4bb90215..0d32ac17b3 100644 --- a/end-to-end-tests/setup/unaffiliated.setup.ts +++ b/end-to-end-tests/setup/unaffiliated.setup.ts @@ -47,7 +47,6 @@ test("authenticate with unaffiliated user", async ({ page.getByText( "Successfully linked the Browser Extension to your PixieBrix account", ), - { timeout: 12_000 }, ); await expect( page.getByText(E2E_TEST_USER_EMAIL_UNAFFILIATED), @@ -74,7 +73,7 @@ test("authenticate with unaffiliated user", async ({ ); await localIntegrationsPage.goto(); - const popupPromise = context.waitForEvent("page", { timeout: 5000 }); + const popupPromise = context.waitForEvent("page"); await localIntegrationsPage.createNewIntegration("Google Drive"); const googleAuthPopup = await popupPromise; diff --git a/end-to-end-tests/setup/utils.ts b/end-to-end-tests/setup/utils.ts index 20928d9018..3cf1ae84d8 100644 --- a/end-to-end-tests/setup/utils.ts +++ b/end-to-end-tests/setup/utils.ts @@ -31,7 +31,7 @@ export const openExtensionConsoleFromAdmin = async ( await adminPage .locator("button") .filter({ hasText: "Open Extension Console" }) - .click(); + .click({ timeout: 5000 }); extensionConsolePage = context .pages() @@ -40,12 +40,14 @@ export const openExtensionConsoleFromAdmin = async ( if (!extensionConsolePage) { throw new Error("Extension console page not found"); } + }).toPass({ timeout: 20_000 }); - await expect(extensionConsolePage.locator("#container")).toContainText( - "Extension Console", - ); - await expect(extensionConsolePage.getByText(userName)).toBeVisible(); - }).toPass({ timeout: 15_000 }); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- extensionConsolePage is defined + await expect(extensionConsolePage!.locator("#container")).toContainText( + "Extension Console", + ); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- extensionConsolePage is defined + await expect(extensionConsolePage!.getByText(userName)).toBeVisible(); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- extensionConsolePage is defined return extensionConsolePage!; diff --git a/end-to-end-tests/tests/bricks/sidebarEffects.spec.ts b/end-to-end-tests/tests/bricks/sidebarEffects.spec.ts index f1e68dfff3..d2b75e5073 100644 --- a/end-to-end-tests/tests/bricks/sidebarEffects.spec.ts +++ b/end-to-end-tests/tests/bricks/sidebarEffects.spec.ts @@ -20,6 +20,7 @@ import { ActivateModPage } from "../../pageObjects/extensionConsole/modsPage"; // @ts-expect-error -- https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only import { type Page, test as base } from "@playwright/test"; import { runModViaQuickBar, getSidebarPage, isSidebarOpen } from "../../utils"; +import { DEFAULT_TIMEOUT } from "../../../playwright.config"; test.describe("sidebar effect bricks", () => { test("toggle sidebar brick", async ({ page, extensionId }) => { @@ -46,6 +47,6 @@ test.describe("sidebar effect bricks", () => { await expect(() => { expect(isSidebarOpen(page, extensionId)).toBe(false); - }).toPass({ timeout: 5000 }); + }).toPass({ timeout: DEFAULT_TIMEOUT }); }); }); diff --git a/end-to-end-tests/tests/extensionConsole/activation.spec.ts b/end-to-end-tests/tests/extensionConsole/activation.spec.ts index c13153e4b6..523efa9bd3 100644 --- a/end-to-end-tests/tests/extensionConsole/activation.spec.ts +++ b/end-to-end-tests/tests/extensionConsole/activation.spec.ts @@ -32,6 +32,7 @@ import { type Serializable } from "playwright-core/types/structs"; import { SERVICE_URL } from "../../env"; import { ExtensionsShortcutsPage } from "../../pageObjects/extensionsShortcutsPage"; import { FloatingActionButton } from "../../pageObjects/floatingActionButton"; +import { DEFAULT_TIMEOUT } from "../../../playwright.config"; test("can activate a mod with no config options", async ({ page, @@ -121,7 +122,6 @@ test("can activate a mod with built-in integration", async ({ test("validates activating a mod with required integrations", async ({ page, extensionId, - context, }) => { const modId = "@e2e-testing/summarize-text-open-ai"; const modActivationPage = new ActivateModPage(page, extensionId, modId); @@ -160,13 +160,18 @@ test("can activate a mod with a database", async ({ page, extensionId }) => { await expect(sideBarPage.getByTestId("card").getByText(note)).toBeVisible(); - // Get the correct container element, as the note text and delete button are wrapped in a div - await sideBarPage - .getByText(`${note} Delete Note`) - .getByRole("button", { name: "Delete Note" }) - .click(); - - await expect(sideBarPage.getByTestId("card").getByText(note)).toBeHidden(); + // Wrapped in a toPass block in case the delete button isn't clicked successfully due to page shifting + await expect(async () => { + // Get the correct container element, as the note text and delete button are wrapped in a div + await sideBarPage + .getByText(`${note} Delete Note`) + .getByRole("button", { name: "Delete Note" }) + .click(); + + await expect(sideBarPage.getByTestId("card").getByText(note)).toBeHidden({ + timeout: 5000, + }); + }).toPass({ timeout: DEFAULT_TIMEOUT }); }); test("activating a mod when the quickbar shortcut is not configured", async ({ @@ -240,14 +245,10 @@ test("can activate a mod via url", async ({ page, extensionId }) => { await page.goto(activationLink); - await expect(async () => { - await expect(page).toHaveURL( - `chrome-extension://${extensionId}/options.html#/marketplace/activate/${modIdUrlEncoded}`, - ); - }).toPass({ timeout: 5000 }); - await expect(page.getByRole("code")).toContainText(modId, { - timeout: 10_000, - }); + await page.waitForURL( + `chrome-extension://${extensionId}/options.html#/marketplace/activate/${modIdUrlEncoded}`, + ); + await expect(page.getByRole("code")).toContainText(modId); const modActivationPage = new ActivateModPage(page, extensionId, modId); await modActivationPage.clickActivateAndWaitForModsPageRedirect(); diff --git a/end-to-end-tests/tests/extensionConsole/modsPage.spec.ts b/end-to-end-tests/tests/extensionConsole/modsPage.spec.ts index b172e79238..de3ecf5cce 100644 --- a/end-to-end-tests/tests/extensionConsole/modsPage.spec.ts +++ b/end-to-end-tests/tests/extensionConsole/modsPage.spec.ts @@ -19,6 +19,7 @@ import { expect, test } from "../../fixtures/testBase"; import { ModsPage } from "../../pageObjects/extensionConsole/modsPage"; // @ts-expect-error -- https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only import { test as base } from "@playwright/test"; +import { DEFAULT_TIMEOUT } from "../../../playwright.config"; test("can open mod in the workshop", async ({ page, extensionId }) => { const modId = "@e2e-testing/shared-notes-sidebar"; @@ -32,6 +33,6 @@ test("can open mod in the workshop", async ({ page, extensionId }) => { expect(pageTitle).toBe("Edit [Testing] St... | PixieBrix"); }).toPass({ // Clicking Edit in Workshop fetches editable packages to determine the surrogate package id - timeout: 10_000, + timeout: DEFAULT_TIMEOUT, }); }); diff --git a/end-to-end-tests/tests/pageEditor/addStarterBrick.spec.ts b/end-to-end-tests/tests/pageEditor/addStarterBrick.spec.ts index 672a24badb..8cf9ff0326 100644 --- a/end-to-end-tests/tests/pageEditor/addStarterBrick.spec.ts +++ b/end-to-end-tests/tests/pageEditor/addStarterBrick.spec.ts @@ -157,6 +157,11 @@ test("Add starter brick to mod", async ({ verifyModDefinitionSnapshot, chromiumChannel, }) => { + test.slow( + true, + "Longer test due to verifying each starter brick definition in one user flow", + ); + await page.goto("/"); const pageEditorPage = await newPageEditorPage(page.url()); const brickPipeline = pageEditorPage.brickActionsPanel.bricks; diff --git a/end-to-end-tests/tests/pageEditor/liveEditing.spec.ts b/end-to-end-tests/tests/pageEditor/liveEditing.spec.ts index d3aaea88e4..68dd2e471f 100644 --- a/end-to-end-tests/tests/pageEditor/liveEditing.spec.ts +++ b/end-to-end-tests/tests/pageEditor/liveEditing.spec.ts @@ -19,27 +19,15 @@ import { expect, test } from "../../fixtures/testBase"; // @ts-expect-error -- https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only import { test as base } from "@playwright/test"; import { ActivateModPage } from "../../pageObjects/extensionConsole/modsPage"; -import { - getSidebarPage, - isMsEdge, - isSidebarOpen, - PRE_RELEASE_BROWSER_WORKFLOW_NAME, -} from "../../utils"; +import { getSidebarPage, isMsEdge, isSidebarOpen } from "../../utils"; +import { FloatingActionButton } from "../../pageObjects/floatingActionButton"; test("live editing behavior", async ({ page, extensionId, - modDefinitionsMap, newPageEditorPage, - verifyModDefinitionSnapshot, chromiumChannel, }) => { - test.fixme( - process.env.GITHUB_WORKFLOW === PRE_RELEASE_BROWSER_WORKFLOW_NAME && - isMsEdge(chromiumChannel), - "Skipping test for MS Edge in pre-release workflow, see https://github.com/pixiebrix/pixiebrix-extension/issues/9125", - ); - await test.step("Activate test mod and navigate to testing site", async () => { const modId = "@e2e-testing/page-editor-live-editing-test"; const activationPage = new ActivateModPage(page, extensionId, modId); @@ -87,6 +75,16 @@ test("live editing behavior", async ({ expect(isSidebarOpen(page, extensionId)).toBe(false); + /* eslint-disable-next-line playwright/no-conditional-in-test -- MS Edge has a bug where the page editor + * cannot open the sidebar, so we need to open it manually. + * https://www.loom.com/share/fbad85e901794161960b737b27a13677 + */ + if (isMsEdge(chromiumChannel)) { + await page.bringToFront(); + const floatingActionButton = new FloatingActionButton(page); + await floatingActionButton.toggleSidebar(); + } + await pageEditor.editorPane.renderPanelButton.click(); const sidebar = await getSidebarPage(page, extensionId); await expect( diff --git a/end-to-end-tests/tests/regressions/doNotCloseSidebarOnPageEditorSave.spec.ts b/end-to-end-tests/tests/regressions/doNotCloseSidebarOnPageEditorSave.spec.ts index 94b5a1e082..a90893e692 100644 --- a/end-to-end-tests/tests/regressions/doNotCloseSidebarOnPageEditorSave.spec.ts +++ b/end-to-end-tests/tests/regressions/doNotCloseSidebarOnPageEditorSave.spec.ts @@ -18,13 +18,14 @@ import { test, expect } from "../../fixtures/testBase"; // @ts-expect-error -- https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only import { type Page, test as base } from "@playwright/test"; -import { getSidebarPage } from "../../utils"; +import { getSidebarPage, isMsEdge } from "../../utils"; +import { FloatingActionButton } from "../../pageObjects/floatingActionButton"; -// eslint-disable-next-line playwright/no-skipped-test -- There is a race condition that makes this flaky, we will fix later -test.skip("#8104: Do not automatically close the sidebar when saving in the Page Editor", async ({ +test("#8104: Do not automatically close the sidebar when saving in the Page Editor", async ({ page, newPageEditorPage, extensionId, + chromiumChannel, }) => { await page.goto("/"); const pageEditorPage = await newPageEditorPage(page.url()); @@ -33,6 +34,16 @@ test.skip("#8104: Do not automatically close the sidebar when saving in the Page starterBrickName: "Sidebar Panel", }); + /* eslint-disable-next-line playwright/no-conditional-in-test -- MS Edge has a bug where the page editor + * cannot open the sidebar, so we need to open it manually. + * https://www.loom.com/share/fbad85e901794161960b737b27a13677 + */ + if (isMsEdge(chromiumChannel)) { + await page.bringToFront(); + const floatingActionButton = new FloatingActionButton(page); + await floatingActionButton.toggleSidebar(); + } + const sidebar = await getSidebarPage(page, extensionId); await expect( sidebar.getByRole("tab", { name: "Sidebar Panel" }), diff --git a/end-to-end-tests/tests/regressions/sidebarLinks.spec.ts b/end-to-end-tests/tests/regressions/sidebarLinks.spec.ts index 044e2dfcf2..6c6b45ab55 100644 --- a/end-to-end-tests/tests/regressions/sidebarLinks.spec.ts +++ b/end-to-end-tests/tests/regressions/sidebarLinks.spec.ts @@ -121,7 +121,7 @@ test("#8206: clicking links from the sidebar doesn't crash browser", async ({ name: "Active Mods", }); // `activeModsHeading` may be initially be detached and hidden, so toBeVisible() would immediately fail - await ensureVisibility(activeModsHeading, { timeout: 10_000 }); + await ensureVisibility(activeModsHeading); }); await test.step("Clicking markdown text link", async () => { diff --git a/end-to-end-tests/tests/regressions/welcomeStarterBricks.spec.ts b/end-to-end-tests/tests/regressions/welcomeStarterBricks.spec.ts index f3f0241138..53c0831b6f 100644 --- a/end-to-end-tests/tests/regressions/welcomeStarterBricks.spec.ts +++ b/end-to-end-tests/tests/regressions/welcomeStarterBricks.spec.ts @@ -50,7 +50,15 @@ test("#8740: can view the starter mods on the pixiebrix.com/welcome page", async const sideBarPage = await getSidebarPage(page, extensionId); - await expect(sideBarPage.getByRole("tab", { name: "Mods" })).toBeVisible(); + try { + await expect(sideBarPage.getByRole("tab", { name: "Mods" })).toBeVisible(); + } catch { + // There seems to be a race condition where a welcome mod is already loaded, which will then hide the Mod launcher tab + // If this happens, we can open the mod launcher tab directly + await sideBarPage.getByLabel("Open Mod Launcher").click(); + // eslint-disable-next-line playwright/no-conditional-expect -- see above comment + await expect(sideBarPage.getByRole("tab", { name: "Mods" })).toBeVisible(); + } await expect( sideBarPage.locator(".list-group").getByRole("heading").first(), diff --git a/end-to-end-tests/tests/runtime/sidebar/sidebarController.spec.ts b/end-to-end-tests/tests/runtime/sidebar/sidebarController.spec.ts index 1b701a861c..740e5fdee8 100644 --- a/end-to-end-tests/tests/runtime/sidebar/sidebarController.spec.ts +++ b/end-to-end-tests/tests/runtime/sidebar/sidebarController.spec.ts @@ -41,7 +41,7 @@ test.describe("sidebar controller", () => { const frame = page.frameLocator("iframe"); await frame.getByRole("link", { name: "Show Sidebar Immediately" }).click(); - // Don't use getSidebarPage because it automatically clicks the MV3 focus dialog. + // Don't use getSidebarPage because it automatically clicks the focus dialog. await expect(() => { expect(isSidebarOpen(page, extensionId)).toBe(false); }).toPass({ timeout: 5000 }); diff --git a/end-to-end-tests/tests/smoke/floatingActionButton.spec.ts b/end-to-end-tests/tests/smoke/floatingActionButton.spec.ts index daac26e6cf..51a4ae1b4a 100644 --- a/end-to-end-tests/tests/smoke/floatingActionButton.spec.ts +++ b/end-to-end-tests/tests/smoke/floatingActionButton.spec.ts @@ -19,6 +19,7 @@ import { test, expect } from "../../fixtures/testBase"; import { ActivateModPage } from "../../pageObjects/extensionConsole/modsPage"; import { FloatingActionButton } from "../../pageObjects/floatingActionButton"; import { getSidebarPage, isSidebarOpen } from "../../utils"; +import { DEFAULT_TIMEOUT } from "../../../playwright.config"; test.describe("sidebar page smoke test", () => { test("can toggle the sidebar from the floating action button and view the related mod's sidebar panel", async ({ @@ -45,7 +46,7 @@ test.describe("sidebar page smoke test", () => { await expect(() => { expect(isSidebarOpen(page, extensionId)).toBe(false); - }).toPass({ timeout: 5000 }); + }).toPass({ timeout: DEFAULT_TIMEOUT }); }); test("can hide the floating action button", async ({ page, extensionId }) => { @@ -53,7 +54,7 @@ test.describe("sidebar page smoke test", () => { const floatingActionButton = new FloatingActionButton(page); const actionButton = await floatingActionButton.getActionButton(); - await expect(actionButton).toBeVisible({ timeout: 10_000 }); + await expect(actionButton).toBeVisible(); await floatingActionButton.hideFloatingActionButton(); diff --git a/end-to-end-tests/utils.ts b/end-to-end-tests/utils.ts index 5a0873f4c2..e013f54686 100644 --- a/end-to-end-tests/utils.ts +++ b/end-to-end-tests/utils.ts @@ -23,7 +23,11 @@ import { type BrowserContext, } from "@playwright/test"; import { TOTP } from "otpauth"; -import { type SupportedChannel, SupportedChannels } from "../playwright.config"; +import { + DEFAULT_TIMEOUT, + type SupportedChannel, + SupportedChannels, +} from "../playwright.config"; export const PRE_RELEASE_BROWSER_WORKFLOW_NAME = "Pre-release Browsers"; @@ -72,7 +76,7 @@ export async function ensureVisibility( ) { await expect(async () => { await expect(locator).toBeVisible({ timeout: 0 }); // Retry handling is done by the outer expect - }).toPass({ timeout: 5000, ...options }); + }).toPass({ timeout: DEFAULT_TIMEOUT, ...options }); } // Run a mod via the Quickbar. @@ -134,7 +138,9 @@ export async function getSidebarPage( // The sidebar sometimes requires the user to interact with modal to open the sidebar via a user gesture const conditionallyPerformUserGesture = async () => { - await page.getByRole("button", { name: "Open Sidebar" }).click(); + await page + .getByRole("button", { name: "Open Sidebar" }) + .click({ timeout: 5000 }); return findSidebarPage(page, extensionId); }; @@ -144,7 +150,7 @@ export async function getSidebarPage( findSidebarPage(page, extensionId), ]); expect(sidebarPage).toBeDefined(); - }).toPass({ timeout: 5000 }); + }).toPass({ timeout: DEFAULT_TIMEOUT }); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- checked above return sidebarPage!; diff --git a/playwright.config.ts b/playwright.config.ts index b99b0baf9e..cb5e85c6f9 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -53,6 +53,9 @@ const getChromiumChannelsFromEnv = (): SupportedChannel[] => { }); }; +/** Default timeout used for each action and assertion */ +export const DEFAULT_TIMEOUT = 20_000; + /** * See https://playwright.dev/docs/test-configuration. */ @@ -73,8 +76,16 @@ export default defineConfig<{ chromiumChannel: string }>({ /* Timeout for the entire test run */ globalTimeout: 30 * 60 * 1000, // 30 minutes expect: { - /* Timeout for each assertion. If a particular interaction is timing out, adjust its specific timeout value rather than this global setting */ - timeout: 5000, + /** + * Timeout for each assertion. If a particular interaction is timing out, adjust its specific timeout value rather than this global setting. + * + * Set to 20s due to spikes in API latency. See example traces: + * GET api/bricks/ + * https://app.datadoghq.com/apm/trace/1816851494275985657?graphType=flamegraph&shouldShowLegend=true&sort=time&spanID=14068679180114270950&timeHint=1728332087443.742 + * POST api/bricks/ + * https://app.datadoghq.com/apm/trace/7735170839924641545?graphType=flamegraph&shouldShowLegend=true&sort=time&spanID=13697932419891897088&timeHint=1728331856832.3618 + */ + timeout: DEFAULT_TIMEOUT, toHaveScreenshot: { maxDiffPixelRatio: 0.1, }, @@ -86,6 +97,8 @@ export default defineConfig<{ chromiumChannel: string }>({ ["html", { outputFolder: "./end-to-end-tests/.report" }], ["json", { outputFile: "./end-to-end-tests/.report/report.json" }], ], + // /* Repeat each test N times. Useful for catching flaky test. */ + // repeatEach: 3, /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ @@ -95,7 +108,7 @@ export default defineConfig<{ chromiumChannel: string }>({ trace: CI ? "on-first-retry" : "retain-on-failure", /* Set the default timeout for actions such as `click` */ - actionTimeout: 5000, + actionTimeout: DEFAULT_TIMEOUT, /* Set the default timeout for page navigations */ navigationTimeout: 10_000, diff --git a/scripts/show-pr-e2e-report.sh b/scripts/show-pr-e2e-report.sh index 8d8046ca73..5eae5cc322 100755 --- a/scripts/show-pr-e2e-report.sh +++ b/scripts/show-pr-e2e-report.sh @@ -28,19 +28,22 @@ then fi # Check if a named argument was provided -while getopts ":p:" opt; do +while getopts ":p:r:" opt; do case $opt in p) PR_ID="$OPTARG" ;; + r) RUN_ID="$OPTARG" + ;; \?) echo "Invalid option -$OPTARG" >&2 + exit 1 ;; esac done -# Check if PR_ID is set -if [ -z "$PR_ID" ] +# Check if either PR_ID or RUN_ID is set +if [ -z "$PR_ID" ] && [ -z "$RUN_ID" ] then - echo "Pull request ID is required. Use -p to provide it." + echo "Either pull request ID (-p) or run ID (-r) is required." exit 1 fi @@ -48,8 +51,13 @@ fi rm -rf .playwright-report/* mkdir .playwright-report +# Get the RUN_ID if PR_ID was provided +if [ -n "$PR_ID" ] +then + RUN_ID=$(gh pr checks "$PR_ID" --repo pixiebrix/pixiebrix-extension --json 'name,link' --jq ".[] | select(.name == \"Create report\") | .link" | cut -d'/' -f8) +fi + # Get the URL of the playwright-report artifact from the GitHub API -RUN_ID=$(gh pr checks "$PR_ID" --repo pixiebrix/pixiebrix-extension --json 'name,link' --jq ".[] | select(.name == \"Create report\") | .link" | cut -d'/' -f8) ARTIFACT_URL=$(gh api "repos/pixiebrix/pixiebrix-extension/actions/runs/$RUN_ID" --jq ".artifacts_url" | xargs gh api --jq ".artifacts[] | select(.name == \"end-to-end-tests-report\") | .archive_download_url") echo "Artifact URL: $ARTIFACT_URL"