#8953 Add GitHub Workflow for running e2e tests on pre-release browse…

…rs (#8982)

* add pre-release channels to playwright config

* convert channel map to flat map

* add projects as default imput to e2e-tests-workflow

* fix edge -> msedge

* validate chromium channels from E2E_CHROMIUM_CHANNELS

* add E2E_CHROMIUM_CHANNELS to end-to-end-tests.yml

* catch json parsing errors

* remove json parsing from end-to-end-tests.yml env variables

* fix json defined in single quotes

* introduce e2e-test-unstable-browsers workflow

* add push and workfow-dispatch to yml

* make browser installation dynamic

* remove magic strings in playwright.config.ts

* replace msedge conditionals with msedge & msedge beta

* replace msedge conditionals with msedge & msedge beta in sidebarAuth

* fix strict null error in getChromiumChannelsFromEnv

* replace remaining msedge magic string instances

* replace remaining magic strings for chrome

* rename workflow file

* fix lint error

* remove on push from workflow

* temporarily reenable push

* change workflow name again

* remove push again

* add .push back in to test workflow

* refactor extract DEFAULT_CHANNELS

* add env variable documentation for E2E_CHROMIUM_CHANNELS

* remove .push and add chrome and msedge

* fix type error

* fix supportedChannels type

* fix lint error

---------

Co-authored-by: Eduardo Fungairino <[email protected]>

.env.example

@@ -25,6 +25,8 @@ E2E_TEST_USER_EMAIL_UNAFFILIATED=extension-e2e-test.unaffiliated@pixiebrix.test
 E2E_TEST_USER_PASSWORD_UNAFFILIATED=
 E2E_TEST_USER_EMAIL_AFFILIATED=[email protected]
 E2E_TEST_USER_PASSWORD_AFFILIATED=
+# Optional configuration for running tests against specific browser versions
+# E2E_CHROMIUM_CHANNELS=["chrome-beta", "msedge-beta"]
 
 E2E_GOOGLE_TEST_USER_EMAIL=[email protected]
 # check https://start.1password.com/open/i?a=INRFJCYAHREKFCNIGK4M7FYRHM&h=pixiebrix.1password.com&i=xvn24iq54s4az7fpkskcjf7fte&v=os4xtpugongs7htmnsbvoeaujm for the password and otp key

.github/workflows/e2e-test-pre-release-browsers.yml

@@ -0,0 +1,16 @@
+name: Pre-release Browsers
+
+on:
+  schedule:
+    # Run nightly at 1:00 AM EST (5:00 UTC)
+    - cron: "0 5 * * *"
+  workflow_dispatch:
+
+jobs:
+  end-to-end-tests:
+    name: end-to-end-tests
+    uses: ./.github/workflows/end-to-end-tests.yml
+    secrets: inherit
+    with:
+      # Include stable browsers for comparison
+      projects: '["chrome-beta", "msedge-beta", "chromium", "chrome", "msedge"]'

.github/workflows/end-to-end-tests.yml

@@ -7,14 +7,19 @@ on:
         description: "Branch, tag, or commit on which to run the tests"
         required: false
         type: string
+      projects:
+        description: "Json parsable array of strings of projects to test. GitHub workflows does not support array inputs natively."
+        required: false
+        default: '["chrome", "msedge"]'
+        type: string
 
 jobs:
   run-tests:
     name: run-tests
     strategy:
       fail-fast: false
       matrix:
-        project: [chrome, edge]
+        project: ${{ fromJson(inputs.projects) }}
         shardIndex: [1, 2, 3]
         shardTotal: [3]
     timeout-minutes: 60
@@ -36,6 +41,7 @@ jobs:
       E2E_GOOGLE_TEST_USER_PASSWORD: ${{ secrets.E2E_GOOGLE_TEST_USER_PASSWORD }}
       E2E_GOOGLE_TEST_USER_OTP_KEY: ${{ secrets.E2E_GOOGLE_TEST_USER_OTP_KEY }}
       DEV_EVENT_TELEMETRY: true
+      E2E_CHROMIUM_CHANNELS: ${{ inputs.projects }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -49,7 +55,7 @@ jobs:
       - name: Install playwright browsers
         uses: Wandalen/[email protected]
         with:
-          command: npx playwright install chrome msedge
+          command: npx playwright install ${{ join(fromJson(inputs.projects), ' ') }}
           with: |
             fail_ci_if_error: true
             verbose: true

.secrets.baseline

@@ -152,13 +152,22 @@
         "line_number": 193
       }
     ],
+    ".github/workflows/e2e-test-pre-release-browsers.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": ".github/workflows/e2e-test-pre-release-browsers.yml",
+        "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
+        "is_verified": false,
+        "line_number": 13
+      }
+    ],
     ".github/workflows/end-to-end-tests.yml": [
       {
         "type": "Base64 High Entropy String",
         "filename": ".github/workflows/end-to-end-tests.yml",
         "hashed_secret": "97ed79340d21c99620980bc3a2ecf3a4064ae75c",
         "is_verified": false,
-        "line_number": 27
+        "line_number": 32
       }
     ],
     ".github/workflows/release.yaml": [
@@ -264,5 +273,5 @@
       }
     ]
   },
-  "generated_at": "2024-07-14T14:40:50Z"
+  "generated_at": "2024-08-09T23:11:01Z"
 }

end-to-end-tests/README.md

@@ -37,6 +37,9 @@ One-time setup:
   is resolved, you must include an unused playwright import as shown
   in [this test](https://github.com/pixiebrix/pixiebrix-extension/blob/7826c6549be0dbcbab32a8dfbaef472a3fdc22e9/end-to-end-tests/tests/workshopPageSmoke.spec.ts#L21)
   for the IDE to recognize the test)
+- You can optionally run tests against specific browsers and/or browser versions by setting environment variable
+  `E2E_CHROMIUM_CHANNELS`, which takes an array of channel values. Current expected values include `chrome`, `msedge`,
+  `chromium`, `chrome-beta`, and `msedge-beta`. See `.env.example` for example configuration.
 
 ## Writing Tests
 

end-to-end-tests/env.ts

@@ -39,6 +39,7 @@ const optionalEnvVariables = [
   "E2E_GOOGLE_TEST_USER_EMAIL",
   "E2E_GOOGLE_TEST_USER_PASSWORD",
   "E2E_GOOGLE_TEST_USER_OTP_KEY",
+  "E2E_CHROMIUM_CHANNELS",
 ] as const;
 
 type RequiredEnvVariables = Record<
@@ -82,4 +83,5 @@ export const {
   E2E_GOOGLE_TEST_USER_EMAIL,
   E2E_GOOGLE_TEST_USER_PASSWORD,
   E2E_GOOGLE_TEST_USER_OTP_KEY,
+  E2E_CHROMIUM_CHANNELS,
 } = process.env as OptionalEnvVariables;

end-to-end-tests/fixtures/authentication.ts

@@ -29,6 +29,10 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import * as os from "node:os";
 import { test as envSetup } from "./environmentCheck";
+import {
+  type SupportedChannel,
+  SupportedChannels,
+} from "../../playwright.config";
 
 const profileNameFromTestPath = (testFilePath: string) => {
   // Split the file path into directory and file name
@@ -58,10 +62,10 @@ export const test = mergeTests(
   envSetup,
   base.extend<{
     contextAndPage: { context: BrowserContext; page: Page };
-    chromiumChannel: "chrome" | "msedge";
+    chromiumChannel: SupportedChannel;
     additionalRequiredEnvVariables: string[];
   }>({
-    chromiumChannel: ["chrome", { option: true }],
+    chromiumChannel: [SupportedChannels.CHROME, { option: true }],
     additionalRequiredEnvVariables: [
       "REQUIRE_OPTIONAL_PERMISSIONS_IN_MANIFEST",
     ],

end-to-end-tests/fixtures/pageContext.ts

@@ -25,6 +25,10 @@ import {
 } from "./utils";
 import { ModsPage } from "../pageObjects/extensionConsole/modsPage";
 import { PageEditorPage } from "../pageObjects/pageEditor/pageEditorPage";
+import {
+  type SupportedChannel,
+  SupportedChannels,
+} from "../../playwright.config";
 
 // This environment variable is used to attach the browser sidepanel window that opens automatically to Playwright.
 // See https://github.com/microsoft/playwright/issues/26693
@@ -39,14 +43,14 @@ export const test = base.extend<
     context: BrowserContext;
     extensionId: string;
     profileName: "unaffiliated" | "affiliated";
-    chromiumChannel: "chrome" | "msedge";
+    chromiumChannel: SupportedChannel;
     newPageEditorPage: (urlToConnectTo: string) => Promise<PageEditorPage>;
   },
   {
     checkRequiredEnvironmentVariables: () => void;
   }
 >({
-  chromiumChannel: ["chrome", { option: true }],
+  chromiumChannel: [SupportedChannels.CHROME, { option: true }],
   profileName: "unaffiliated",
   async context({ chromiumChannel, profileName }, use) {
     let authSetupProfileDirectory: string;

end-to-end-tests/fixtures/utils.ts

@@ -18,9 +18,10 @@
 import { type BrowserContext, chromium } from "@playwright/test";
 import { CI, PWDEBUG, SLOWMO } from "../env";
 import path from "node:path";
+import { type SupportedChannel } from "../../playwright.config";
 
 export const launchPersistentContextWithExtension = async (
-  chromiumChannel: "chrome" | "msedge",
+  chromiumChannel: SupportedChannel,
   profileDirectory: string,
 ) => {
   const pathToExtension = path.join(__dirname, "../../dist");
@@ -67,7 +68,7 @@ export const getExtensionId = async (context: BrowserContext) => {
 
 export const getAuthProfilePathFile = (
   profileName: string,
-  chromiumChannel: "chrome" | "msedge",
+  chromiumChannel: SupportedChannel,
 ) =>
   path.join(
     __dirname,

end-to-end-tests/pageObjects/extensionsShortcutsPage.ts

@@ -18,20 +18,26 @@
 import { expect, type Page } from "@playwright/test";
 import { getModifierKey, getModifierSymbol } from "end-to-end-tests/utils";
 import { BasePageObject } from "./basePageObject";
+import {
+  type SupportedChannel,
+  SupportedChannels,
+} from "../../playwright.config";
 
-function getExtensionShortcutsUrl(chromiumChannel: "chrome" | "msedge") {
+function getExtensionShortcutsUrl(chromiumChannel: SupportedChannel) {
   switch (chromiumChannel) {
-    case "chrome": {
+    case SupportedChannels.CHROME:
+    case SupportedChannels.CHROME_BETA:
+    case SupportedChannels.CHROMIUM: {
       return "chrome://extensions/shortcuts";
     }
 
-    case "msedge": {
+    case SupportedChannels.MSEDGE:
+    case SupportedChannels.MSEDGE_BETA: {
       return "edge://extensions/shortcuts";
     }
 
     default: {
-      const exhaustiveCheck: never = chromiumChannel;
-      throw new Error(`Unexpected channel: ${exhaustiveCheck}`);
+      throw new Error(`Unexpected channel: ${chromiumChannel}`);
     }
   }
 }
@@ -48,7 +54,7 @@ export class ExtensionsShortcutsPage extends BasePageObject {
 
   constructor(
     page: Page,
-    private readonly chromiumChannel: "chrome" | "msedge",
+    private readonly chromiumChannel: SupportedChannel,
   ) {
     super(page);
     this.pageUrl = getExtensionShortcutsUrl(this.chromiumChannel);
@@ -61,7 +67,11 @@ export class ExtensionsShortcutsPage extends BasePageObject {
   async goto() {
     await this.page.goto(this.pageUrl);
 
-    if (this.chromiumChannel === "chrome") {
+    if (
+      [SupportedChannels.CHROME, SupportedChannels.CHROME_BETA].includes(
+        this.chromiumChannel,
+      )
+    ) {
       await expect(
         this.getByRole("heading", { name: /PixieBrix/ }),
       ).toBeVisible();
@@ -75,7 +85,11 @@ export class ExtensionsShortcutsPage extends BasePageObject {
 
     const shortcut = await getShortcut(this.page);
 
-    if (this.chromiumChannel === "chrome") {
+    if (
+      [SupportedChannels.CHROME, SupportedChannels.CHROME_BETA].includes(
+        this.chromiumChannel,
+      )
+    ) {
       await expect(this.page.getByPlaceholder(/shortcut set: /i)).toHaveValue(
         shortcut,
       );
@@ -112,7 +126,11 @@ export class ExtensionsShortcutsPage extends BasePageObject {
     const modifierKey = await getModifierKey(this.page);
     const shortcut = await getShortcut(this.page);
 
-    if (this.chromiumChannel === "chrome") {
+    if (
+      [SupportedChannels.CHROME, SupportedChannels.CHROME_BETA].includes(
+        this.chromiumChannel,
+      )
+    ) {
       await expect(
         this.getByLabel(/Shortcut Toggle Quick Bar for PixieBrix/),
       ).toBeEmpty();

end-to-end-tests/tests/extensionConsole/activation.spec.ts

@@ -31,6 +31,7 @@ import { type Serializable } from "playwright-core/types/structs";
 import { SERVICE_URL } from "../../env";
 import { ExtensionsShortcutsPage } from "../../pageObjects/extensionsShortcutsPage";
 import { FloatingActionButton } from "../../pageObjects/floatingActionButton";
+import { SupportedChannels } from "../../../playwright.config";
 
 test("can activate a mod with no config options", async ({
   page,
@@ -162,11 +163,17 @@ test("activating a mod when the quickbar shortcut is not configured", async ({
   await test.step("Clear the quickbar shortcut before activing a quickbar mod", async () => {
     const os = await getBrowserOs(firstTab);
     // See https://github.com/pixiebrix/pixiebrix-extension/issues/6268
-    // eslint-disable-next-line playwright/no-conditional-in-test -- Existing bug where shortcut isn't set on Edge in Windows/Linux
-    if (os === "MacOS" || chromiumChannel === "chrome") {
+    /* eslint-disable playwright/no-conditional-in-test -- Existing bug where shortcut isn't set on Edge in Windows/Linux */
+    if (
+      os === "MacOS" ||
+      [SupportedChannels.CHROME, SupportedChannels.CHROME_BETA].includes(
+        chromiumChannel,
+      )
+    ) {
       await shortcutsPage.clearQuickbarShortcut();
     }
   });
+  /* eslint-enable playwright/no-conditional-in-test */
 
   let modActivationPage: ActivateModPage;
   const secondTab = await context.newPage();