🧪 test(ci): harden Cucumber reliability

.github/tests/ci-verify-workflow.test.ts

@@ -179,55 +179,100 @@ test('ci-verify can dispatch the complete release-candidate matrix manually', ()
   expect(workflow.jobs?.['load-test-behavior']?.needs).toStrictEqual(['build']);
 });
 
-test('ci-verify runs Cucumber in the pinned Playwright browser image', () => {
+test('ci-verify keeps Cucumber browser-free and retries only dependency installation', () => {
   const workflow = loadWorkflow();
-  const playwrightImage =
-    'mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48';
+  // Worst-case bounded retries plus readiness leave enough time for the
+  // suite and post-failure artifact handling.
+  expect(workflow.jobs?.e2e?.['timeout-minutes']).toBe(40);
+  expect(workflow.jobs?.e2e?.permissions).toStrictEqual({
+    actions: 'read',
+    contents: 'read',
+  });
 
-  // Keep host-side installs from downloading a browser. The Cucumber browser
-  // comes from the same immutable image used by the dedicated Playwright gate.
+  // Cucumber owns API/WebSocket contracts. Browser behavior is covered by the
+  // separately release-gated Playwright workflow.
   expect(workflow.env?.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD).toBeUndefined();
 
   for (const jobId of ['load-test-ci', 'load-test-behavior']) {
     expect(workflow.jobs?.[jobId]?.env?.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD).toBe('1');
     expect(getWorkflowStep(jobId, 'Install e2e dependencies')).toBeDefined();
   }
 
-  expect(workflow.jobs?.e2e?.env).toMatchObject({
-    PLAYWRIGHT_IMAGE: playwrightImage,
+  expect(workflow.jobs?.e2e?.env).toStrictEqual({
     PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1',
   });
   expect(getWorkflowStep('e2e', 'Cache Playwright browsers')).toBeUndefined();
-  expect(getWorkflowStep('e2e', 'Install e2e dependencies')).toBeUndefined();
-
-  const verifyImage = getWorkflowStep('e2e', 'Verify Playwright image matches package')?.run;
-  expect(verifyImage).toContain(
-    "require('./e2e/package.json').devDependencies['@playwright/test']",
+  expect(getWorkflowStep('e2e', 'Verify Playwright image matches package')).toBeUndefined();
+  expect(getWorkflowStep('e2e', 'Pull Playwright container')).toBeUndefined();
+  expect(getWorkflowStep('e2e', 'Download QA image')).toMatchObject({
+    uses: expectedActionUse('actions/download-artifact'),
+    with: {
+      name: 'qa-image-${{ github.run_id }}',
+      path: 'artifacts/qa',
+    },
+  });
+  expect(getWorkflowStep('e2e', 'Load QA image')?.run).toBe(
+    'docker load < artifacts/qa/drydock-dev-image.tar.gz',
   );
-  expect(verifyImage).toContain('PLAYWRIGHT_IMAGE uses $image_version');
 
-  expect(getWorkflowStep('e2e', 'Pull Playwright container')).toMatchObject({
+  expect(getWorkflowStep('e2e', 'Start drydock')).toMatchObject({
+    env: {
+      DD_E2E_IMAGE: 'drydock:dev',
+      DD_E2E_SKIP_BUILD: 'true',
+    },
+  });
+
+  const install = getWorkflowStep('e2e', 'Install e2e dependencies');
+  expect(install).toMatchObject({
+    uses: expectedActionUse('nick-fields/retry'),
     with: {
-      command: 'docker pull "$PLAYWRIGHT_IMAGE"',
+      timeout_minutes: 3,
+      max_attempts: 3,
+    },
+  });
+  expect(install?.with?.command).toBe('cd e2e && npm ci --no-audit --no-fund');
+  expect(install?.with?.command).not.toContain('npm run cucumber');
+  expect(getWorkflowStep('e2e', 'Run Cucumber support tests')).toMatchObject({
+    run: 'npm run test:support',
+    'working-directory': 'e2e',
+  });
+  expect(getWorkflowStep('e2e', 'Setup test containers')).toMatchObject({
+    with: {
+      timeout_minutes: 6,
+      max_attempts: 2,
     },
   });
 
   const cucumber = getWorkflowStep('e2e', 'Run Cucumber e2e tests');
-  expect(cucumber?.['working-directory']).toBeUndefined();
-  expect(cucumber?.run).toContain('docker run --rm');
-  expect(cucumber?.run).toContain('--network host');
-  expect(cucumber?.run).toContain('--user 1001:1001');
-  expect(cucumber?.run).toContain('-e PLAYWRIGHT_BROWSERS_PATH=/ms-playwright');
-  expect(cucumber?.run).toContain('-e PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD');
-  expect(cucumber?.run).not.toContain('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1');
+  expect(cucumber?.['working-directory']).toBe('e2e');
   expect(cucumber?.env?.DD_PORT).toBe('${{ steps.drydock.outputs.dd_port }}');
-  expect(cucumber?.run).toContain('-e DD_PORT');
-  expect(cucumber?.run).toContain('-v "${{ github.workspace }}:/work"');
-  expect(cucumber?.run).toContain('-w /work/e2e');
-  expect(cucumber?.run).toContain('"$PLAYWRIGHT_IMAGE"');
-  expect(cucumber?.run).toContain(
-    'npm ci --no-audit --no-fund && npm run cucumber -- --tags "not @requires_gitlab" --retry 1',
-  );
+  expect(cucumber?.run).toContain('npm run cucumber -- --tags "not @requires_gitlab"');
+  expect(cucumber?.run).not.toContain('docker run');
+  expect(cucumber?.run).not.toContain('npm ci');
+  expect(cucumber?.run).not.toContain('--retry');
+  expect(cucumber?.run).toContain('--format json:reports/cucumber.json');
+  expect(cucumber?.run).toContain('--format junit:reports/cucumber.xml');
+  expect(cucumber?.run).toContain('--format html:reports/cucumber.html');
+
+  const diagnostics = getWorkflowStep('e2e', 'Collect Cucumber diagnostics');
+  expect(diagnostics).toMatchObject({
+    if: 'failure() || cancelled()',
+    env: {
+      DD_PASSWORD: 'doe',
+      DD_USERNAME: 'john',
+    },
+  });
+  expect(diagnostics?.run).toContain('--user "${DD_USERNAME}:${DD_PASSWORD}"');
+  expect(diagnostics?.run?.match(/--max-time 10/g)).toHaveLength(2);
+  expect(diagnostics?.run).not.toContain('Authorization: Basic');
+  expect(getWorkflowStep('e2e', 'Upload Cucumber diagnostics')).toMatchObject({
+    if: 'always()',
+    uses: expectedActionUse('actions/upload-artifact'),
+    with: {
+      path: expect.stringContaining('e2e/reports'),
+      'if-no-files-found': 'warn',
+    },
+  });
 });
 
 test('DAST auth steps mask derived basic auth credentials', () => {

.github/tests/e2e-playwright-workflow.test.ts

@@ -1,11 +1,18 @@
+import { readFileSync } from 'node:fs';
 import { fileURLToPath } from 'node:url';
 
+import yaml from 'yaml';
+
 import {
   getWorkflowStep as getWorkflowStepFrom,
   loadWorkflow as loadWorkflowFrom,
 } from './workflow-test-utils';
 
 const workflowPath = fileURLToPath(new URL('../workflows/e2e-playwright.yml', import.meta.url));
+const qaComposePath = fileURLToPath(new URL('../../test/qa-compose.yml', import.meta.url));
+const playwrightConfigPath = fileURLToPath(
+  new URL('../../e2e/playwright.config.ts', import.meta.url),
+);
 const loadWorkflow = loadWorkflowFrom.bind(undefined, workflowPath);
 const getWorkflowStep = getWorkflowStepFrom.bind(undefined, workflowPath);
 
@@ -34,3 +41,63 @@ test('Playwright can be dispatched against a frozen release candidate', () => {
   expect(workflow.jobs?.changes?.if).toBeUndefined();
   expect(workflow.jobs?.playwright?.needs).toStrictEqual(['changes']);
 });
+
+test('Playwright QA isolates the full browser suite from the production API request budget', () => {
+  const qaCompose = readFileSync(qaComposePath, 'utf8');
+
+  expect(qaCompose).toMatch(/^\s*-\s+DD_SERVER_RATELIMIT_MAX=10000\s*$/mu);
+  expect(qaCompose).toMatch(/^\s*-\s+DD_ICON_PROXY_RATE_LIMIT_MAX=1000\s*$/mu);
+});
+
+test('Playwright QA fails closed when required remote fixtures cannot be seeded', () => {
+  const qaCompose = yaml.parse(readFileSync(qaComposePath, 'utf8')) as {
+    services?: Record<
+      string,
+      {
+        command?: string[];
+        depends_on?: Record<string, { condition?: string }>;
+        volumes?: string[];
+      }
+    >;
+  };
+
+  expect(qaCompose.services?.drydock?.depends_on?.['remote-bootstrap']).toStrictEqual({
+    condition: 'service_completed_successfully',
+  });
+  const remoteBootstrap = qaCompose.services?.['remote-bootstrap'];
+  expect(remoteBootstrap?.volumes).toContain('/var/run/docker.sock:/var/run/docker-host.sock');
+  const bootstrapCommand = remoteBootstrap?.command?.join('\n') ?? '';
+  expect(bootstrapCommand).toContain('set -eu');
+  expect(bootstrapCommand).toContain('pull_with_retry');
+  expect(bootstrapCommand).toMatch(/\[\s*"\$\$attempt"\s+-ge\s+3\s*\]/u);
+  expect(bootstrapCommand).toContain('docker --host "$$host_docker" pull');
+  expect(bootstrapCommand).toContain('docker --host "$$host_docker" save --output "$$archive"');
+  expect(bootstrapCommand).toContain('docker load --input "$$archive"');
+});
+
+test('Playwright health fixture waits until Docker observes the unhealthy transition', () => {
+  const qaCompose = yaml.parse(readFileSync(qaComposePath, 'utf8')) as {
+    services?: Record<
+      string,
+      {
+        command?: string[];
+        healthcheck?: { test?: string[] };
+      }
+    >;
+  };
+
+  const fixture = qaCompose.services?.['health-transition'];
+  const fixtureCommand = fixture?.command?.join('\n') ?? '';
+  const healthcheck = fixture?.healthcheck?.test?.join('\n') ?? '';
+
+  expect(fixtureCommand).toContain('/state/observed-unhealthy');
+  expect(fixtureCommand).toContain('healthcheck did not observe unhealthy state');
+  expect(healthcheck).toContain('touch /state/observed-unhealthy');
+});
+
+test('Playwright preserves first-failure evidence without retrying the whole suite', () => {
+  const playwrightConfig = readFileSync(playwrightConfigPath, 'utf8');
+
+  expect(playwrightConfig).toMatch(/^\s*retries:\s*0,\s*$/mu);
+  expect(playwrightConfig).toContain("trace: 'retain-on-failure'");
+});

.github/tests/release-cut-retry-workflow.test.ts

@@ -60,6 +60,20 @@ test('release-cut has no hand-rolled fixed retry loops', () => {
   expect(handRolledRetrySteps).toStrictEqual([]);
 });
 
+test('release-cut gates on Playwright for the exact release source SHA', () => {
+  const workflow = loadWorkflow(workflowPath);
+  const step = getStep('Wait for successful E2E Playwright on release source SHA');
+
+  expect(workflow.env?.E2E_PLAYWRIGHT_WORKFLOW_FILE).toBe('e2e-playwright.yml');
+  expect(step).toMatchObject({
+    uses: './.github/actions/wait-for-successful-branch-ci',
+    with: {
+      'workflow-file': '${{ env.E2E_PLAYWRIGHT_WORKFLOW_FILE }}',
+      'target-sha': '${{ steps.source.outputs.source_sha }}',
+    },
+  });
+});
+
 test('release-cut asserts package and lockfile versions for every workspace package', () => {
   const versionStep = getStep('Assert tag version matches package versions');
   const expectedVersionFiles = [

.github/workflows/ci-verify.yml

@@ -1059,14 +1059,16 @@ jobs:
     name: "🥒 E2E: Cucumber"
     if: github.event_name == 'workflow_dispatch' || (github.event_name != 'schedule' && needs.changes.outputs.runtime == 'true')
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    # Covers the bounded npm/setup retry budgets, readiness waits, scenario
+    # execution, and post-failure evidence collection without racing job expiry.
+    timeout-minutes: 40
     needs: [build, changes]
     permissions:
+      actions: read
       contents: read
     env:
-      # Use the same immutable browser runtime as the dedicated Playwright
-      # workflow. This avoids stale cache restores and runtime CDN downloads.
-      PLAYWRIGHT_IMAGE: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48
+      # Browser coverage belongs to the separately release-gated Playwright
+      # workflow. Do not download Chromium for Cucumber's API/stream contracts.
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
 
     steps:
@@ -1086,41 +1088,40 @@ jobs:
           node-version: 24
           package-manager-cache: false
 
-      - name: Verify Playwright image matches package
-        shell: bash
-        run: |
-          set -euo pipefail
-
-          package_version="$(node -p "require('./e2e/package.json').devDependencies['@playwright/test']")"
-          if [[ ! "$PLAYWRIGHT_IMAGE" =~ :v([0-9]+\.[0-9]+\.[0-9]+)-noble(@sha256:[0-9a-f]{64})?$ ]]; then
-            echo "::error title=Invalid Playwright image tag::Expected $PLAYWRIGHT_IMAGE to use :v<major>.<minor>.<patch>-noble optionally pinned with @sha256:<digest>."
-            exit 1
-          fi
+      - name: Download QA image
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          name: qa-image-${{ github.run_id }}
+          path: artifacts/qa
 
-          image_version="${BASH_REMATCH[1]}"
-          if [[ "$package_version" != "$image_version" ]]; then
-            echo "::error title=Playwright version mismatch::PLAYWRIGHT_IMAGE uses $image_version, but e2e/package.json pins @playwright/test to $package_version."
-            exit 1
-          fi
+      - name: Load QA image
+        run: docker load < artifacts/qa/drydock-dev-image.tar.gz
 
-      - name: Pull Playwright container
+      - name: Install e2e dependencies
         uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60  # v4.0.0
         with:
-          timeout_minutes: 5
+          timeout_minutes: 3
           max_attempts: 3
-          retry_wait_seconds: 30
-          command: docker pull "$PLAYWRIGHT_IMAGE"
+          retry_wait_seconds: 15
+          command: cd e2e && npm ci --no-audit --no-fund
+
+      - name: Run Cucumber support tests
+        run: npm run test:support
+        working-directory: e2e
 
       - name: Setup test containers
         uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60  # v4.0.0
         with:
-          timeout_minutes: 10
-          max_attempts: 3
+          timeout_minutes: 6
+          max_attempts: 2
           retry_wait_seconds: 10
           command: ./scripts/setup-test-containers.sh
 
       - name: Start drydock
         id: drydock
+        env:
+          DD_E2E_IMAGE: drydock:dev
+          DD_E2E_SKIP_BUILD: 'true'
         run: ./scripts/start-drydock.sh
 
       - name: Run Cucumber e2e tests
@@ -1129,30 +1130,56 @@ jobs:
         # @requires_gitlab scenarios here to match, otherwise they fail looking
         # for a missing container. The token-gated local path that runs them
         # lives in scripts/run-e2e-tests.sh.
-        # The pinned image supplies the browser revision expected by
-        # @playwright/test. Host networking keeps localhost API/browser access
-        # identical to the former host-side invocation.
         env:
           DD_PORT: ${{ steps.drydock.outputs.dd_port }}
+        working-directory: e2e
         run: |
-          docker run --rm \
-            --init \
-            --network host \
-            --ipc=host \
-            --user 1001:1001 \
-            -e CI=true \
-            -e HOME=/tmp \
-            -e PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
-            -e PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD \
-            -e DD_PORT \
-            -v "${{ github.workspace }}:/work" \
-            -w /work/e2e \
-            "$PLAYWRIGHT_IMAGE" \
-            sh -c 'npm ci --no-audit --no-fund && npm run cucumber -- --tags "not @requires_gitlab" --retry 1'
-
-      - name: Show drydock logs on failure
-        if: failure()
-        run: docker logs drydock
+          mkdir -p reports
+          npm run cucumber -- --tags "not @requires_gitlab" --format progress --format json:reports/cucumber.json --format junit:reports/cucumber.xml --format html:reports/cucumber.html
+
+      - name: Collect Cucumber diagnostics
+        if: failure() || cancelled()
+        env:
+          DD_PASSWORD: doe
+          DD_PORT: ${{ steps.drydock.outputs.dd_port }}
+          DD_USERNAME: john
+        run: |
+          set +e
+          diagnostics=artifacts/cucumber
+          mkdir -p "$diagnostics/fixtures"
+
+          docker ps -a --no-trunc > "$diagnostics/docker-ps.txt" 2>&1
+          docker logs --timestamps drydock > "$diagnostics/drydock.log" 2>&1
+          curl --silent --show-error \
+            --max-time 10 \
+            --output "$diagnostics/health.json" \
+            --write-out '%{http_code}\n' \
+            "http://localhost:${DD_PORT}/health" > "$diagnostics/health-status.txt" 2>&1
+          curl --silent --show-error \
+            --max-time 10 \
+            --user "${DD_USERNAME}:${DD_PASSWORD}" \
+            "http://localhost:${DD_PORT}/api/v1/containers" \
+            > "$diagnostics/containers.json" 2> "$diagnostics/containers-curl-error.txt"
+
+          while IFS= read -r container_id; do
+            [ -n "$container_id" ] || continue
+            container_name="$(docker inspect --format '{{.Name}}' "$container_id" | sed 's#^/##' | tr -c 'A-Za-z0-9_.-' '_')"
+            docker inspect --format '{{json .State}}' "$container_id" \
+              > "$diagnostics/fixtures/${container_name}.state.json" 2>&1
+            docker logs --timestamps --tail 200 "$container_id" \
+              > "$diagnostics/fixtures/${container_name}.log" 2>&1
+          done < <(docker ps -aq --filter label=dd.watch=true)
+
+      - name: Upload Cucumber diagnostics
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: cucumber-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            e2e/reports
+            artifacts/cucumber
+          if-no-files-found: warn
+          retention-days: 14
 
       - name: Cleanup
         if: always()