diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml index d3660082f..34846f18c 100644 --- a/.github/workflows/mutation-testing.yml +++ b/.github/workflows/mutation-testing.yml @@ -35,7 +35,7 @@ jobs: name: Backend Mutation (Stryker.NET) if: ${{ !inputs.frontend_only }} runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 180 steps: - name: Checkout uses: actions/checkout@v7 @@ -46,14 +46,18 @@ jobs: dotnet-version: 8.0.x - name: Install Stryker.NET - run: dotnet tool install --global dotnet-stryker + run: dotnet tool install --global dotnet-stryker --version 4.16.0 + + - name: Validate Stryker.NET configuration + shell: pwsh + run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest - name: Restore solution run: dotnet restore backend/Taskdeck.sln - name: Run Stryker.NET - working-directory: backend - run: dotnet stryker --config-file stryker-config.json + working-directory: backend/tests/Taskdeck.Domain.Tests + run: dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput - name: Upload Stryker report if: always() @@ -61,6 +65,7 @@ jobs: with: name: stryker-net-report path: backend/StrykerOutput/**/reports/ + if-no-files-found: error retention-days: 30 frontend-mutation: diff --git a/backend/stryker-config.json b/backend/stryker-config.json index b7694f1ba..511f20120 100644 --- a/backend/stryker-config.json +++ b/backend/stryker-config.json @@ -1,10 +1,6 @@ { "stryker-config": { "project": "Taskdeck.Domain.csproj", - "test-projects": [ - "tests/Taskdeck.Domain.Tests/Taskdeck.Domain.Tests.csproj" - ], - "solution": "Taskdeck.sln", "mutation-level": "Standard", "reporters": [ "html", @@ -19,7 +15,7 @@ }, "language-version": "latest", "verbosity": "info", - "ignored-methods": [], - "excluded-mutations": [] + "ignore-methods": [], + "ignore-mutations": [] } } diff --git a/docs/IMPLEMENTATION_MASTERPLAN.md b/docs/IMPLEMENTATION_MASTERPLAN.md index 6b2d91321..7e842c79e 100644 --- a/docs/IMPLEMENTATION_MASTERPLAN.md +++ b/docs/IMPLEMENTATION_MASTERPLAN.md @@ -10,6 +10,10 @@ Companion Active Docs: - `docs/MANUAL_TEST_CHECKLIST.md` - `docs/GOLDEN_PRINCIPLES.md` +## Delivery update (2026-07-27, mutation-testing runway) + +- **Bounded backend mutation evidence (`#1500`, PR `#1503`, maintainer-held):** repaired the Domain Stryker.NET lane by pinning 4.16.0, validating its config/workflow contract with a 25-check preflight, running from the single Domain test-project context, capping the job at 180 minutes, and failing artifact upload when no report exists. Backend-only run [30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d` completed in 192 seconds over 3,682 mutants (2,351 killed, 576 survived, 2 timed out, 753 skipped), for a 70.75% baseline. Its two-file `stryker-net-report` artifact is 874,386 bytes with SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`. The score is calibration evidence, not a merge gate; expansion/ratcheting remains deferred until the existing 3-4-run policy baseline exists. + ## Delivery update (2026-07-27, test substrate and Windows API diagnostics) - **Dockerless Testcontainers gate (`#1518`):** the PostgreSQL collection fixture now evaluates `DockerAvailableCheck` before `PostgreSqlBuilder.Build`, restoring the documented graceful-skip behavior on hosts without Docker. Public-constructor and injected-factory regressions prove the unavailable path cannot construct a container, while a timed-out Docker CLI probe is killed as a process tree and boundedly reaped. The local exact tree finishes with 7 passed / 28 skipped / 0 failed and no surviving `docker info`; at reviewed head `6650370b`, the full serialized backend passed 7,465 tests with 33 intentional skips and hosted Container Integration proved the positive lifecycle with 35 passed / 0 skipped / 0 failed. Four DCO-signed-off commits merged as `ff1ee066`; exact-main Required CI `30257236213` and CodeQL `30257235538` are green. diff --git a/docs/MANUAL_TEST_CHECKLIST.md b/docs/MANUAL_TEST_CHECKLIST.md index d4f2a6afe..3a891f733 100644 --- a/docs/MANUAL_TEST_CHECKLIST.md +++ b/docs/MANUAL_TEST_CHECKLIST.md @@ -1054,7 +1054,11 @@ Status legend: `[ ]` = not yet performed, `[x]` = verified. 19. [ ] Verify mutation testing workflow is NOT listed in `ci-required.yml` (non-blocking, nightly only). 20. [ ] Verify `StrykerOutput` directories are in `.gitignore`. 21. [ ] Run `npm run mutation:test` locally to validate frontend Stryker setup (after `npm install`). -22. [ ] Run `dotnet stryker --config-file stryker-config.json` locally in `backend/` to validate backend setup. +22. [ ] Validate the backend Stryker setup from the repository root. + - Run: `powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest` + - Run: `cd backend/tests/Taskdeck.Domain.Tests` + - Run: `dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput` + - Expected: the preflight passes and Stryker uses only `Taskdeck.Domain.Tests` for the `Taskdeck.Domain` mutation run. ### Z8. Platform: PWA and Offline Readiness (PR #802 — merged) diff --git a/docs/STATUS.md b/docs/STATUS.md index efe13500c..3cbf0c236 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -2,6 +2,9 @@ Last Updated: 2026-07-27 +Backend mutation-testing runway repair (2026-07-27, `#1500`): +- **The scheduled Domain mutation lane now reaches a bounded, artifact-backed result instead of failing before execution.** Stryker.NET is pinned to 4.16.0; a 25-check PowerShell preflight rejects obsolete schema keys, wrong solution/test context, workflow timeout drift, and non-failing artifact uploads; and the workflow runs from `Taskdeck.Domain.Tests` with a 180-minute ceiling. Backend-only run [30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d` completed in 192 seconds: 3,682 mutants created, 2,351 killed, 576 survived, 2 timed out, 753 skipped, score 70.75%. The non-empty two-file `stryker-net-report` artifact is 874,386 bytes (SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`). The score remains a non-blocking calibration signal; PR `#1503` and merge remain maintainer-held because the workflow changes operational CI posture. + Dockerless Testcontainers gate repair (2026-07-27, `#1518`): - **The PostgreSQL integration collection now reaches its documented graceful-skip path before Testcontainers validates Docker.** `PostgresContainerFixture` checks `DockerAvailableCheck` before constructing the container, so a Dockerless host runs seven host-native integration tests and reports all 28 container-dependent tests skipped instead of throwing from the fixture constructor. The real public-constructor and injected-factory regressions prove that the unavailable path does not construct or dispose a phantom container. A timed-out Docker CLI probe is killed with its process tree and given a bounded reap before the check returns; the local exact tree finished with 7 passed / 28 skipped / 0 failed and zero `docker info` processes before or after. At reviewed head `6650370b`, the full serialized backend passed 7,465 tests with 33 intentional skips and hosted Container Integration proved the positive startup/connection/teardown path with 35 passed / 0 skipped / 0 failed. Four DCO-signed-off commits merged as `ff1ee066`; exact-main Required CI run `30257236213` and CodeQL run `30257235538` are green. diff --git a/docs/TESTING_GUIDE.md b/docs/TESTING_GUIDE.md index ce56d8b59..b7b12a6f0 100644 --- a/docs/TESTING_GUIDE.md +++ b/docs/TESTING_GUIDE.md @@ -642,18 +642,24 @@ CI: `reusable-visual-regression.yml` in extended CI (testing/visual label). Uplo ### Mutation Testing (TST-05, `#90`/`#796`) -Backend (Stryker.NET): targets `Taskdeck.Domain` with `Taskdeck.Domain.Tests`. Thresholds: break=60, high=80. +Backend (Stryker.NET 4.16.0): targets `Taskdeck.Domain` from the `Taskdeck.Domain.Tests` project context. Thresholds: break=0, low=60, high=80. The checked-in preflight rejects obsolete config keys and solution-context/workflow drift before the long mutation run. Frontend (Stryker JS): targets `captureStore`, `boardStore`, and `board/*.ts` submodules with vitest runner. Run commands: ```bash # Backend -cd backend && dotnet tool install dotnet-stryker && dotnet stryker +powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +# PowerShell 7/Linux runner equivalent: pwsh -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +dotnet tool install --global dotnet-stryker --version 4.16.0 +cd backend/tests/Taskdeck.Domain.Tests +dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput # Frontend cd frontend/taskdeck-web && npm run mutation:test ``` -CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. Non-blocking, reports uploaded as artifacts. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. +CI: `mutation-testing.yml` runs weekly (Sunday 04:00 UTC) + manual dispatch. The backend job has a finite 180-minute ceiling for the full Domain mutation set and fails if no report artifact exists. Mutation score remains non-blocking. Policy at `docs/testing/MUTATION_TESTING_POLICY.md`. + +Verified baseline: backend-only run [30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d` completed in 192 seconds. Stryker created 3,682 mutants; 2,351 were killed, 576 survived, 2 timed out, and 753 were skipped, for a 70.75% score. The uploaded two-file `stryker-net-report` artifact is 874,386 bytes (SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`). ### Container Integration Tests (TST-06, `#91`/`#804`) @@ -812,9 +818,12 @@ Mutation testing is available as a non-blocking quality signal for detecting wea ### Running locally ```bash -# Backend (requires dotnet-stryker global tool) -cd backend -dotnet stryker --config-file stryker-config.json +# Backend (from the repository root; uses the workflow-pinned tool version) +powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +# PowerShell 7/Linux runner equivalent: pwsh -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +dotnet tool install --global dotnet-stryker --version 4.16.0 +cd backend/tests/Taskdeck.Domain.Tests +dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput # Frontend cd frontend/taskdeck-web @@ -823,7 +832,7 @@ npm run mutation:test ### CI -Weekly workflow (Sunday 04:00 UTC) + manual dispatch via `.github/workflows/mutation-testing.yml`. Reports uploaded as artifacts. +Weekly workflow (Sunday 04:00 UTC) + manual dispatch via `.github/workflows/mutation-testing.yml`. The backend job has a 180-minute ceiling and missing backend reports fail artifact upload. ### Policy and triage diff --git a/docs/agentic/FAILURE_LEDGER.md b/docs/agentic/FAILURE_LEDGER.md index 7eb72ce44..6337c4de5 100644 --- a/docs/agentic/FAILURE_LEDGER.md +++ b/docs/agentic/FAILURE_LEDGER.md @@ -25,6 +25,7 @@ Rows sharing a surface and first tracking issue in `future_fix` show only their | 2026-07-26 | non_blocking_risk | agent/tool-command-composition | Resolution record for the repeated #1490 PowerShell/native command-composition failures; the original recurrence remains in append-only history | No workaround remains; use the copyable safe forms and classify future failures through the ledger process | #1490 resolved by PR #1491 merge 61f718af941c56c31b8b7595333b9debd6e47da8: the MCP tooling guide now covers collected foreach output, optional rg exits, safe mu... | resolved | | 2026-07-26 | blocker | agent/python-launcher | Resolution staged for #1487: exact Windows handlers and agent-utility permissions use py -3 -B, POSIX permissions retain python3 -B, and smoke children use the ... | Until the T4-class hook configuration is maintainer-merged, continue invoking py -3 -B explicitly on native Windows | #1487 resolution is staged on the human-held PR with sequential CPython 3.14 and 3.13 smoke coverage; append the final resolved record after merge and issue clo... | open | | 2026-07-26 | blocker | agent/powershell-deny-floor | The repo PreToolUse deny policy is matched only to Bash, so native PowerShell tool commands do not traverse the repository's destructive-command policy | Treat the current configured-handler smoke as Bash-payload-only proof; rely on existing tool permissions and do not claim native-PowerShell interception | #1497: add reviewed T4 native-PowerShell deny-policy coverage with direct allow and deny canaries before claiming interception | open | +| 2026-07-27 | non_blocking_risk | ci/mutation-testing-backend | Resolution record for the 16-run backend mutation evidence outage; the original failures remain in append-only history. | No workaround remains for producing bounded Domain mutation evidence; retain the non-blocking score posture while collecting the policy's 3-4-run calibration wi... | #1500 / PR #1503 repaired the lane at exact workflow head 307add004fbe142321a6ec11be21fab708824d5d. Backend-only run 30236307062 completed in 192 seconds with a... | resolved | | 2026-07-27 | pre_existing_noise | test/sqlite-concurrency | Correction to the #1373 resolution record: Required API Integration (windows-latest) on PR #1505 source head 6ee17a4849427d679722a9d1e034d4622536f4fe returned H... | Keep every recurrence red and use the bounded #1512 correlation/exception-classification diagnostics to capture the first server-side cause; do not retry the re... | #1373 resolved the proven #1282 SQLite registration-parity cause; successor #1512 remains open after its diagnostic-only slice until a recurrence identifies the... | open | | 2026-07-27 | blocker | test/testcontainers-dockerless-skip | Taskdeck.Integration.Tests constructed its PostgreSQL Testcontainer in the collection-fixture constructor, so Testcontainers validated an unavailable Docker end... | Run the focused non-container projects while #1518 is in review; do not retry a known-hanging Docker probe merely to obtain a local full-suite verdict | #1518 defers PostgreSqlBuilder.Build until after DockerAvailableCheck succeeds, proves the unavailable path never invokes the factory, and retains Docker-backed... | resolved | diff --git a/docs/agentic/failure_ledger.jsonl b/docs/agentic/failure_ledger.jsonl index f90410b47..53e504827 100644 --- a/docs/agentic/failure_ledger.jsonl +++ b/docs/agentic/failure_ledger.jsonl @@ -25,5 +25,7 @@ {"ts":"2026-07-25T23:48:15Z","class":"blocker","surface":"agent/python-launcher","failure":"Native-Windows python and python3 commands resolve to unusable Microsoft Store aliases, so configured agent hooks and documented failure-ledger utilities do not run even though CPython is installed behind py -3","workaround":"Invoke py -3 -B explicitly on Windows and python3 -B on POSIX; do not install another interpreter or mutate global PATH","future_fix":"#1487: make platform launchers explicit across configured handlers, permissions, smoke execution, and mirrored guidance","status":"open"} {"ts":"2026-07-26T10:56:00Z","class":"blocker","surface":"agent/python-launcher","failure":"Resolution staged for #1487: exact Windows handlers and agent-utility permissions use py -3 -B, POSIX permissions retain python3 -B, and smoke children use the active sys.executable -B","workaround":"Until the T4-class hook configuration is maintainer-merged, continue invoking py -3 -B explicitly on native Windows","future_fix":"#1487 resolution is staged on the human-held PR with sequential CPython 3.14 and 3.13 smoke coverage; append the final resolved record after merge and issue closure","status":"open"} {"ts":"2026-07-26T11:36:55Z","class":"blocker","surface":"agent/powershell-deny-floor","failure":"The repo PreToolUse deny policy is matched only to Bash, so native PowerShell tool commands do not traverse the repository's destructive-command policy","workaround":"Treat the current configured-handler smoke as Bash-payload-only proof; rely on existing tool permissions and do not claim native-PowerShell interception","future_fix":"#1497: add reviewed T4 native-PowerShell deny-policy coverage with direct allow and deny canaries before claiming interception","status":"open"} +{"ts":"2026-07-27T00:00:00Z","class":"non_blocking_risk","surface":"ci/mutation-testing-backend","failure":"All 16 recorded scheduled Mutation Testing runs failed to produce backend Stryker evidence; the latest runs 30191449339 and 29676656172 both stopped because Stryker.NET 4.16.0 rejects ignored-methods and excluded-mutations as invalid schema keys.","workaround":"Treat the frontend lane as independent evidence, but do not claim backend mutation coverage until a backend-only exact-head run completes and uploads a non-empty report artifact.","future_fix":"#1500 pins Stryker.NET 4.16.0, uses ignore-methods and ignore-mutations, and adds a local preflight that rejects obsolete keys; resolve this row only after the PR exact-head backend-only dispatch completes mutation execution with a non-empty Stryker artifact.","status":"open"} +{"ts":"2026-07-27T04:12:28Z","class":"non_blocking_risk","surface":"ci/mutation-testing-backend","failure":"Resolution record for the 16-run backend mutation evidence outage; the original failures remain in append-only history.","workaround":"No workaround remains for producing bounded Domain mutation evidence; retain the non-blocking score posture while collecting the policy's 3-4-run calibration window.","future_fix":"#1500 / PR #1503 repaired the lane at exact workflow head 307add004fbe142321a6ec11be21fab708824d5d. Backend-only run 30236307062 completed in 192 seconds with a 70.75% score over 3,682 mutants and uploaded a non-empty 874,386-byte two-file report artifact (SHA-256 0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8).","status":"resolved"} {"ts":"2026-07-27T06:50:27Z","class":"pre_existing_noise","surface":"test/sqlite-concurrency","failure":"Correction to the #1373 resolution record: Required API Integration (windows-latest) on PR #1505 source head 6ee17a4849427d679722a9d1e034d4622536f4fe returned HTTP 500 for 2 of 5 concurrent card-create requests while Ubuntu passed; an identical-head rerun and local stress passed, so the causal exception and any SQLite/SQLITE_BUSY classification remain unproven","workaround":"Keep every recurrence red and use the bounded #1512 correlation/exception-classification diagnostics to capture the first server-side cause; do not retry the request, quarantine the test, or accept HTTP 500 as success","future_fix":"#1373 resolved the proven #1282 SQLite registration-parity cause; successor #1512 remains open after its diagnostic-only slice until a recurrence identifies the root cause and a narrow corrective fix is proven by the full backend and exact-head Windows/Ubuntu gates","status":"open"} {"ts":"2026-07-27T09:20:44Z","class":"blocker","surface":"test/testcontainers-dockerless-skip","failure":"Taskdeck.Integration.Tests constructed its PostgreSQL Testcontainer in the collection-fixture constructor, so Testcontainers validated an unavailable Docker endpoint before DockerAvailableCheck could mark the 28 container tests skipped; this blocked the serialized backend gate on Dockerless developer hosts","workaround":"Run the focused non-container projects while #1518 is in review; do not retry a known-hanging Docker probe merely to obtain a local full-suite verdict","future_fix":"#1518 defers PostgreSqlBuilder.Build until after DockerAvailableCheck succeeds, proves the unavailable path never invokes the factory, and retains Docker-backed CI as the positive-lifecycle gate","status":"resolved"} diff --git a/docs/testing/MUTATION_TESTING_POLICY.md b/docs/testing/MUTATION_TESTING_POLICY.md index dcaf258f9..2fbe2db68 100644 --- a/docs/testing/MUTATION_TESTING_POLICY.md +++ b/docs/testing/MUTATION_TESTING_POLICY.md @@ -1,6 +1,6 @@ # Mutation Testing Policy -Last Updated: 2026-04-09 +Last Updated: 2026-07-27 ## Purpose @@ -14,8 +14,11 @@ This is a **quality signal**, not a gatekeeping mechanism. Mutation testing comp - **Target**: `Taskdeck.Domain` project - **Test project**: `Taskdeck.Domain.Tests` -- **Rationale**: Domain contains core business logic (entity state machines, validation rules, invariants) where surviving mutants have the highest impact. Domain is pure C# with no infrastructure dependencies, making mutation runs fast and deterministic. +- **Tool contract**: Stryker.NET `4.16.0` +- **Rationale**: Domain contains core business logic (entity state machines, validation rules, invariants) where surviving mutants have the highest impact. Domain is pure C# with no infrastructure dependencies, making it the narrowest deterministic backend target; its full mutation set is still a long-running workload. - **Config**: `backend/stryker-config.json` +- **Execution context**: run from `backend/tests/Taskdeck.Domain.Tests`; do not add `solution` or `test-projects` to the config because solution context takes precedence and discovers unrelated tests +- **Preflight**: `scripts/ci/Test-StrykerConfig.ps1 -SelfTest` rejects obsolete schema keys, solution-context selectors, and workflow/artifact drift before the long mutation run starts ### Frontend (Stryker JS/TS) @@ -51,12 +54,17 @@ After the first 3-4 runs: ### Backend (local) ```bash -# Install Stryker.NET as a global tool (once) -dotnet tool install --global dotnet-stryker +# From the repository root, validate the checked-in schema contract. +# Native Windows PowerShell: +powershell -NoProfile -File scripts/ci/Test-StrykerConfig.ps1 -SelfTest +# PowerShell 7 on Linux/macOS/Windows uses the equivalent `pwsh -File ...` form. -# Run from the backend/ directory -cd backend -dotnet stryker --config-file stryker-config.json +# Install the workflow-pinned Stryker.NET version as a global tool (once). +dotnet tool install --global dotnet-stryker --version 4.16.0 + +# Run from the Domain test project so Stryker discovers only that test project. +cd backend/tests/Taskdeck.Domain.Tests +dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput ``` Report: `backend/StrykerOutput//reports/mutation-report.html` @@ -77,6 +85,9 @@ The mutation testing workflow runs: - **On demand**: via `workflow_dispatch` from the Actions tab Reports are uploaded as GitHub Actions artifacts with 30-day retention. +The backend job has a finite 180-minute ceiling for the full Domain mutation set, and artifact upload fails when no report was produced. + +The first repaired backend-only baseline, [run 30236307062](https://github.com/Chris0Jeky/Taskdeck/actions/runs/30236307062) on exact workflow head `307add004fbe142321a6ec11be21fab708824d5d`, completed in 192 seconds. It created 3,682 mutants: 2,351 killed, 576 survived, 2 timed out, and 753 skipped, for a 70.75% score. The non-empty two-file report artifact is 874,386 bytes (SHA-256 `0e8a9a41b8cd484b6c267bd914c57cda0ffa973f59d8989e89038157605f21c8`). Keep the 0% break threshold until the policy's 3-4-run calibration window exists. ## Interpreting Reports @@ -101,11 +112,11 @@ Reports are uploaded as GitHub Actions artifacts with 30-day retention. When mutation testing reveals surviving mutants: -1. **File an issue** with the label `test-hardening` and link to the mutation report artifact +1. **File an issue** with the existing `testing` and `hardening` labels and link to the mutation report artifact 2. **Categorize** surviving mutants by triage priority (see above) 3. **Bundle fixes**: Group related assertion improvements into a single PR per module rather than one PR per mutant 4. **Do not chase 100%**: Some surviving mutants are acceptable (e.g., log messages, cosmetic formatting). Document intentional exclusions: - - **Backend (Stryker.NET)**: Use `excluded-mutations` or `ignored-methods` in `backend/stryker-config.json` + - **Backend (Stryker.NET 4.16.0)**: Add Stryker patterns as non-empty string entries in the `ignore-mutations` or `ignore-methods` arrays in `backend/stryker-config.json`; the preflight accepts empty arrays and non-empty string entries while rejecting scalar/invalid entries and the obsolete `excluded-mutations` and `ignored-methods` spellings - **Frontend (Stryker JS)**: Adjust `mutate` glob patterns in `stryker.config.mjs` or use inline `// Stryker disable` comments in source files ## Scope Expansion Roadmap diff --git a/scripts/ci/Test-StrykerConfig.ps1 b/scripts/ci/Test-StrykerConfig.ps1 new file mode 100644 index 000000000..f950fe015 --- /dev/null +++ b/scripts/ci/Test-StrykerConfig.ps1 @@ -0,0 +1,328 @@ +[CmdletBinding()] +param( + [string]$ConfigPath, + [string]$WorkflowPath, + [switch]$SelfTest +) + +$ErrorActionPreference = 'Stop' + +$repositoryRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) + +if ([string]::IsNullOrWhiteSpace($ConfigPath)) { + $ConfigPath = Join-Path (Join-Path $repositoryRoot 'backend') 'stryker-config.json' +} + +if ([string]::IsNullOrWhiteSpace($WorkflowPath)) { + $workflowsDirectory = Join-Path (Join-Path $repositoryRoot '.github') 'workflows' + $WorkflowPath = Join-Path $workflowsDirectory 'mutation-testing.yml' +} + +function Test-StrykerConfig { + param( + [Parameter(Mandatory)] + [string]$Path + ) + + $resolvedPath = Resolve-Path -LiteralPath $Path -ErrorAction Stop + + try { + $config = Get-Content -LiteralPath $resolvedPath -Raw | ConvertFrom-Json -ErrorAction Stop + } catch { + throw "Stryker configuration '$resolvedPath' is not valid JSON: $($_.Exception.Message)" + } + + $strykerConfig = $config.'stryker-config' + if ($null -eq $strykerConfig) { + throw "Stryker configuration '$resolvedPath' must contain a stryker-config object." + } + + foreach ($obsoleteKey in @('ignored-methods', 'excluded-mutations')) { + if ($null -ne $strykerConfig.PSObject.Properties[$obsoleteKey]) { + $replacementKey = $obsoleteKey -replace '^excluded-', 'ignore-' -replace '^ignored-', 'ignore-' + throw "Stryker configuration '$resolvedPath' uses obsolete '$obsoleteKey'. Use '$replacementKey' instead." + } + } + + foreach ($solutionContextKey in @('solution', 'test-projects')) { + if ($null -ne $strykerConfig.PSObject.Properties[$solutionContextKey]) { + throw "Stryker configuration '$resolvedPath' must omit '$solutionContextKey' so the workflow stays in Taskdeck.Domain.Tests context." + } + } + + $projectProperty = $strykerConfig.PSObject.Properties['project'] + if ($null -eq $projectProperty -or $projectProperty.Value -cne 'Taskdeck.Domain.csproj') { + throw "Stryker configuration '$resolvedPath' must target project 'Taskdeck.Domain.csproj'." + } + + $mutationLevelProperty = $strykerConfig.PSObject.Properties['mutation-level'] + if ($null -eq $mutationLevelProperty -or $mutationLevelProperty.Value -cne 'Standard') { + throw "Stryker configuration '$resolvedPath' must preserve mutation-level 'Standard'." + } + + $reportersProperty = $strykerConfig.PSObject.Properties['reporters'] + if ($null -eq $reportersProperty -or $reportersProperty.Value -isnot [System.Array]) { + throw "Stryker configuration '$resolvedPath' must define reporters as an array." + } + + $reporters = @($reportersProperty.Value) + if (($reporters -join ',') -cne 'html,json,progress,cleartext') { + throw "Stryker configuration '$resolvedPath' must preserve html, json, progress, and cleartext reporters." + } + + $thresholdsProperty = $strykerConfig.PSObject.Properties['thresholds'] + $thresholds = $thresholdsProperty.Value + if ($null -eq $thresholdsProperty -or + $null -eq $thresholds -or + $thresholds.high -ne 80 -or + $thresholds.low -ne 60 -or + $thresholds.break -ne 0) { + throw "Stryker configuration '$resolvedPath' must preserve thresholds high=80, low=60, break=0." + } + + foreach ($exclusionKey in @('ignore-methods', 'ignore-mutations')) { + $property = $strykerConfig.PSObject.Properties[$exclusionKey] + if ($null -eq $property -or $property.Value -isnot [System.Array]) { + throw "Stryker configuration '$resolvedPath' must contain '$exclusionKey' as an array." + } + + for ($entryIndex = 0; $entryIndex -lt $property.Value.Count; $entryIndex++) { + $entry = $property.Value[$entryIndex] + if ($entry -isnot [string] -or [string]::IsNullOrWhiteSpace($entry)) { + throw "Stryker configuration '$resolvedPath' entry $entryIndex in '$exclusionKey' must be a non-empty string." + } + } + } +} + +function Assert-ExactWorkflowLine { + param( + [Parameter(Mandatory)] + [string]$Block, + [Parameter(Mandatory)] + [string]$ExpectedLine, + [Parameter(Mandatory)] + [string]$ContractName + ) + + $count = @(($Block -split "`n") | Where-Object { $_ -ceq $ExpectedLine }).Count + if ($count -ne 1) { + throw "Mutation workflow must contain exactly one $ContractName line '$ExpectedLine' in backend-mutation; found $count." + } +} + +function Assert-ExactWorkflowFragment { + param( + [Parameter(Mandatory)] + [string]$Block, + [Parameter(Mandatory)] + [string]$ExpectedFragment, + [Parameter(Mandatory)] + [string]$ContractName + ) + + $count = 0 + $searchIndex = 0 + while ($true) { + $matchIndex = $Block.IndexOf($ExpectedFragment, $searchIndex, [System.StringComparison]::Ordinal) + if ($matchIndex -lt 0) { + break + } + + $count++ + $searchIndex = $matchIndex + $ExpectedFragment.Length + } + + if ($count -ne 1) { + throw "Mutation workflow must contain exactly one $ContractName block in backend-mutation; found $count." + } +} + +function Test-MutationWorkflowContract { + param( + [Parameter(Mandatory)] + [string]$Path + ) + + $resolvedPath = Resolve-Path -LiteralPath $Path -ErrorAction Stop + $workflow = [System.IO.File]::ReadAllText($resolvedPath) + $normalizedWorkflow = $workflow.Replace("`r`n", "`n").Replace("`r", "`n") + $backendMarker = " backend-mutation:`n" + $frontendMarker = " frontend-mutation:`n" + $backendStart = $normalizedWorkflow.IndexOf($backendMarker, [System.StringComparison]::Ordinal) + $frontendStart = $normalizedWorkflow.IndexOf($frontendMarker, [System.StringComparison]::Ordinal) + + if ($backendStart -lt 0 -or $frontendStart -le $backendStart) { + throw "Mutation workflow '$resolvedPath' must contain backend-mutation before frontend-mutation." + } + + $backendBlock = $normalizedWorkflow.Substring($backendStart, $frontendStart - $backendStart) + $requiredLines = [ordered]@{ + 'finite timeout' = ' timeout-minutes: 180' + 'pinned tool install' = ' run: dotnet tool install --global dotnet-stryker --version 4.16.0' + 'configuration self-test' = ' run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest' + } + + foreach ($entry in $requiredLines.GetEnumerator()) { + Assert-ExactWorkflowLine -Block $backendBlock -ExpectedLine $entry.Value -ContractName $entry.Key + } + + $strykerStep = @( + ' - name: Run Stryker.NET', + ' working-directory: backend/tests/Taskdeck.Domain.Tests', + ' run: dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput' + ) -join "`n" + Assert-ExactWorkflowFragment -Block $backendBlock -ExpectedFragment $strykerStep -ContractName 'test-project Stryker step' + + $artifactStep = @( + ' - name: Upload Stryker report', + ' if: always()', + ' uses: actions/upload-artifact@v7', + ' with:', + ' name: stryker-net-report', + ' path: backend/StrykerOutput/**/reports/', + ' if-no-files-found: error', + ' retention-days: 30' + ) -join "`n" + Assert-ExactWorkflowFragment -Block $backendBlock -ExpectedFragment $artifactStep -ContractName 'fail-closed report artifact step' + + $strykerCommandCount = [regex]::Matches( + $backendBlock, + '(?m)^\s+run:\s+(?:dotnet stryker|(?:\S+/)?dotnet-stryker)(?:\s|$)' + ).Count + if ($strykerCommandCount -ne 1) { + throw "Mutation workflow '$resolvedPath' must invoke Stryker exactly once in backend-mutation; found $strykerCommandCount." + } +} + +function Assert-Rejected { + param( + [Parameter(Mandatory)] + [scriptblock]$Action, + [Parameter(Mandatory)] + [string]$Scenario, + [Parameter(Mandatory)] + [string]$ExpectedMessage + ) + + try { + & $Action + } catch { + if (-not $_.Exception.Message.Contains($ExpectedMessage)) { + throw "The Stryker preflight rejected '$Scenario' for an unexpected reason: $($_.Exception.Message)" + } + + return + } + + throw "The Stryker preflight accepted invalid scenario '$Scenario'." +} + +function Write-TextVariant { + param( + [Parameter(Mandatory)] + [string]$Source, + [Parameter(Mandatory)] + [string]$Destination, + [Parameter(Mandatory)] + [string]$Expected, + [Parameter(Mandatory)] + [string]$Replacement + ) + + $content = [System.IO.File]::ReadAllText($Source) + $firstMatch = $content.IndexOf($Expected, [System.StringComparison]::Ordinal) + $lastMatch = $content.LastIndexOf($Expected, [System.StringComparison]::Ordinal) + if ($firstMatch -lt 0 -or $firstMatch -ne $lastMatch) { + throw "Self-test fixture expected exactly one source fragment '$Expected'." + } + + [System.IO.File]::WriteAllText($Destination, $content.Replace($Expected, $Replacement)) +} + +function Invoke-StrykerConfigSelfTest { + $temporaryDirectory = Join-Path ([System.IO.Path]::GetTempPath()) ("taskdeck-stryker-config-$([guid]::NewGuid().ToString('N'))") + $validConfigPath = Join-Path $temporaryDirectory 'valid.json' + $validWorkflowPath = Join-Path $temporaryDirectory 'valid.yml' + + try { + New-Item -ItemType Directory -Path $temporaryDirectory -ErrorAction Stop | Out-Null + Copy-Item -LiteralPath $ConfigPath -Destination $validConfigPath -ErrorAction Stop + Copy-Item -LiteralPath $WorkflowPath -Destination $validWorkflowPath -ErrorAction Stop + Test-StrykerConfig -Path $validConfigPath + Test-MutationWorkflowContract -Path $validWorkflowPath + + $validExclusionVariants = @( + @('ignore-methods', ' "ignore-methods": [],', ' "ignore-methods": ["ToString", "Console.Write*"],'), + @('ignore-mutations', ' "ignore-mutations": []', ' "ignore-mutations": ["string", "logical"]') + ) + + for ($validIndex = 0; $validIndex -lt $validExclusionVariants.Count; $validIndex++) { + $validVariant = $validExclusionVariants[$validIndex] + $validVariantPath = Join-Path $temporaryDirectory "valid-$validIndex.json" + Write-TextVariant -Source $ConfigPath -Destination $validVariantPath -Expected $validVariant[1] -Replacement $validVariant[2] + Test-StrykerConfig -Path $validVariantPath + } + + $configVariants = @( + @('obsolete ignored-methods key', ' "ignore-methods": [],', ' "ignored-methods": [],', "uses obsolete 'ignored-methods'"), + @('obsolete excluded-mutations key', ' "ignore-mutations": []', ' "excluded-mutations": []', "uses obsolete 'excluded-mutations'"), + @('solution context', ' "project": "Taskdeck.Domain.csproj",', (@(' "project": "Taskdeck.Domain.csproj",', ' "solution": "Taskdeck.sln",') -join "`n"), "must omit 'solution'"), + @('test-project selector', ' "project": "Taskdeck.Domain.csproj",', (@(' "project": "Taskdeck.Domain.csproj",', ' "test-projects": ["tests/Taskdeck.Domain.Tests/Taskdeck.Domain.Tests.csproj"],') -join "`n"), "must omit 'test-projects'"), + @('wrong mutation target', ' "project": "Taskdeck.Domain.csproj",', ' "project": "Taskdeck.Application.csproj",', "must target project 'Taskdeck.Domain.csproj'"), + @('ignore-methods scalar', ' "ignore-methods": [],', ' "ignore-methods": "ToString",', "must contain 'ignore-methods' as an array"), + @('ignore-mutations scalar', ' "ignore-mutations": []', ' "ignore-mutations": "string"', "must contain 'ignore-mutations' as an array"), + @('ignore-methods null entry', ' "ignore-methods": [],', ' "ignore-methods": [null],', "entry 0 in 'ignore-methods' must be a non-empty string"), + @('ignore-mutations non-string entry', ' "ignore-mutations": []', ' "ignore-mutations": [42]', "entry 0 in 'ignore-mutations' must be a non-empty string"), + @('ignore-methods empty entry', ' "ignore-methods": [],', ' "ignore-methods": [""],', "entry 0 in 'ignore-methods' must be a non-empty string"), + @('ignore-mutations whitespace entry', ' "ignore-mutations": []', ' "ignore-mutations": [" "]', "entry 0 in 'ignore-mutations' must be a non-empty string"), + @('changed mutation level', ' "mutation-level": "Standard",', ' "mutation-level": "Advanced",', "must preserve mutation-level 'Standard'"), + @('missing JSON reporter', ' "json",', ' "dashboard",', 'must preserve html, json, progress, and cleartext reporters'), + @('changed score threshold', ' "break": 0', ' "break": 60', 'must preserve thresholds high=80, low=60, break=0') + ) + + for ($configIndex = 0; $configIndex -lt $configVariants.Count; $configIndex++) { + $configVariant = $configVariants[$configIndex] + $variantPath = Join-Path $temporaryDirectory "config-$configIndex.json" + Write-TextVariant -Source $ConfigPath -Destination $variantPath -Expected $configVariant[1] -Replacement $configVariant[2] + Assert-Rejected -Scenario $configVariant[0] -ExpectedMessage $configVariant[3] -Action { + Test-StrykerConfig -Path $variantPath + } + } + + $workflowVariants = @( + @('timeout', ' timeout-minutes: 180', ' timeout-minutes: 60', 'finite timeout'), + @('tool version', ' run: dotnet tool install --global dotnet-stryker --version 4.16.0', ' run: dotnet tool install --global dotnet-stryker --version 4.17.0', 'pinned tool install'), + @('preflight mode', ' run: ./scripts/ci/Test-StrykerConfig.ps1 -SelfTest', ' run: ./scripts/ci/Test-StrykerConfig.ps1', 'configuration self-test'), + @('working-directory', ' working-directory: backend/tests/Taskdeck.Domain.Tests', ' working-directory: backend', 'test-project Stryker step'), + @('config path', ' run: dotnet stryker --config-file ../../stryker-config.json --output ../../StrykerOutput', ' run: dotnet stryker --config-file stryker-config.json', 'test-project Stryker step'), + @('artifact path', ' path: backend/StrykerOutput/**/reports/', ' path: backend/tests/Taskdeck.Domain.Tests/StrykerOutput/**/reports/', 'fail-closed report artifact step'), + @('missing artifact policy', ' if-no-files-found: error', ' if-no-files-found: warn', 'fail-closed report artifact step') + ) + + foreach ($workflowVariant in $workflowVariants) { + $variantPath = Join-Path $temporaryDirectory "$($workflowVariant[0]).yml" + Write-TextVariant -Source $WorkflowPath -Destination $variantPath -Expected $workflowVariant[1] -Replacement $workflowVariant[2] + Assert-Rejected -Scenario "workflow $($workflowVariant[0]) drift" -ExpectedMessage $workflowVariant[3] -Action { + Test-MutationWorkflowContract -Path $variantPath + } + } + + $validContractCount = 2 + $validExclusionVariants.Count + $rejectedFixtureCount = $configVariants.Count + $workflowVariants.Count + Write-Host "Stryker preflight self-test passed: $($validContractCount + $rejectedFixtureCount) checks ($validContractCount valid contracts; $rejectedFixtureCount rejected drift fixtures)." + } finally { + if (Test-Path -LiteralPath $temporaryDirectory) { + Remove-Item -LiteralPath $temporaryDirectory -Recurse -Force + } + } +} + +Test-StrykerConfig -Path $ConfigPath +Test-MutationWorkflowContract -Path $WorkflowPath + +if ($SelfTest) { + Invoke-StrykerConfigSelfTest +} + +Write-Host "Stryker configuration and workflow preflight passed: $ConfigPath; $WorkflowPath"