tests.yml

name: Tests

permissions:
  contents: read

on:
  push:
    paths:
      - ".cargo/*.toml"
      - ".github/workflows/tests.yml"
      - "Cargo.*"
      - "mutants_attrs/**"
      - "src/**"
      - "testdata/**"
      - "tests/**"

# see https://matklad.github.io/2021/09/04/fast-rust-builds.html
env:
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  CI: 1
  RUST_BACKTRACE: short
  RUSTFLAGS: "-W rust-2021-compatibility"
  RUSTUP_MAX_RETRIES: 10
  CARGO_MUTANTS_MINIMUM_TEST_TIMEOUT: 60

jobs:
  # Before anything else, run a quick test on just stable: this is significantly
  # faster than Windows or macOS and should catch most issues, and lets us get
  # started on the longer-running mutants and other tests.
  #
  # Also, build a Linux binary that we can use for the later mutants runs.
  quick-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
      - name: Show Cargo and rustc version
        run: |
          cargo --version
          rustc --version
      - name: Build
        run: cargo build --all-targets
      - name: Check rustfmt
        run: cargo fmt --all --check
      - name: Check clippy
        run: cargo clippy --all-targets --all-features -- -D warnings
      - name: Check typos
        uses: crate-ci/typos@master
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Test
        run: cargo test --workspace
      - name: Build release binary
        run: cargo build --release
      - name: Upload binary artifact
        uses: actions/upload-artifact@v4
        with:
          name: cargo-mutants-linux
          path: |
            target/release/cargo-mutants

  test:
    needs: [quick-test]
    strategy:
      matrix:
        os: [macOS-latest, ubuntu-latest, windows-latest]
        version: [stable, nightly, "1.78"]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.version }}
          components: rustfmt, clippy
      - name: Show Cargo and rustc version
        run: |
          cargo --version
          rustc --version
      - uses: Swatinem/rust-cache@v2
      - name: Check rustfmt
        run: cargo fmt --all --check
      - name: Check clippy
        if: matrix.version == 'stable'
        # Clippy checks can vary between versions in a way that makes it a bit
        # fiddly to satisfy them all, so only insist that they pass on stable.
        run: cargo clippy --all-targets --all-features -- -D warnings
      - name: Build
        run: cargo build --all-targets
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Test
        run: cargo test --workspace
      - run: cargo update
      - name: Test after cargo update
        run: cargo test --workspace
      - name: Downgrade to minimal versions
        if: matrix.version == 'nightly'
        run: cargo +nightly -Zdirect-minimal-versions update
      - name: Test on minimal versions
        if: matrix.version == 'nightly'
        run: cargo test
      - name: Install locked
        run: cargo install --path . --locked
      - name: Install unlocked
        run: cargo install --path .

  tests-from-tarball:
    needs: [quick-test]
    strategy:
      matrix:
        os: [ubuntu-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@nightly
      - uses: Swatinem/rust-cache@v2
      - run: cargo package --no-verify
      - name: Upload package artifact
        uses: actions/upload-artifact@v4
        with:
          name: cargo-mutants-package
          path: |
            target/package
      - name: Unpack package
        run: |
          cd target/package
          ls -l
          tar xvf cargo-mutants*.crate
      - name: Install nextest using install-action
        uses: taiki-e/install-action@v2
        with:
          tool: nextest
      - name: Run tests from package
        # A clunky way to cd into the likely unpacked name, excluding the directory,
        # still matching e.g. `1.2.3-pre2`.
        run: |
          cd target/package/cargo-mutants-*[0-9]
          cargo test

  branch-mutants:
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref != 'refs/heads/main'
    needs: [quick-test]
    strategy:
      matrix:
        test_tool: [cargo, nextest]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Relative diff
        run: |
          git branch -av
          git diff origin/main.. | tee git.diff
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Download cargo-mutants binary
        uses: actions/download-artifact@v4
        with:
          name: cargo-mutants-linux
      - name: Install cargo-mutants binary
        run: |
          install cargo-mutants ~/.cargo/bin/
      - name: Mutants in-diff
        run: >
          cargo mutants --no-shuffle -vV --in-diff=git.diff --in-place
          --test-tool=${{matrix.test_tool}} --timeout=500 --build-timeout=500
          --exclude=windows.rs --exclude=console.rs
      - name: Archive mutants.out
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: mutants-incremental-${{ matrix.test_tool}}.out
          path: mutants.out

  pr-mutants:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    needs: [quick-test]
    strategy:
      matrix:
        test_tool: [cargo, nextest]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Relative diff
        run: |
          git branch -av
          git diff origin/${{ github.base_ref }}.. | tee git.diff
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Download cargo-mutants binary
        uses: actions/download-artifact@v4
        with:
          name: cargo-mutants-linux
      - name: Install cargo-mutants binary
        run: |
          install cargo-mutants ~/.cargo/bin/
      - name: Mutants in-diff
        # Normally this would have --in-place, but for the sake of exercising more cases, it does not.
        run: >
          cargo mutants --no-shuffle -vV --in-diff git.diff
          --test-tool=${{matrix.test_tool}} --timeout=500 --build-timeout=500
          --exclude=windows.rs --exclude=console.rs
      - name: Archive mutants.out
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: mutants-incremental-${{ matrix.test_tool}}.out
          path: mutants.out

  cargo-mutants:
    runs-on: ubuntu-latest
    needs: [quick-test]
    strategy:
      fail-fast: false # We want to get all the mutant failures
      matrix:
        shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
        test_tool: [cargo]
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: beta
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@v2
        name: Install nextest using install-action
        with:
          tool: nextest
      - name: Download cargo-mutants binary
        uses: actions/download-artifact@v4
        with:
          name: cargo-mutants-linux
      - name: Install cargo-mutants binary
        run: |
          install cargo-mutants ~/.cargo/bin/
      - name: Mutants
        # Skip baselines because this job only runs after the baseline has been separately run.
        run: >
          cargo mutants --no-shuffle -vV --shard ${{ matrix.shard }}/10
          --test-tool ${{ matrix.test_tool }} --baseline=skip --timeout=500
          --build-timeout=500 --in-place --exclude=windows.rs
          --exclude=console.rs
      - name: Archive mutants.out
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: mutants-${{matrix.test_tool}}-shard${{matrix.shard}}.out
          path: mutants.out