workflows/eval.yml: Run on dev branch pushes and compute outpath diff…

…erence

.github/workflows/eval.yml

@@ -1,6 +1,15 @@
 name: Eval
 
-on: pull_request_target
+on:
+  pull_request_target:
+  push:
+    # Keep this synced with ci/request-reviews/dev-branches.txt
+    branches:
+      - master
+      - staging
+      - release-*
+      - staging-*
+      - haskell-updates
 
 permissions:
   contents: read
@@ -11,6 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       mergedSha: ${{ steps.merged.outputs.mergedSha }}
+      baseSha: ${{ steps.baseSha.outputs.baseSha }}
       systems: ${{ steps.systems.outputs.systems }}
     steps:
       # Important: Because of `pull_request_target`, this doesn't check out the PR,
@@ -24,23 +34,39 @@ jobs:
         id: merged
         env:
           GH_TOKEN: ${{ github.token }}
+          GH_EVENT: ${{ github.event_name }}
         run: |
-          if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
-            echo "Checking the merge commit $mergedSha"
-            echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
-          else
-            # Skipping so that no notifications are sent
-            echo "Skipping the rest..."
-          fi
+          case "$GH_EVENT" in
+            push)
+              echo "mergedSha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
+              ;;
+            pull_request_target)
+              if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
+                echo "Checking the merge commit $mergedSha"
+                echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
+              else
+                # Skipping so that no notifications are sent
+                echo "Skipping the rest..."
+              fi
+              ;;
+          esac
           rm -rf base
       - name: Check out the PR at the test merge commit
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         # Add this to _all_ subsequent steps to skip them
         if: steps.merged.outputs.mergedSha
         with:
           ref: ${{ steps.merged.outputs.mergedSha }}
+          fetch-depth: 2
           path: nixpkgs
 
+      - name: Determine base commit
+        if: github.event_name == 'pull_request_target' && steps.merged.outputs.mergedSha
+        id: baseSha
+        run: |
+          baseSha=$(git -C nixpkgs rev-parse HEAD^1)
+          echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT"
+
       - name: Install Nix
         uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
         if: steps.merged.outputs.mergedSha
@@ -124,18 +150,71 @@ jobs:
       - name: Combine all output paths and eval stats
         run: |
           nix-build nixpkgs/ci -A eval.combine \
-            --arg resultsDir ./intermediate
+            --arg resultsDir ./intermediate \
+            -o prResult
 
       - name: Upload the combined results
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
           name: result
-          path: result/*
+          path: prResult/*
+
+      - name: Get base run id
+        if: needs.attrs.outputs.baseSha
+        id: baseRunId
+        run: |
+          set -e
+
+          # TODO: Wait until it's done
+          # Get the latest eval.yml workflow run for the PR's base commit
+          if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
+            -f head_sha="$BASE_SHA" \
+            --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]')
+            || [[ -z "$run" ]]; then
+            echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
+            exit 0
+          fi
+          echo "Comparing against $(jq .html_url <<< "$run")"
+          runId=$(jq .id <<< "$run")
+          conclusion=$(jq .conclusion <<< "$run")
+
+          while [[ "$conclusion" == null ]]; do
+            echo "Workflow not done, waiting 10 seconds before checking again"
+            sleep 10
+            conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
+          done
+
+          if [[ "$conclusion" != "success" ]]; then
+            echo "Workflow was not successful, cannot make comparison"
+            exit 0
+          fi
+
+          echo "baseRunId=$runId" >> "$GITHUB_OUTPUT"
+        env:
+          REPOSITORY: ${{ github.repository }}
+          BASE_SHA: ${{ needs.attrs.outputs.baseSha }}
 
+      - uses: actions/download-artifact@v4
+        if: steps.baseRunId.outputs.baseRunId
+        with:
+          name: result
+          path: baseResult
+          github-token: ${{ github.token }}
+          run-id: ${{ steps.baseRunId.outputs.baseRunId }}
+
+      - name: Compare against the base branch
+        if: steps.baseRunId.outputs.baseRunId
+        run: |
+          nix-build nixpkgs/ci -A eval.compare \
+            --arg beforeResultDir ./baseResult \
+            --arg afterResultDir ./prResult \
+            -o comparison
 
-      # TODO: Run this workflow also on `push` (on at least the main development branches)
-      # Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)
-      # to have completed the eval, then use
-      # gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>
-      # and follow it to the artifact results, where you can then download the outpaths.json from the base branch
-      # That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers
+          # TODO: Request reviews from maintainers for packages whose files are modified in the PR
+
+      - name: Upload the combined results
+        if: steps.baseRunId.outputs.baseRunId
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: comparison
+          path: comparison/*

ci/eval/default.nix

@@ -238,6 +238,22 @@ let
           jq -s from_entries > $out/stats.json
       '';
 
+  compare =
+    { beforeResultDir, afterResultDir }:
+    runCommand "compare" {
+      nativeBuildInputs = [
+        jq
+      ];
+    } ''
+      mkdir $out
+      jq -n -f ${./filter.jq} \
+        --slurpfile before ${beforeResultDir}/outpaths.json \
+        --slurpfile after ${afterResultDir}/outpaths.json \
+        > $out/changed-paths.json
+
+      # TODO: Compare eval stats
+    '';
+
   full =
     {
       # Whether to evaluate just a single system, by default all are evaluated
@@ -262,12 +278,14 @@ let
       resultsDir = results;
     };
 
+
 in
 {
   inherit
     attrpathsSuperset
     singleSystem
     combine
+    compare
     # The above three are used by separate VMs in a GitHub workflow,
     # while the below is intended for testing on a single local machine
     full

ci/request-reviews/dev-branches.txt

@@ -1,5 +1,6 @@
 # Trusted development branches:
 # These generally require PRs to update and are built by Hydra.
+# Keep this synced with the branches in .github/workflows/eval.yml
 master
 staging
 release-*