WIP

.github/workflows/eval.yml

@@ -6,12 +6,12 @@ permissions:
   contents: read
 
 jobs:
-  tests:
-    name: eval-check
+  attrs:
+    name: Attrs
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        system: [x86_64-linux, aarch64-linux, aarch64-darwin, x86_64-darwin]
+    outputs:
+      systems: ${{ steps.systems.outputs.systems }}
+      mergedSha: ${{ steps.merged.outputs.mergedSha }}
     steps:
       # Important: Because of `pull_request_target`, this doesn't check out the PR,
       # but rather the base branch of the PR, which is needed so we don't run untrusted code
@@ -20,26 +20,92 @@ jobs:
           path: base
           sparse-checkout: ci
       - name: Resolving the merge commit
+        id: merged
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
             echo "Checking the merge commit $mergedSha"
-            echo "mergedSha=$mergedSha" >> "$GITHUB_ENV"
+            echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
           else
             # Skipping so that no notifications are sent
             echo "Skipping the rest..."
           fi
           rm -rf base
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         # Add this to _all_ subsequent steps to skip them
-        if: env.mergedSha
+        if: steps.merged.outputs.mergedSha
         with:
           ref: ${{ env.mergedSha }}
+          path: nixpkgs
+
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+        if: steps.merged.outputs.mergedSha
+
+      - id: systems
+        if: steps.merged.outputs.mergedSha
+        run: |
+          nix-build nixpkgs/ci -A eval.attrpathsSuperset
+          echo "systems=$(<result/systems.json)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: steps.merged.outputs.mergedSha
+        with:
+          name: paths.json
+          path: result/paths.json
+
+  eval:
+    name: Eval
+    runs-on: ubuntu-latest
+    needs: attrs
+    if: needs.attrs.outputs.mergedSha
+    strategy:
+      matrix:
+        system: ${{ fromJSON(needs.attrs.outputs.systems) }}
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: paths.json
+          path: paths.json
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ needs.attrs.outputs.mergedSha }}
+          path: nixpkgs
+
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+
+      - name: Check eval
+        run: nix-build nixpkgs/ci -A eval.singleSystem --argstr evalSystem ${{ matrix.system }} --arg attrpathFile ./paths.json
+
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: needs.attrs.outputs.mergedSha
+        with:
+          name: result-${ matrix.system }}
+          path: result/paths
+
+  combine:
+    name: Combined
+    runs-on: ubuntu-latest
+    needs: eval
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: result-*
+          path: results
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ needs.attrs.outputs.mergedSha }}
+          path: nixpkgs
 
       - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
-        if: env.mergedSha
 
       - name: Check eval
-        if: env.mergedSha
-        run: ./ci/eval-nixpkgs.sh --system "${{ matrix.system }}"
+        run: nix-build nixpkgs/ci -A eval.combine --arg resultsDir ./results
+
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: needs.attrs.outputs.mergedSha
+        with:
+          name: result/outpaths.json
+          path: outpaths.json

ci/default.nix

@@ -26,4 +26,5 @@ in
   inherit pkgs;
   requestReviews = pkgs.callPackage ./request-reviews { };
   codeownersValidator = pkgs.callPackage ./codeowners-validator { };
+  eval = pkgs.callPackage ./eval { };
 }