Skip to content

Commit 13717ef

Browse files
infinisilinfinisil
committed
workflows/eval.yml: Run on dev branch pushes and compute outpath difference
1 parent 0892479 commit 13717ef

File tree

4 files changed

+234
-16
lines changed

4 files changed

+234
-16
lines changed

‎.github/workflows/eval.yml

Lines changed: 95 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
name: Eval
22

3-
on: pull_request_target
3+
on:
4+
pull_request_target:
5+
push:
6+
# Keep this synced with ci/request-reviews/dev-branches.txt
7+
branches:
8+
- master
9+
- staging
10+
- release-*
11+
- staging-*
12+
- haskell-updates
413

514
permissions:
615
contents: read
@@ -11,6 +20,7 @@ jobs:
1120
runs-on: ubuntu-latest
1221
outputs:
1322
mergedSha: ${{ steps.merged.outputs.mergedSha }}
23+
baseSha: ${{ steps.baseSha.outputs.baseSha }}
1424
systems: ${{ steps.systems.outputs.systems }}
1525
steps:
1626
# Important: Because of `pull_request_target`, this doesn't check out the PR,
@@ -24,23 +34,39 @@ jobs:
2434
id: merged
2535
env:
2636
GH_TOKEN: ${{ github.token }}
37+
GH_EVENT: ${{ github.event_name }}
2738
run: |
28-
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
29-
echo "Checking the merge commit $mergedSha"
30-
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
31-
else
32-
# Skipping so that no notifications are sent
33-
echo "Skipping the rest..."
34-
fi
39+
case "$GH_EVENT" in
40+
push)
41+
echo "mergedSha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
42+
;;
43+
pull_request_target)
44+
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
45+
echo "Checking the merge commit $mergedSha"
46+
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
47+
else
48+
# Skipping so that no notifications are sent
49+
echo "Skipping the rest..."
50+
fi
51+
;;
52+
esac
3553
rm -rf base
3654
- name: Check out the PR at the test merge commit
3755
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
3856
# Add this to _all_ subsequent steps to skip them
3957
if: steps.merged.outputs.mergedSha
4058
with:
4159
ref: ${{ steps.merged.outputs.mergedSha }}
60+
fetch-depth: 2
4261
path: nixpkgs
4362

63+
- name: Determine base commit
64+
if: github.event_name == 'pull_request_target' && steps.merged.outputs.mergedSha
65+
id: baseSha
66+
run: |
67+
baseSha=$(git -C nixpkgs rev-parse HEAD^1)
68+
echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT"
69+
4470
- name: Install Nix
4571
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
4672
if: steps.merged.outputs.mergedSha
@@ -124,18 +150,71 @@ jobs:
124150
- name: Combine all output paths and eval stats
125151
run: |
126152
nix-build nixpkgs/ci -A eval.combine \
127-
--arg resultsDir ./intermediate
153+
--arg resultsDir ./intermediate \
154+
-o prResult
128155
129156
- name: Upload the combined results
130157
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
131158
with:
132159
name: result
133-
path: result/*
160+
path: prResult/*
161+
162+
- name: Get base run id
163+
if: needs.attrs.outputs.baseSha
164+
id: baseRunId
165+
run: |
166+
set -e
167+
168+
# TODO: Wait until it's done
169+
# Get the latest eval.yml workflow run for the PR's base commit
170+
if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
171+
-f head_sha="$BASE_SHA" \
172+
--jq '.workflow_runs | sort_by(.run_started_at) | .[-1]')
173+
|| [[ -z "$run" ]]; then
174+
echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
175+
exit 0
176+
fi
177+
echo "Comparing against $(jq .html_url <<< "$run")"
178+
runId=$(jq .id <<< "$run")
179+
conclusion=$(jq .conclusion <<< "$run")
180+
181+
while [[ "$conclusion" == null ]]; do
182+
echo "Workflow not done, waiting 10 seconds before checking again"
183+
sleep 10
184+
conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
185+
done
186+
187+
if [[ "$conclusion" != "success" ]]; then
188+
echo "Workflow was not successful, cannot make comparison"
189+
exit 0
190+
fi
191+
192+
echo "baseRunId=$runId" >> "$GITHUB_OUTPUT"
193+
env:
194+
REPOSITORY: ${{ github.repository }}
195+
BASE_SHA: ${{ needs.attrs.outputs.baseSha }}
134196

197+
- uses: actions/download-artifact@v4
198+
if: steps.baseRunId.outputs.baseRunId
199+
with:
200+
name: result
201+
path: baseResult
202+
github-token: ${{ github.token }}
203+
run-id: ${{ steps.baseRunId.outputs.baseRunId }}
204+
205+
- name: Compare against the base branch
206+
if: steps.baseRunId.outputs.baseRunId
207+
run: |
208+
nix-build nixpkgs/ci -A eval.compare \
209+
--arg beforeResultDir ./baseResult \
210+
--arg afterResultDir ./prResult \
211+
-o comparison
135212
136-
# TODO: Run this workflow also on `push` (on at least the main development branches)
137-
# Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)
138-
# to have completed the eval, then use
139-
# gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>
140-
# and follow it to the artifact results, where you can then download the outpaths.json from the base branch
141-
# That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers
213+
# TODO: Request reviews from maintainers for packages whose files are modified in the PR
214+
215+
- name: Upload the combined results
216+
if: steps.baseRunId.outputs.baseRunId
217+
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
218+
with:
219+
name: comparison
220+
path: comparison/*

‎ci/eval/default.nix

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,22 @@ let
238238
jq -s from_entries > $out/stats.json
239239
'';
240240

241+
compare =
242+
{ beforeResultDir, afterResultDir }:
243+
runCommand "compare" {
244+
nativeBuildInputs = [
245+
jq
246+
];
247+
} ''
248+
mkdir $out
249+
jq -n -f ${./filter.jq} \
250+
--slurpfile before ${beforeResultDir}/outpaths.json \
251+
--slurpfile after ${afterResultDir}/outpaths.json \
252+
> $out/changed-paths.json
253+
254+
# TODO: Compare eval stats
255+
'';
256+
241257
full =
242258
{
243259
# Whether to evaluate just a single system, by default all are evaluated
@@ -262,12 +278,14 @@ let
262278
resultsDir = results;
263279
};
264280

281+
265282
in
266283
{
267284
inherit
268285
attrpathsSuperset
269286
singleSystem
270287
combine
288+
compare
271289
# The above three are used by separate VMs in a GitHub workflow,
272290
# while the below is intended for testing on a single local machine
273291
full

‎ci/eval/filter.jq

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Turns
2+
#
3+
# {
4+
# "hello.aarch64-linux": "a",
5+
# "hello.x86_64-linux": "b",
6+
# "hello.aarch64-darwin": "c",
7+
# "hello.x86_64-darwin": "d"
8+
# }
9+
#
10+
# into
11+
#
12+
# {
13+
# "hello": {
14+
# "linux": {
15+
# "aarch64": "a",
16+
# "x86_64": "b"
17+
# },
18+
# "darwin": {
19+
# "aarch64": "c",
20+
# "x86_64": "d"
21+
# }
22+
# }
23+
# }
24+
#
25+
# while filtering out any attribute paths that don't match this pattern
26+
def expand_system:
27+
to_entries
28+
| map(
29+
.key |= split(".")
30+
| select(.key | length > 1)
31+
| .double = (.key[-1] | split("-"))
32+
| select(.double | length == 2)
33+
)
34+
| group_by(.key[0:-1])
35+
| map(
36+
{
37+
key: .[0].key[0:-1] | join("."),
38+
value:
39+
group_by(.double[1])
40+
| map(
41+
{
42+
key: .[0].double[1],
43+
value: map(.key = .double[0]) | from_entries
44+
}
45+
)
46+
| from_entries
47+
})
48+
| from_entries
49+
;
50+
51+
# Transposes
52+
#
53+
# {
54+
# "a": [ "x", "y" ],
55+
# "b": [ "x" ],
56+
# }
57+
#
58+
# into
59+
#
60+
# {
61+
# "x": [ "a", "b" ],
62+
# "y": [ "a" ]
63+
# }
64+
def transpose:
65+
[
66+
to_entries[]
67+
| {
68+
key: .key,
69+
value: .value[]
70+
}
71+
]
72+
| group_by(.value)
73+
| map({
74+
key: .[0].value,
75+
value: map(.key)
76+
})
77+
| from_entries
78+
;
79+
80+
# Computes the key difference for two objects:
81+
# {
82+
# added: [ <keys only in the second object> ],
83+
# removed: [ <keys only in the first object> ],
84+
# changed: [ <keys with different values between the two objects> ],
85+
# }
86+
#
87+
def diff($before; $after):
88+
{
89+
added: $after | delpaths($before | keys | map([.])) | keys,
90+
removed: $before | delpaths($after | keys | map([.])) | keys,
91+
changed:
92+
$before
93+
| to_entries
94+
| map(
95+
$after."\(.key)" as $after2
96+
| select(
97+
# Filter out attributes that don't exist anymore
98+
($after2 != null)
99+
and
100+
# Filter out attributes that are the same as the new value
101+
(.value != $after2)
102+
)
103+
| .key
104+
)
105+
}
106+
;
107+
108+
($before[0] | expand_system) as $before
109+
| ($after[0] | expand_system) as $after
110+
| .attrdiff = diff($before; $after)
111+
| .rebuildsByKernel = (
112+
.attrdiff.changed
113+
| map({
114+
key: .,
115+
value: diff($before."\(.)"; $after."\(.)").changed
116+
})
117+
| from_entries
118+
| transpose
119+
)
120+
| .rebuildCountByKernel = (.rebuildsByKernel | with_entries(.value |= length))

‎ci/request-reviews/dev-branches.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Trusted development branches:
22
# These generally require PRs to update and are built by Hydra.
3+
# Keep this synced with the branches in .github/workflows/eval.yml
34
master
45
staging
56
release-*

0 commit comments

Comments
 (0)