Skip to content

Commit 1053654

Browse files
infinisilinfinisil
committed
workflows/eval.yml: Run on dev branch pushes and compute outpath difference
1 parent 0892479 commit 1053654

File tree

4 files changed

+296
-16
lines changed

4 files changed

+296
-16
lines changed

‎.github/workflows/eval.yml

Lines changed: 125 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
name: Eval
22

3-
on: pull_request_target
3+
on:
4+
pull_request_target:
5+
push:
6+
# Keep this synced with ci/request-reviews/dev-branches.txt
7+
branches:
8+
- master
9+
- staging
10+
- release-*
11+
- staging-*
12+
- haskell-updates
13+
- python-updates
414

515
permissions:
616
contents: read
@@ -11,6 +21,7 @@ jobs:
1121
runs-on: ubuntu-latest
1222
outputs:
1323
mergedSha: ${{ steps.merged.outputs.mergedSha }}
24+
baseSha: ${{ steps.baseSha.outputs.baseSha }}
1425
systems: ${{ steps.systems.outputs.systems }}
1526
steps:
1627
# Important: Because of `pull_request_target`, this doesn't check out the PR,
@@ -24,23 +35,39 @@ jobs:
2435
id: merged
2536
env:
2637
GH_TOKEN: ${{ github.token }}
38+
GH_EVENT: ${{ github.event_name }}
2739
run: |
28-
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
29-
echo "Checking the merge commit $mergedSha"
30-
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
31-
else
32-
# Skipping so that no notifications are sent
33-
echo "Skipping the rest..."
34-
fi
40+
case "$GH_EVENT" in
41+
push)
42+
echo "mergedSha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
43+
;;
44+
pull_request_target)
45+
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
46+
echo "Checking the merge commit $mergedSha"
47+
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
48+
else
49+
# Skipping so that no notifications are sent
50+
echo "Skipping the rest..."
51+
fi
52+
;;
53+
esac
3554
rm -rf base
3655
- name: Check out the PR at the test merge commit
3756
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
3857
# Add this to _all_ subsequent steps to skip them
3958
if: steps.merged.outputs.mergedSha
4059
with:
4160
ref: ${{ steps.merged.outputs.mergedSha }}
61+
fetch-depth: 2
4262
path: nixpkgs
4363

64+
- name: Determine base commit
65+
if: github.event_name == 'pull_request_target' && steps.merged.outputs.mergedSha
66+
id: baseSha
67+
run: |
68+
baseSha=$(git -C nixpkgs rev-parse HEAD^1)
69+
echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT"
70+
4471
- name: Install Nix
4572
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
4673
if: steps.merged.outputs.mergedSha
@@ -105,6 +132,8 @@ jobs:
105132
name: Process
106133
runs-on: ubuntu-latest
107134
needs: [ outpaths, attrs ]
135+
outputs:
136+
baseRunId: ${{ steps.baseRunId.outputs.baseRunId }}
108137
steps:
109138
- name: Download output paths and eval stats for all systems
110139
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -124,18 +153,98 @@ jobs:
124153
- name: Combine all output paths and eval stats
125154
run: |
126155
nix-build nixpkgs/ci -A eval.combine \
127-
--arg resultsDir ./intermediate
156+
--arg resultsDir ./intermediate \
157+
-o prResult
128158
129159
- name: Upload the combined results
130160
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
131161
with:
132162
name: result
133-
path: result/*
163+
path: prResult/*
134164

165+
- name: Get base run id
166+
if: needs.attrs.outputs.baseSha
167+
id: baseRunId
168+
run: |
169+
set -e
170+
171+
# TODO: Wait until it's done
172+
# Get the latest eval.yml workflow run for the PR's base commit
173+
if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
174+
-f head_sha="$BASE_SHA" \
175+
--jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
176+
|| [[ -z "$run" ]]; then
177+
echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
178+
exit 0
179+
fi
180+
echo "Comparing against $(jq .html_url <<< "$run")"
181+
runId=$(jq .id <<< "$run")
182+
conclusion=$(jq -r .conclusion <<< "$run")
183+
184+
while [[ "$conclusion" == null ]]; do
185+
echo "Workflow not done, waiting 10 seconds before checking again"
186+
sleep 10
187+
conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
188+
done
189+
190+
if [[ "$conclusion" != "success" ]]; then
191+
echo "Workflow was not successful, cannot make comparison"
192+
exit 0
193+
fi
135194
136-
# TODO: Run this workflow also on `push` (on at least the main development branches)
137-
# Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)
138-
# to have completed the eval, then use
139-
# gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>
140-
# and follow it to the artifact results, where you can then download the outpaths.json from the base branch
141-
# That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers
195+
echo "baseRunId=$runId" >> "$GITHUB_OUTPUT"
196+
env:
197+
REPOSITORY: ${{ github.repository }}
198+
BASE_SHA: ${{ needs.attrs.outputs.baseSha }}
199+
GH_TOKEN: ${{ github.token }}
200+
201+
- uses: actions/download-artifact@v4
202+
if: steps.baseRunId.outputs.baseRunId
203+
with:
204+
name: result
205+
path: baseResult
206+
github-token: ${{ github.token }}
207+
run-id: ${{ steps.baseRunId.outputs.baseRunId }}
208+
209+
- name: Compare against the base branch
210+
if: steps.baseRunId.outputs.baseRunId
211+
run: |
212+
nix-build nixpkgs/ci -A eval.compare \
213+
--arg beforeResultDir ./baseResult \
214+
--arg afterResultDir ./prResult \
215+
-o comparison
216+
217+
# TODO: Request reviews from maintainers for packages whose files are modified in the PR
218+
219+
- name: Upload the combined results
220+
if: steps.baseRunId.outputs.baseRunId
221+
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
222+
with:
223+
name: comparison
224+
path: comparison/*
225+
226+
# Separate job to have a very tightly scoped PR write token
227+
tag:
228+
name: Tag
229+
runs-on: ubuntu-latest
230+
needs: process
231+
if: needs.process.outputs.baseRunId
232+
permissions:
233+
pull-requests: write
234+
steps:
235+
- name: Download process result
236+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
237+
with:
238+
name: comparison
239+
path: comparison
240+
241+
- name: Tagging pull request
242+
run: |
243+
ls -laa
244+
ls -laa comparison
245+
gh api \
246+
--method POST \
247+
/repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \
248+
--input <(jq -c '{ labels: .labels }' comparison/changed-paths.json)
249+
env:
250+
GH_TOKEN: ${{ github.token }}

‎ci/eval/default.nix

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,22 @@ let
238238
jq -s from_entries > $out/stats.json
239239
'';
240240

241+
compare =
242+
{ beforeResultDir, afterResultDir }:
243+
runCommand "compare" {
244+
nativeBuildInputs = [
245+
jq
246+
];
247+
} ''
248+
mkdir $out
249+
jq -n -f ${./filter.jq} \
250+
--slurpfile before ${beforeResultDir}/outpaths.json \
251+
--slurpfile after ${afterResultDir}/outpaths.json \
252+
> $out/changed-paths.json
253+
254+
# TODO: Compare eval stats
255+
'';
256+
241257
full =
242258
{
243259
# Whether to evaluate just a single system, by default all are evaluated
@@ -262,12 +278,14 @@ let
262278
resultsDir = results;
263279
};
264280

281+
265282
in
266283
{
267284
inherit
268285
attrpathsSuperset
269286
singleSystem
270287
combine
288+
compare
271289
# The above three are used by separate VMs in a GitHub workflow,
272290
# while the below is intended for testing on a single local machine
273291
full

‎ci/eval/filter.jq

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# Turns
2+
#
3+
# {
4+
# "hello.aarch64-linux": "a",
5+
# "hello.x86_64-linux": "b",
6+
# "hello.aarch64-darwin": "c",
7+
# "hello.x86_64-darwin": "d"
8+
# }
9+
#
10+
# into
11+
#
12+
# {
13+
# "hello": {
14+
# "linux": {
15+
# "aarch64": "a",
16+
# "x86_64": "b"
17+
# },
18+
# "darwin": {
19+
# "aarch64": "c",
20+
# "x86_64": "d"
21+
# }
22+
# }
23+
# }
24+
#
25+
# while filtering out any attribute paths that don't match this pattern
26+
def expand_system:
27+
to_entries
28+
| map(
29+
.key |= split(".")
30+
| select(.key | length > 1)
31+
| .double = (.key[-1] | split("-"))
32+
| select(.double | length == 2)
33+
)
34+
| group_by(.key[0:-1])
35+
| map(
36+
{
37+
key: .[0].key[0:-1] | join("."),
38+
value:
39+
group_by(.double[1])
40+
| map(
41+
{
42+
key: .[0].double[1],
43+
value: map(.key = .double[0]) | from_entries
44+
}
45+
)
46+
| from_entries
47+
})
48+
| from_entries
49+
;
50+
51+
# Transposes
52+
#
53+
# {
54+
# "a": [ "x", "y" ],
55+
# "b": [ "x" ],
56+
# }
57+
#
58+
# into
59+
#
60+
# {
61+
# "x": [ "a", "b" ],
62+
# "y": [ "a" ]
63+
# }
64+
def transpose:
65+
[
66+
to_entries[]
67+
| {
68+
key: .key,
69+
value: .value[]
70+
}
71+
]
72+
| group_by(.value)
73+
| map({
74+
key: .[0].value,
75+
value: map(.key)
76+
})
77+
| from_entries
78+
;
79+
80+
# Computes the key difference for two objects:
81+
# {
82+
# added: [ <keys only in the second object> ],
83+
# removed: [ <keys only in the first object> ],
84+
# changed: [ <keys with different values between the two objects> ],
85+
# }
86+
#
87+
def diff($before; $after):
88+
{
89+
added: $after | delpaths($before | keys | map([.])) | keys,
90+
removed: $before | delpaths($after | keys | map([.])) | keys,
91+
changed:
92+
$before
93+
| to_entries
94+
| map(
95+
$after."\(.key)" as $after2
96+
| select(
97+
# Filter out attributes that don't exist anymore
98+
($after2 != null)
99+
and
100+
# Filter out attributes that are the same as the new value
101+
(.value != $after2)
102+
)
103+
| .key
104+
)
105+
}
106+
;
107+
108+
($before[0] | expand_system) as $before
109+
| ($after[0] | expand_system) as $after
110+
| .attrdiff = diff($before; $after)
111+
| .rebuildsByKernel = (
112+
.attrdiff.changed
113+
| map({
114+
key: .,
115+
value: diff($before."\(.)"; $after."\(.)").changed
116+
})
117+
| from_entries
118+
| transpose
119+
)
120+
| .rebuildCountByKernel = (
121+
.rebuildsByKernel
122+
| with_entries(.value |= length)
123+
| pick(.linux, .darwin)
124+
| {
125+
linux: (.linux // 0),
126+
darwin: (.darwin // 0),
127+
}
128+
)
129+
| .labels = (
130+
.rebuildCountByKernel
131+
| to_entries
132+
| map(
133+
"10.rebuild-\(.key): " +
134+
if .value == 0 then
135+
"0"
136+
elif .value <= 10 then
137+
"1-10"
138+
elif .value <= 100 then
139+
"11-100"
140+
elif .value <= 500 then
141+
"101-500"
142+
elif .value <= 1000 then
143+
"501-1000"
144+
elif .value <= 2500 then
145+
"1001-2500"
146+
elif .value <= 5000 then
147+
"2501-5000"
148+
else
149+
"5000+"
150+
end
151+
)
152+
)

‎ci/request-reviews/dev-branches.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Trusted development branches:
22
# These generally require PRs to update and are built by Hydra.
3+
# Keep this synced with the branches in .github/workflows/eval.yml
34
master
45
staging
56
release-*

0 commit comments

Comments
 (0)