Skip to content

Commit 3364d66

Browse files
infinisilinfinisil
authored
Merge pull request #100 from Mic92/gha-eval-fixes
ci/parallel.nix: nixfmt
2 parents 4874faf + af2417f commit 3364d66

File tree

3 files changed

+92
-70
lines changed

3 files changed

+92
-70
lines changed

‎.github/workflows/eval.yml

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,6 @@ jobs:
4040
- uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
4141
if: env.mergedSha
4242

43-
- name: Enable swap
44-
if: env.mergedSha
45-
run: |
46-
sudo fallocate -l 10G /swapfile
47-
sudo chmod 600 /swapfile
48-
sudo mkswap /swapfile
49-
sudo swapon /swapfile
50-
5143
- name: Check eval
5244
if: env.mergedSha
5345
run: ./ci/eval-nixpkgs.sh --system "${{ matrix.system }}"

‎ci/eval-nixpkgs.sh

Lines changed: 56 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,88 @@
11
#!/usr/bin/env nix-shell
2-
#!nix-shell -i bash -p moreutils -I nixpkgs=channel:nixpkgs-unstable
2+
#!nix-shell -i bash -p coreutils moreutils -I nixpkgs=channel:nixpkgs-unstable
33

44
set -euxo pipefail
55

6-
system="x86_64-linux"
7-
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
6+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
87
NIXPKGS_PATH="$(readlink -f "$SCRIPT_DIR"/..)"
98

9+
system="x86_64-linux"
10+
quick_test=0
11+
CORES=$(nproc)
12+
1013
parseArgs() {
11-
while [[ $# -gt 0 ]]; do
12-
case $1 in
13-
--system)
14-
system=$2
15-
shift 2
16-
;;
17-
*)
18-
echo "Unknown argument: $1"
19-
exit 1
20-
;;
21-
esac
22-
done
14+
while [[ $# -gt 0 ]]; do
15+
arg=$1
16+
shift
17+
case "$arg" in
18+
--system)
19+
system=$1
20+
shift 1
21+
;;
22+
--cores)
23+
CORES=$1
24+
shift 1
25+
;;
26+
--quick-test)
27+
quick_test=1
28+
;;
29+
*)
30+
echo "Unknown argument: $arg"
31+
exit 1
32+
;;
33+
esac
34+
done
2335
}
2436

2537
main() {
2638
parseArgs "$@"
2739
tmpdir=$(mktemp -d)
2840
trap 'rm -rf "$tmpdir"' EXIT
2941

30-
nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths > "$tmpdir/paths.json"
42+
nix-instantiate --eval --strict --json --arg enableWarnings false "$NIXPKGS_PATH"/pkgs/top-level/release-attrpaths-superset.nix -A paths >"$tmpdir/paths.json"
3143

32-
CORES=$(nproc)
3344
# Originally @amjoseph: note that the number of processes spawned is four times
3445
# the number of cores -- this helps in two ways:
3546
# 1. Keeping cores busy while I/O operations are in flight
3647
# 2. Since the amount of time needed for the jobs is *not* balanced
3748
# this minimizes the "tail latency" for the very last job to finish
3849
# (on one core) by making the job size smaller.
39-
NUM_CHUNKS=$(( 4 * CORES ))
40-
50+
local num_chunks=$((4 * CORES))
51+
local seq_end=$((num_chunks - 1))
52+
if [[ $quick_test -eq 1 ]]; then
53+
seq_end=0
54+
fi
4155

4256
(
43-
set +e
44-
parallel -j "$CORES" \
45-
nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \
46-
--arg systems "[\"$system\"]" \
47-
-f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \
48-
--arg numChunks "$NUM_CHUNKS" --show-trace --arg myChunk \
49-
-- $(seq 0 $(( NUM_CHUNKS - 1 ))) > "$tmpdir/paths"
50-
echo $? > "$tmpdir/exit-code"
57+
set +e
58+
seq 0 $seq_end | xargs -P "$CORES" -I {} nix-env -qaP --no-name --out-path --arg checkMeta true --arg includeBroken true \
59+
--arg systems "[\"$system\"]" \
60+
-f "$NIXPKGS_PATH"/ci/parallel.nix --arg attrPathFile "$tmpdir"/paths.json \
61+
--arg numChunks "$num_chunks" --show-trace --arg myChunk {} >"$tmpdir/paths"
62+
echo $? >"$tmpdir/exit-code"
5163
) &
5264
pid=$!
5365
while kill -0 "$pid"; do
5466
free -g >&2
5567
sleep 20
5668
done
57-
jq --raw-input --slurp 'split("\n") | map(select(. != "") | split(" ") | map(select(. != "")) | { key: .[0], value: .[1] }) | from_entries' "$tmpdir/paths"
69+
70+
# Extract derivation outputs for each attribute into a JSON object
71+
# i.e. { "attr1": { "out": "/nix/store/...", "dev": "/nix/store/..." }, ... }
72+
jq --raw-input --slurp '
73+
split("\n") |
74+
map(select(. != "") | split(" ") | map(select(. != ""))) |
75+
map(
76+
{
77+
key: .[0],
78+
value: .[1] | split(";") | map(split("=") |
79+
if length == 1 then
80+
{ key: "out", value: .[0] }
81+
else
82+
{ key: .[0], value: .[1] }
83+
end) | from_entries}
84+
) | from_entries
85+
' "$tmpdir/paths"
5886
exit "$(cat "$tmpdir/exit-code")"
5987
}
6088

‎ci/parallel.nix

Lines changed: 36 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,63 @@
11
/*
2-
Invocation:
2+
Invocation:
33
4+
Invocation; note that the number of processes spawned is four times
5+
the number of cores -- this helps in two ways:
46
5-
Invocation; note that the number of processes spawned is four times
6-
the number of cores -- this helps in two ways:
7-
8-
1. Keeping cores busy while I/O operations are in flight
9-
10-
2. Since the amount of time needed for the jobs is *not* balanced
11-
this minimizes the "tail latency" for the very last job to finish
12-
(on one core) by making the job size smaller.
7+
1. Keeping cores busy while I/O operations are in flight
138
9+
2. Since the amount of time needed for the jobs is *not* balanced
10+
this minimizes the "tail latency" for the very last job to finish
11+
(on one core) by making the job size smaller.
1412
*/
1513
# see pkgs/top-level/nohydra
16-
{ lib ? import ../lib
17-
, checkMeta
18-
, includeBroken ? true
19-
, path ? ./..
20-
, systems
21-
, myChunk
22-
, numChunks
23-
, attrPathFile
14+
{
15+
lib ? import ../lib,
16+
checkMeta,
17+
includeBroken ? true,
18+
path ? ./..,
19+
systems,
20+
myChunk,
21+
numChunks,
22+
attrPathFile,
2423
}:
2524

2625
let
2726
attrPaths = builtins.fromJSON (builtins.readFile attrPathFile);
2827
chunkSize = (lib.length attrPaths) / numChunks;
2928
myPaths =
3029
let
31-
dropped = lib.drop (chunkSize*myChunk) attrPaths;
30+
dropped = lib.drop (chunkSize * myChunk) attrPaths;
3231
in
33-
if myChunk == numChunks - 1
34-
then dropped
35-
else lib.take chunkSize dropped;
32+
if myChunk == numChunks - 1 then dropped else lib.take chunkSize dropped;
3633

3734
unfiltered = import ../pkgs/top-level/release-outpaths.nix {
38-
inherit checkMeta path includeBroken systems;
35+
inherit
36+
checkMeta
37+
path
38+
includeBroken
39+
systems
40+
;
3941
};
4042

41-
f = i: m: a:
42-
lib.mapAttrs (name: values:
43+
f =
44+
i: m: a:
45+
lib.mapAttrs (
46+
name: values:
4347
if a ? ${name} then
44-
if lib.any (value: lib.length value <= i + 1) values then
45-
a.${name}
46-
else
47-
f (i + 1) values a.${name}
48+
if lib.any (value: lib.length value <= i + 1) values then a.${name} else f (i + 1) values a.${name}
4849
else
4950
null
5051
) (lib.groupBy (a: lib.elemAt a i) m);
5152

5253
filtered = f 0 myPaths unfiltered;
5354

54-
recurseEverywhere = val:
55-
if lib.isDerivation val || !(lib.isAttrs val)
56-
then val
57-
else (builtins.mapAttrs (_: v: recurseEverywhere v) val)
58-
// { recurseForDerivations = true; };
55+
recurseEverywhere =
56+
val:
57+
if lib.isDerivation val || !(lib.isAttrs val) then
58+
val
59+
else
60+
(builtins.mapAttrs (_: v: recurseEverywhere v) val) // { recurseForDerivations = true; };
5961

6062
in
61-
recurseEverywhere filtered
63+
recurseEverywhere filtered

0 commit comments

Comments
 (0)