From f80ecb0fdd593550a1db3afb7bca9a1a08270b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 22 Nov 2024 08:25:36 +0100 Subject: [PATCH 01/19] ci/editorconfig-v2: useless use of cat (cherry picked from commit b998723321461f1edfc373702dc57897d1cb6e44) --- .github/workflows/editorconfig-v2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/editorconfig-v2.yml b/.github/workflows/editorconfig-v2.yml index 99bee8b301228..07afb60bc3ae0 100644 --- a/.github/workflows/editorconfig-v2.yml +++ b/.github/workflows/editorconfig-v2.yml @@ -42,7 +42,7 @@ jobs: nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/c473cc8714710179df205b153f4e9fa007107ff9.tar.gz - name: Checking EditorConfig run: | - cat "$HOME/changed_files" | nix-shell -p editorconfig-checker --run 'xargs -r editorconfig-checker -disable-indent-size' + < "$HOME/changed_files" nix-shell -p editorconfig-checker --run 'xargs -r editorconfig-checker -disable-indent-size' - if: ${{ failure() }} run: | echo "::error :: Hey! It looks like your changes don't follow our editorconfig settings. Read https://editorconfig.org/#download to configure your editor so you never see this error again." From 934a91c9b741546fa0734a1a9bfa2804eb6e4a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 22 Nov 2024 08:31:48 +0100 Subject: [PATCH 02/19] ci/check-nixf-tidy: replace sed with variable substitution Update .github/workflows/check-nixf-tidy.yml Co-authored-by: Zhong Jianxin (cherry picked from commit 2adf40958121195ad7adb5d1926a876977fe7faa) --- .github/workflows/check-nixf-tidy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-nixf-tidy.yml b/.github/workflows/check-nixf-tidy.yml index de74ae8b5372a..481ae2df4c31f 100644 --- a/.github/workflows/check-nixf-tidy.yml +++ b/.github/workflows/check-nixf-tidy.yml @@ -107,7 +107,7 @@ jobs: echo "$errors" else # just print in plain text - echo "$errors" | sed 's/^:://' + echo "${errors/::/}" echo # add one empty line fi failedFiles+=("$dest") From f2280139b398447fa6ec53d9ed254508b1d70ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 22 Nov 2024 08:32:01 +0100 Subject: [PATCH 03/19] add actionlint script (cherry picked from commit eeb87082a9c5cd3eb72ecde71a06831256ef748b) --- .github/workflows/lint-actions.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 .github/workflows/lint-actions.sh diff --git a/.github/workflows/lint-actions.sh b/.github/workflows/lint-actions.sh new file mode 100755 index 0000000000000..43d6e801caf68 --- /dev/null +++ b/.github/workflows/lint-actions.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash actionlint shellcheck -I nixpkgs=../.. +set -euo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +cd "$SCRIPT_DIR/../.." +actionlint From 5191c1a687ec79928a1e253badda2814bd75df75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 29 Nov 2024 23:34:12 +0100 Subject: [PATCH 04/19] ci/check-shell: only run if `shell.nix` or `./ci/**` is changed saves a bit of CI time (cherry picked from commit cb016f116bf8134bb2095cdec67cf37eae747a04) --- .github/workflows/check-shell.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/check-shell.yml b/.github/workflows/check-shell.yml index 82bc43fb92941..18ec9780b56b9 100644 --- a/.github/workflows/check-shell.yml +++ b/.github/workflows/check-shell.yml @@ -2,6 +2,9 @@ name: "Check shell" on: pull_request_target: + paths: + - 'shell.nix' + - './ci/**' permissions: {} From e30b136818342b548a4614c5100453193d1a8c0e Mon Sep 17 00:00:00 2001 From: Zhong Jianxin Date: Sat, 30 Nov 2024 10:32:25 +0800 Subject: [PATCH 05/19] ci/check-shell: fix `ci/**` path (cherry picked from commit c318085efa8153fa8cb957d3e2ff1c63dd76413f) --- .github/workflows/check-shell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-shell.yml b/.github/workflows/check-shell.yml index 18ec9780b56b9..316813879e81c 100644 --- a/.github/workflows/check-shell.yml +++ b/.github/workflows/check-shell.yml @@ -4,7 +4,7 @@ on: pull_request_target: paths: - 'shell.nix' - - './ci/**' + - 'ci/**' permissions: {} From 887ea689dac5a7bda1b8068576fd5b6234eaae12 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 27 Aug 2024 00:32:49 +0200 Subject: [PATCH 06/19] workflows/check-nix-format: Improve error message Looks like the error message could be a bit clearer still: https://github.com/NixOS/nixpkgs/pull/337109#issuecomment-2311175326 (cherry picked from commit 249d4a97d51c33b94fb2653c3bfd2659d45f9889) --- .github/workflows/check-nix-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-nix-format.yml b/.github/workflows/check-nix-format.yml index 501f1823b3a86..19f80085c5cc5 100644 --- a/.github/workflows/check-nix-format.yml +++ b/.github/workflows/check-nix-format.yml @@ -87,7 +87,7 @@ jobs: if (( "${#unformattedFiles[@]}" > 0 )); then echo "Some new/changed Nix files are not properly formatted" - echo "Please go to the Nixpkgs root directory, run \`nix-shell\`, then:" + echo "Please format them using the Nixpkgs-specific \`nixfmt\` by going to the Nixpkgs root directory, running \`nix-shell\`, then:" echo "nixfmt ${unformattedFiles[*]@Q}" echo "Make sure your branch is up to date with master, rebase if not." echo "If you're having trouble, please ping @NixOS/nix-formatting" From 1388b51a482ea907675a25cc7ccbc506e6380cae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:32:53 +0000 Subject: [PATCH 07/19] build(deps): bump korthout/backport-action from 3.0.2 to 3.1.0 Bumps [korthout/backport-action](https://github.com/korthout/backport-action) from 3.0.2 to 3.1.0. - [Release notes](https://github.com/korthout/backport-action/releases) - [Commits](https://github.com/korthout/backport-action/compare/bd410d37cdcae80be6d969823ff5a225fe5c833f...be567af183754f6a5d831ae90f648954763f17f5) --- updated-dependencies: - dependency-name: korthout/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] (cherry picked from commit 8ba961591c8591212d171b74a803ad89e9117c57) --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 36db5d7cf0aca..159175ceb6346 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -28,7 +28,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} token: ${{ steps.app-token.outputs.token }} - name: Create backport PRs - uses: korthout/backport-action@bd410d37cdcae80be6d969823ff5a225fe5c833f # v3.0.2 + uses: korthout/backport-action@be567af183754f6a5d831ae90f648954763f17f5 # v3.1.0 with: # Config README: https://github.com/korthout/backport-action#backport-action copy_labels_pattern: 'severity:\ssecurity' From 3f4f84fdc78dd7ee6a2019bed430fcf0e4d4fdd7 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 7 Dec 2024 16:31:33 +0100 Subject: [PATCH 08/19] ci/eval: re-implement compare in nix (cherry picked from commit f94b4bd94574ee5dc59dba345e22258a6f077bd5) --- ci/eval/compare.jq | 164 ------------------ ci/eval/compare/default.nix | 53 ++++++ .../{ => compare}/generate-step-summary.jq | 0 ci/eval/compare/utils.nix | 130 ++++++++++++++ ci/eval/default.nix | 28 ++- 5 files changed, 193 insertions(+), 182 deletions(-) delete mode 100644 ci/eval/compare.jq create mode 100644 ci/eval/compare/default.nix rename ci/eval/{ => compare}/generate-step-summary.jq (100%) create mode 100644 ci/eval/compare/utils.nix diff --git a/ci/eval/compare.jq b/ci/eval/compare.jq deleted file mode 100644 index 9f3a032d13841..0000000000000 --- a/ci/eval/compare.jq +++ /dev/null @@ -1,164 +0,0 @@ -# Turns -# -# { -# "hello.aarch64-linux": "a", -# "hello.x86_64-linux": "b", -# "hello.aarch64-darwin": "c", -# "hello.x86_64-darwin": "d" -# } -# -# into -# -# { -# "hello": { -# "linux": { -# "aarch64": "a", -# "x86_64": "b" -# }, -# "darwin": { -# "aarch64": "c", -# "x86_64": "d" -# } -# } -# } -# -# while filtering out any attribute paths that don't match this pattern -def expand_system: - to_entries - | map( - .key |= split(".") - | select(.key | length > 1) - | .double = (.key[-1] | split("-")) - | select(.double | length == 2) - ) - | group_by(.key[0:-1]) - | map( - { - key: .[0].key[0:-1] | join("."), - value: - group_by(.double[1]) - | map( - { - key: .[0].double[1], - value: map(.key = .double[0]) | from_entries - } - ) - | from_entries - }) - | from_entries - ; - -# Transposes -# -# { -# "a": [ "x", "y" ], -# "b": [ "x" ], -# } -# -# into -# -# { -# "x": [ "a", "b" ], -# "y": [ "a" ] -# } -def transpose: - [ - to_entries[] - | { - key: .key, - value: .value[] - } - ] - | group_by(.value) - | map({ - key: .[0].value, - value: map(.key) - }) - | from_entries - ; - -# Computes the key difference for two objects: -# { -# added: [ ], -# removed: [ ], -# changed: [ ], -# } -# -def diff($before; $after): - { - added: $after | delpaths($before | keys | map([.])) | keys, - removed: $before | delpaths($after | keys | map([.])) | keys, - changed: - $before - | to_entries - | map( - $after."\(.key)" as $after2 - | select( - # Filter out attributes that don't exist anymore - ($after2 != null) - and - # Filter out attributes that are the same as the new value - (.value != $after2) - ) - | .key - ) - } - ; - -($before[0] | expand_system) as $before -| ($after[0] | expand_system) as $after -| .attrdiff = diff($before; $after) -| .rebuildsByKernel = ( - [ - ( - .attrdiff.changed[] - | { - key: ., - value: diff($before."\(.)"; $after."\(.)").changed - } - ) - , - ( - .attrdiff.added[] - | { - key: ., - value: ($after."\(.)" | keys) - } - ) - ] - | from_entries - | transpose -) -| .rebuildCountByKernel = ( - .rebuildsByKernel - | with_entries(.value |= length) - | pick(.linux, .darwin) - | { - linux: (.linux // 0), - darwin: (.darwin // 0), - } -) -| .labels = ( - .rebuildCountByKernel - | to_entries - | map( - "10.rebuild-\(.key): " + - if .value == 0 then - "0" - elif .value <= 10 then - "1-10" - elif .value <= 100 then - "11-100" - elif .value <= 500 then - "101-500" - elif .value <= 1000 then - "501-1000" - elif .value <= 2500 then - "1001-2500" - elif .value <= 5000 then - "2501-5000" - else - "5001+" - end - ) -) diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix new file mode 100644 index 0000000000000..8b5e9059cd5b8 --- /dev/null +++ b/ci/eval/compare/default.nix @@ -0,0 +1,53 @@ +{ + lib, + jq, + runCommand, + writeText, + supportedSystems, + ... +}: +{ beforeResultDir, afterResultDir }: +let + inherit (import ./utils.nix { inherit lib; }) + diff + groupByKernel + extractPackageNames + getLabels + uniqueStrings + ; + + getAttrs = dir: builtins.fromJSON (builtins.readFile "${dir}/outpaths.json"); + beforeAttrs = getAttrs beforeResultDir; + afterAttrs = getAttrs afterResultDir; + + diffAttrs = diff beforeAttrs afterAttrs; + + changed-paths = + let + rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed); + + rebuildsByKernel = groupByKernel rebuilds; + rebuildCountByKernel = lib.mapAttrs ( + kernel: kernelRebuilds: lib.length kernelRebuilds + ) rebuildsByKernel; + in + writeText "changed-paths.json" ( + builtins.toJSON { + attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs; + inherit rebuildsByKernel rebuildCountByKernel; + labels = getLabels rebuildCountByKernel; + } + ); +in +runCommand "compare" + { + nativeBuildInputs = [ jq ]; + } + '' + mkdir $out + + cp ${changed-paths} $out/changed-paths.json + + jq -r -f ${./generate-step-summary.jq} < ${changed-paths} > $out/step-summary.md + # TODO: Compare eval stats + '' diff --git a/ci/eval/generate-step-summary.jq b/ci/eval/compare/generate-step-summary.jq similarity index 100% rename from ci/eval/generate-step-summary.jq rename to ci/eval/compare/generate-step-summary.jq diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix new file mode 100644 index 0000000000000..22353499d6993 --- /dev/null +++ b/ci/eval/compare/utils.nix @@ -0,0 +1,130 @@ +{ lib, ... }: +rec { + # Borrowed from https://github.com/NixOS/nixpkgs/pull/355616 + uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list); + + _processSystemPath = + packageSystemPath: + let + # python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"] + # splittedPath = lib.splitString "." attrName; + splittedPath = lib.splitString "." packageSystemPath; + + # ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"] + packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath; + in + { + # "python312Packages.torch" + name = lib.concatStringsSep "." packagePath; + + # "aarch64-linux" + system = lib.last splittedPath; + }; + + # Turns + # [ + # "hello.aarch64-linux" + # "hello.x86_64-linux" + # "hello.aarch64-darwin" + # "hello.x86_64-darwin" + # "bye.x86_64-darwin" + # "bye.aarch64-darwin" + # ] + # + # into + # + # [ + # "hello" + # "bye" + # ] + extractPackageNames = + packageSystemPaths: + builtins.attrNames ( + builtins.removeAttrs (builtins.groupBy ( + packageSystemPath: (_processSystemPath packageSystemPath).name + ) packageSystemPaths) [ "" ] + ); + + # Computes a diff between two attrs + # { + # added: [ ], + # removed: [ ], + # changed: [ ], + # } + # + diff = + let + filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs); + in + old: new: { + added = filterKeys (n: _: !(old ? ${n})) new; + removed = filterKeys (n: _: !(new ? ${n})) old; + changed = filterKeys ( + n: v: + # Filter out attributes that don't exist anymore + (new ? ${n}) + + # Filter out attributes that are the same as the new value + && (v != (new.${n})) + ) old; + }; + + # Turns + # [ + # "hello.aarch64-linux" + # "hello.x86_64-linux" + # "hello.aarch64-darwin" + # "hello.x86_64-darwin" + # "bye.x86_64-darwin" + # "bye.aarch64-darwin" + # ] + # + # into + # + # { + # linux = [ + # "hello" + # ]; + # darwin = [ + # "hello" + # "bye" + # ]; + # } + groupByKernel = + systemPaths: + let + systemPaths' = builtins.map _processSystemPath systemPaths; + + filterKernel = + kernel: + builtins.attrNames ( + builtins.groupBy (systemPath: systemPath.name) ( + builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths' + ) + ); + in + lib.genAttrs [ "linux" "darwin" ] filterKernel; + + getLabels = lib.mapAttrs ( + kernel: rebuildCount: + let + number = + if rebuildCount == 0 then + "0" + else if rebuildCount <= 10 then + "1-10" + else if rebuildCount <= 500 then + "101-500" + else if rebuildCount <= 1000 then + "501-1000" + else if rebuildCount <= 2500 then + "1001-2500" + else if rebuildCount <= 5000 then + "2501-5000" + else + "5001+"; + + in + "10.rebuild-${kernel}: ${number}" + ); +} diff --git a/ci/eval/default.nix b/ci/eval/default.nix index ef107d4ce517a..e55093be57c52 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -2,6 +2,7 @@ lib, runCommand, writeShellScript, + writeText, linkFarm, time, procps, @@ -246,24 +247,15 @@ let jq -s from_entries > $out/stats.json ''; - compare = - { beforeResultDir, afterResultDir }: - runCommand "compare" - { - nativeBuildInputs = [ - jq - ]; - } - '' - mkdir $out - jq -n -f ${./compare.jq} \ - --slurpfile before ${beforeResultDir}/outpaths.json \ - --slurpfile after ${afterResultDir}/outpaths.json \ - > $out/changed-paths.json - - jq -r -f ${./generate-step-summary.jq} < $out/changed-paths.json > $out/step-summary.md - # TODO: Compare eval stats - ''; + compare = import ./compare { + inherit + lib + jq + runCommand + writeText + supportedSystems + ; + }; full = { From ff9bb621de89c49c579099f5e6197fd9549ac20b Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Tue, 10 Dec 2024 09:58:51 +0100 Subject: [PATCH 09/19] ci/eval: fix compare label assignment (cherry picked from commit 214cb79aa634a47c137ccf8cd243758bb185a129) --- ci/eval/compare/utils.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix index 22353499d6993..2794b5eda4ec3 100644 --- a/ci/eval/compare/utils.nix +++ b/ci/eval/compare/utils.nix @@ -113,6 +113,8 @@ rec { "0" else if rebuildCount <= 10 then "1-10" + else if rebuildCount <= 100 then + "11-100" else if rebuildCount <= 500 then "101-500" else if rebuildCount <= 1000 then From e88b48d259bc5eba62d06ccafc7b6c6041cb10c9 Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Mon, 9 Dec 2024 23:22:12 +0100 Subject: [PATCH 10/19] ci/eval: add rebuildsByPlatform to the comparison result (cherry picked from commit 518ae8fd583e3e2ba9bc7c5b3643b3d7823a77aa) --- ci/eval/compare/default.nix | 69 +++++++++++- ci/eval/compare/utils.nix | 203 +++++++++++++++++++++++++----------- 2 files changed, 207 insertions(+), 65 deletions(-) diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix index 8b5e9059cd5b8..4f3b943a5a135 100644 --- a/ci/eval/compare/default.nix +++ b/ci/eval/compare/default.nix @@ -3,14 +3,66 @@ jq, runCommand, writeText, - supportedSystems, ... }: { beforeResultDir, afterResultDir }: let + /* + Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs. + Note: "platforms" are "x86_64-linux", "aarch64-darwin", ... + + --- + Inputs: + - beforeResultDir, afterResultDir: The evaluation result from before and after the change. + They can be obtained by running `nix-build -A ci.eval.full` on both revisions. + + --- + Outputs: + - changed-paths.json: Various information about the changes: + { + attrdiff: { + added: ["package1"], + changed: ["package2", "package3"], + removed: ["package4"], + }, + labels: [ + "10.rebuild-darwin: 1-10", + "10.rebuild-linux: 1-10" + ], + rebuildsByKernel: { + darwin: ["package1", "package2"], + linux: ["package1", "package2", "package3"] + }, + rebuildCountByKernel: { + darwin: 2, + linux: 3, + }, + rebuildsByPlatform: { + aarch64-darwin: ["package1", "package2"], + aarch64-linux: ["package1", "package2"], + x86_64-linux: ["package1", "package2", "package3"], + x86_64-darwin: ["package1"], + }, + } + - step-summary.md: A markdown render of the changes + + --- + Implementation details: + + Helper functions can be found in ./utils.nix. + Two main "types" are important: + + - `packagePlatformPath`: A string of the form "." + Example: "python312Packages.numpy.x86_64-linux" + + - `packagePlatformAttr`: An attrs representation of a packagePlatformPath: + Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; } + */ inherit (import ./utils.nix { inherit lib; }) diff groupByKernel + convertToPackagePlatformAttrs + groupByPlatform extractPackageNames getLabels uniqueStrings @@ -20,21 +72,30 @@ let beforeAttrs = getAttrs beforeResultDir; afterAttrs = getAttrs afterResultDir; + # Attrs + # - keys: "added", "changed" and "removed" + # - values: lists of `packagePlatformPath`s diffAttrs = diff beforeAttrs afterAttrs; changed-paths = let rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed); + rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds; - rebuildsByKernel = groupByKernel rebuilds; + rebuildsByPlatform = groupByPlatform rebuildsPackagePlatformAttrs; + rebuildsByKernel = groupByKernel rebuildsPackagePlatformAttrs; rebuildCountByKernel = lib.mapAttrs ( kernel: kernelRebuilds: lib.length kernelRebuilds ) rebuildsByKernel; in writeText "changed-paths.json" ( builtins.toJSON { - attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs; - inherit rebuildsByKernel rebuildCountByKernel; + attrdiff = lib.mapAttrs (_: extractPackageNames) diffAttrs; + inherit + rebuildsByPlatform + rebuildsByKernel + rebuildCountByKernel + ; labels = getLabels rebuildCountByKernel; } ); diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix index 2794b5eda4ec3..82ba64e06a173 100644 --- a/ci/eval/compare/utils.nix +++ b/ci/eval/compare/utils.nix @@ -3,55 +3,101 @@ rec { # Borrowed from https://github.com/NixOS/nixpkgs/pull/355616 uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list); - _processSystemPath = - packageSystemPath: + /* + Converts a `packagePlatformPath` into a `packagePlatformAttr` + + Turns + "hello.aarch64-linux" + into + { + name = "hello"; + platform = "aarch64-linux"; + } + */ + convertToPackagePlatformAttr = + packagePlatformPath: let - # python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"] - # splittedPath = lib.splitString "." attrName; - splittedPath = lib.splitString "." packageSystemPath; + # python312Packages.numpy.aarch64-linux -> ["python312Packages" "numpy" "aarch64-linux"] + splittedPath = lib.splitString "." packagePlatformPath; - # ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"] + # ["python312Packages" "numpy" "aarch64-linux"] -> ["python312Packages" "numpy"] packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath; - in - { - # "python312Packages.torch" + + # "python312Packages.numpy" name = lib.concatStringsSep "." packagePath; + in + if name == "" then + null + else + { + # python312Packages.numpy + inherit name; - # "aarch64-linux" - system = lib.last splittedPath; - }; + # "aarch64-linux" + platform = lib.last splittedPath; + }; - # Turns - # [ - # "hello.aarch64-linux" - # "hello.x86_64-linux" - # "hello.aarch64-darwin" - # "hello.x86_64-darwin" - # "bye.x86_64-darwin" - # "bye.aarch64-darwin" - # ] - # - # into - # - # [ - # "hello" - # "bye" - # ] + /* + Converts a list of `packagePlatformPath`s into a list of `packagePlatformAttr`s + + Turns + [ + "hello.aarch64-linux" + "hello.x86_64-linux" + "hello.aarch64-darwin" + "hello.x86_64-darwin" + "bye.x86_64-darwin" + "bye.aarch64-darwin" + "release-checks" <- Will be dropped + ] + into + [ + { name = "hello"; platform = "aarch64-linux"; } + { name = "hello"; platform = "x86_64-linux"; } + { name = "hello"; platform = "aarch64-darwin"; } + { name = "hello"; platform = "x86_64-darwin"; } + { name = "bye"; platform = "aarch64-darwin"; } + { name = "bye"; platform = "x86_64-darwin"; } + ] + */ + convertToPackagePlatformAttrs = + packagePlatformPaths: + builtins.filter (x: x != null) (builtins.map convertToPackagePlatformAttr packagePlatformPaths); + + /* + Converts a list of `packagePlatformPath`s directly to a list of (unique) package names + + Turns + [ + "hello.aarch64-linux" + "hello.x86_64-linux" + "hello.aarch64-darwin" + "hello.x86_64-darwin" + "bye.x86_64-darwin" + "bye.aarch64-darwin" + ] + into + [ + "hello" + "bye" + ] + */ extractPackageNames = - packageSystemPaths: - builtins.attrNames ( - builtins.removeAttrs (builtins.groupBy ( - packageSystemPath: (_processSystemPath packageSystemPath).name - ) packageSystemPaths) [ "" ] - ); - - # Computes a diff between two attrs - # { - # added: [ ], - # removed: [ ], - # changed: [ ], - # } - # + packagePlatformPaths: + let + packagePlatformAttrs = convertToPackagePlatformAttrs (uniqueStrings packagePlatformPaths); + in + uniqueStrings (builtins.map (p: p.name) packagePlatformAttrs); + + /* + Computes the key difference between two attrs + + { + added: [ ], + removed: [ ], + changed: [ ], + } + */ diff = let filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs); @@ -69,43 +115,78 @@ rec { ) old; }; + /* + Group a list of `packagePlatformAttr`s by platforms + + Turns + [ + { name = "hello"; platform = "aarch64-linux"; } + { name = "hello"; platform = "x86_64-linux"; } + { name = "hello"; platform = "aarch64-darwin"; } + { name = "hello"; platform = "x86_64-darwin"; } + { name = "bye"; platform = "aarch64-darwin"; } + { name = "bye"; platform = "x86_64-darwin"; } + ] + into + { + aarch64-linux = [ "hello" ]; + x86_64-linux = [ "hello" ]; + aarch64-darwin = [ "hello" "bye" ]; + x86_64-darwin = [ "hello" "bye" ]; + } + */ + groupByPlatform = + packagePlatformAttrs: + let + packagePlatformAttrsByPlatform = builtins.groupBy (p: p.platform) packagePlatformAttrs; + extractPackageNames = map (p: p.name); + in + lib.mapAttrs (_: extractPackageNames) packagePlatformAttrsByPlatform; + # Turns # [ - # "hello.aarch64-linux" - # "hello.x86_64-linux" - # "hello.aarch64-darwin" - # "hello.x86_64-darwin" - # "bye.x86_64-darwin" - # "bye.aarch64-darwin" + # { name = "hello"; platform = "aarch64-linux"; } + # { name = "hello"; platform = "x86_64-linux"; } + # { name = "hello"; platform = "aarch64-darwin"; } + # { name = "hello"; platform = "x86_64-darwin"; } + # { name = "bye"; platform = "aarch64-darwin"; } + # { name = "bye"; platform = "x86_64-darwin"; } # ] # # into # # { - # linux = [ - # "hello" - # ]; - # darwin = [ - # "hello" - # "bye" - # ]; + # linux = [ "hello" ]; + # darwin = [ "hello" "bye" ]; # } groupByKernel = - systemPaths: + packagePlatformAttrs: let - systemPaths' = builtins.map _processSystemPath systemPaths; - filterKernel = kernel: builtins.attrNames ( - builtins.groupBy (systemPath: systemPath.name) ( - builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths' + builtins.groupBy (p: p.name) ( + builtins.filter (p: lib.hasSuffix kernel p.platform) packagePlatformAttrs ) ); in lib.genAttrs [ "linux" "darwin" ] filterKernel; - getLabels = lib.mapAttrs ( + /* + Maps an attrs of `kernel - rebuild counts` mappings to a list of labels + + Turns + { + linux = 56; + darwin = 8; + } + into + [ + "10.rebuild-darwin: 1-10" + "10.rebuild-linux: 11-100" + ] + */ + getLabels = lib.mapAttrsToList ( kernel: rebuildCount: let number = From c1553b513e6398c5f7c45f4c924ce15f731a50ef Mon Sep 17 00:00:00 2001 From: Gaetan Lepage Date: Sat, 14 Dec 2024 21:32:34 +0100 Subject: [PATCH 11/19] ci/eval: allow precisely choosing which systems to evaluate for (evalSystem -> evalSystems) (cherry picked from commit 6eadbf9c9788cd842066ff9c66dbfd1dfa00b25c) --- ci/eval/README.md | 8 +++++--- ci/eval/default.nix | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/eval/README.md b/ci/eval/README.md index 020e7127f51a9..0436a028ed69e 100644 --- a/ci/eval/README.md +++ b/ci/eval/README.md @@ -5,14 +5,16 @@ The code in this directory is used by the [eval.yml](../../.github/workflows/eva Furthermore it also allows local evaluation using ``` nix-build ci -A eval.full \ - --max-jobs 4 - --cores 2 - --arg chunkSize 10000 + --max-jobs 4 \ + --cores 2 \ + --arg chunkSize 10000 \ + --arg evalSystems '["x86_64-linux" "aarch64-darwin"]' ``` - `--max-jobs`: The maximum number of derivations to run at the same time. Only each [supported system](../supportedSystems.nix) gets a separate derivation, so it doesn't make sense to set this higher than that number. - `--cores`: The number of cores to use for each job. Recommended to set this to the amount of cores on your system divided by `--max-jobs`. - `chunkSize`: The number of attributes that are evaluated simultaneously on a single core. Lowering this decreases memory usage at the cost of increased evaluation time. If this is too high, there won't be enough chunks to process them in parallel, and will also increase evaluation time. +- `evalSystems`: The set of systems for which `nixpkgs` should be evaluated. Defaults to the four official platforms (`x86_64-linux`, `aarch64-linux`, `x86_64-darwin` and `aarch64-darwin`). A good default is to set `chunkSize` to 10000, which leads to about 3.6GB max memory usage per core, so suitable for fully utilising machines with 4 cores and 16GB memory, 8 cores and 32GB memory or 16 cores and 64GB memory. diff --git a/ci/eval/default.nix b/ci/eval/default.nix index e55093be57c52..77430976ffc53 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -259,14 +259,13 @@ let full = { - # Whether to evaluate just a single system, by default all are evaluated - evalSystem ? if quickTest then "x86_64-linux" else null, + # Whether to evaluate on a specific set of systems, by default all are evaluated + evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems, # The number of attributes per chunk, see ./README.md for more info. chunkSize, quickTest ? false, }: let - systems = if evalSystem == null then supportedSystems else [ evalSystem ]; results = linkFarm "results" ( map (evalSystem: { name = evalSystem; @@ -274,7 +273,7 @@ let inherit quickTest evalSystem chunkSize; attrpathFile = attrpathsSuperset + "/paths.json"; }; - }) systems + }) evalSystems ); in combine { From 300a4854ca70bc2ca346ea3d82495a80dca1b973 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 11 Dec 2024 19:32:07 +0100 Subject: [PATCH 12/19] workflows/eval: Improve debuggabilitiy Sometimes successful workflows trigger that condition, see https://github.com/NixOS/nixpkgs/actions/runs/12278493443/job/34260660227: Comparing against "https://github.com/NixOS/nixpkgs/actions/runs/12278495895" Workflow not done, waiting 10 seconds before checking again Workflow was not successful, cannot make comparison Even though $ gh api /repos/NixOS/nixpkgs/actions/runs/12278495895 --jq .conclusion success Let's print the conclusion when it's not "success" to debug this. (cherry picked from commit aae821d4fe992fd29ff30a8c8f6589a90e1d272e) --- .github/workflows/eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 0e857e185e379..bac9394500ac6 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -172,7 +172,7 @@ jobs: done if [[ "$conclusion" != "success" ]]; then - echo "Workflow was not successful, cannot make comparison" + echo "Workflow was not successful (conclusion: $conclusion), cannot make comparison" exit 0 fi From 7f0d2b1b45bf72a8c9324e99b1d22941b8d309ab Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 12 Dec 2024 12:53:37 +0100 Subject: [PATCH 13/19] ci/eval: Avoid noise for failing attribute evals It's currently annoying to see the actual failure in the attrs step, because `time -v` displays like 20 lines, which get repeated, therefore requiring you to scroll up most of the time: https://github.com/NixOS/nixpkgs/actions/runs/12290298121/job/34297218345#step:5:794 This commit fixes that by only displaying the most important stats, the same ones as the chunked system-specific evals. (cherry picked from commit bd5c93ca3d321b5a0dffcede18ec8da72c8d56d5) --- ci/eval/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 77430976ffc53..927dec5a908dc 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -49,7 +49,7 @@ let export NIX_STATE_DIR=$(mktemp -d) mkdir $out export GC_INITIAL_HEAP_SIZE=4g - command time -v \ + command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \ nix-instantiate --eval --strict --json --show-trace \ "$src/pkgs/top-level/release-attrpaths-superset.nix" \ -A paths \ From f775bccc188863af979e653950919d509336de7b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 18 Dec 2024 03:58:43 +0100 Subject: [PATCH 14/19] workflows/eval: Catch empty conclusion Sometimes the conclusion is empty when it's still running/pending or so, which needs to be caught, otherwise it can exit preemptively: https://github.com/NixOS/nixpkgs/pull/364308#issuecomment-2550179941 Co-Authored-By: Reno Dakota (cherry picked from commit 79dca9a646194f2413a572f11224fcdf201eed17) --- .github/workflows/eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index bac9394500ac6..97130b8be323f 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -165,7 +165,7 @@ jobs: runId=$(jq .id <<< "$run") conclusion=$(jq -r .conclusion <<< "$run") - while [[ "$conclusion" == null ]]; do + while [[ "$conclusion" == null || "$conclusion" == "" ]]; do echo "Workflow not done, waiting 10 seconds before checking again" sleep 10 conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion') From bb3bf66877abb73fc6e7e32ce2a9097210e6409e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 12 Dec 2024 13:23:06 +0100 Subject: [PATCH 15/19] ci/eval/compare: truncate step summary to 1024k (cherry picked from commit 44dc31e67631169c80b1d82afa27fab8010d6aca) --- ci/eval/compare/generate-step-summary.jq | 27 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/ci/eval/compare/generate-step-summary.jq b/ci/eval/compare/generate-step-summary.jq index 28597eaec371f..961877cd27974 100644 --- a/ci/eval/compare/generate-step-summary.jq +++ b/ci/eval/compare/generate-step-summary.jq @@ -4,12 +4,27 @@ def truncate(xs; n): end; def itemize_packages(xs): - # we truncate the list to stay below the GitHub limit of 1MB per step summary. - truncate(xs; 3000) | map("- [\(.)](https://search.nixos.org/packages?channel=unstable&show=\(.)&from=0&size=50&sort=relevance&type=packages&query=\(.))") | join("\n"); + truncate(xs; 2000) | + map("- [\(.)](https://search.nixos.org/packages?channel=unstable&show=\(.)&from=0&size=50&sort=relevance&type=packages&query=\(.))") | + join("\n"); + +def get_title(s; xs): + s + " (" + (xs | length | tostring) + ")"; def section(title; xs): - "
" + title + " (" + (xs | length | tostring) + ")\n\n" + itemize_packages(xs) + "
"; + "
" + get_title(title; xs) + "\n\n" + itemize_packages(xs) + "
"; + +def fallback_document(content; n): + if content | utf8bytelength > n then + get_title("Added packages"; .attrdiff.added) + "\n\n" + + get_title("Removed packages"; .attrdiff.removed) + "\n\n" + + get_title("Changed packages"; .attrdiff.changed) + else content + end; -section("Added packages"; .attrdiff.added) + "\n\n" + -section("Removed packages"; .attrdiff.removed) + "\n\n" + -section("Changed packages"; .attrdiff.changed) +# we truncate the list to stay below the GitHub limit of 1MB per step summary. +fallback_document( + section("Added packages"; .attrdiff.added) + "\n\n" + + section("Removed packages"; .attrdiff.removed) + "\n\n" + + section("Changed packages"; .attrdiff.changed); 1000 * 1000 +) From bf9c1332ee20c5fd5cf3d25aa84f5d6fd738069d Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 21 Dec 2024 14:05:30 +0100 Subject: [PATCH 16/19] workflows/eval: evaluate all systems to completion on failure Failing fast hides system-specific evaluation failures, because all of the currently 4 jobs appear as failed. (cherry picked from commit a9dbb26f48d0075ffb3b4aa776c7e9728c42ff55) --- .github/workflows/eval.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 97130b8be323f..34f4d07d47cf9 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -81,6 +81,7 @@ jobs: runs-on: ubuntu-latest needs: [ attrs, get-merge-commit ] strategy: + fail-fast: false matrix: system: ${{ fromJSON(needs.attrs.outputs.systems) }} steps: From fc3d9f32e24ec17ac59b7b8c95c29b40dcec4737 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 11:04:57 +0000 Subject: [PATCH 17/19] build(deps): bump actions/create-github-app-token from 1.11.0 to 1.11.1 Bumps [actions/create-github-app-token](https://github.com/actions/create-github-app-token) from 1.11.0 to 1.11.1. - [Release notes](https://github.com/actions/create-github-app-token/releases) - [Commits](https://github.com/actions/create-github-app-token/compare/5d869da34e18e7287c1daad50e0b8ea0f506ce69...c1a285145b9d317df6ced56c09f525b5c2b6f755) --- updated-dependencies: - dependency-name: actions/create-github-app-token dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] (cherry picked from commit 4c4e9337fa950cb18002df187ad25d907d3894c0) --- .github/workflows/backport.yml | 2 +- .github/workflows/codeowners-v2.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 159175ceb6346..1e5a1a229d548 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -18,7 +18,7 @@ jobs: steps: # Use a GitHub App to create the PR so that CI gets triggered # The App is scoped to Repository > Contents and Pull Requests: write for Nixpkgs - - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 + - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 id: app-token with: app-id: ${{ vars.BACKPORT_APP_ID }} diff --git a/.github/workflows/codeowners-v2.yml b/.github/workflows/codeowners-v2.yml index 5cfeafa8489e2..6329e1d9ea110 100644 --- a/.github/workflows/codeowners-v2.yml +++ b/.github/workflows/codeowners-v2.yml @@ -62,7 +62,7 @@ jobs: - name: Build codeowners validator run: nix-build base/ci -A codeownersValidator - - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 + - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 id: app-token with: app-id: ${{ vars.OWNER_RO_APP_ID }} @@ -94,7 +94,7 @@ jobs: # This is intentional, because we need to request the review of owners as declared in the base branch. - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/create-github-app-token@5d869da34e18e7287c1daad50e0b8ea0f506ce69 # v1.11.0 + - uses: actions/create-github-app-token@c1a285145b9d317df6ced56c09f525b5c2b6f755 # v1.11.1 id: app-token with: app-id: ${{ vars.OWNER_APP_ID }} From fddbd3e9800c214f83674791a883716c168a8539 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 11:05:01 +0000 Subject: [PATCH 18/19] build(deps): bump actions/upload-artifact from 4.4.3 to 4.5.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.4.3 to 4.5.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882...6f51ac03b9356f520e9adb1b1b7802705f340c2b) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] (cherry picked from commit abd256e12b2ed5ea4ae515806dd0fceda0dee8d5) --- .github/workflows/eval.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 34f4d07d47cf9..06cff2b878d3e 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -53,7 +53,7 @@ jobs: echo "systems=$(> "$GITHUB_OUTPUT" - name: Upload the list of all attributes - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: paths path: result/* @@ -111,7 +111,7 @@ jobs: # If it uses too much memory, slightly decrease chunkSize - name: Upload the output paths and eval stats - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: intermediate-${{ matrix.system }} path: result/* @@ -145,7 +145,7 @@ jobs: -o prResult - name: Upload the combined results - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: result path: prResult/* @@ -203,7 +203,7 @@ jobs: - name: Upload the combined results if: steps.baseRunId.outputs.baseRunId - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 with: name: comparison path: comparison/* From db5a0ad7746e5d07aafa4a262ff1bf49858a4a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Mon, 30 Dec 2024 20:01:31 +0100 Subject: [PATCH 19/19] ci: Label 10.rebuild-*-stdenv (#369102) Currently ofborg does this, but there is actually no real reason this shouldn't be done by CI (cherry picked from commit 6d96c9a21e56eefbf7d6e9890fa2905daad5c1be) --- ci/eval/compare/default.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix index 4f3b943a5a135..cfcdd3c1a217c 100644 --- a/ci/eval/compare/default.nix +++ b/ci/eval/compare/default.nix @@ -96,7 +96,12 @@ let rebuildsByKernel rebuildCountByKernel ; - labels = getLabels rebuildCountByKernel; + labels = + (getLabels rebuildCountByKernel) + # Adds "10.rebuild-*-stdenv" label if the "stdenv" attribute was changed + ++ lib.mapAttrsToList (kernel: _: "10.rebuild-${kernel}-stdenv") ( + lib.filterAttrs (_: kernelRebuilds: kernelRebuilds ? "stdenv") rebuildsByKernel + ); } ); in