ci: Avoid invoking cargo on non-rust changes (#121)

Currently the `justfile` invokes `cargo` to determine the validator
version. This means that it will install the Rust version in
`rust-toolchain` when the runner OS includes another Rust version.
This is unnecessary in the proxy-init release workflow.

This change removes the `action-prune-docker` just recipe to a
dedicated script in `.github/docker-prune.sh` so that it can be run
without any of the `justfile` dependencies. This also makes the script
covered by `just sh-lint`.

This change also moves validator packaging into a dedicated
`validator/.justfile` so that invoking the top-level justfile does not
eagerly load defaults with `cargo`.

Signed-off-by: Oliver Gould <[email protected]>

Author: olix0r

.github/docker-prune.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Delete all files under the buildkit blob directory that are not referred
+# to any longer in the cache manifest file
+manifest_sha=$(jq -r .manifests[0].digest < "$RUNNER_TEMP/.buildx-cache/index.json")
+manifest=${manifest_sha#"sha256:"}
+files=("$manifest")
+while IFS= read -r f; do
+    files+=("$f")
+done < <(jq -r '.manifests[].digest | sub("^sha256:"; "")' < "$RUNNER_TEMP/.buildx-cache/blobs/sha256/$manifest")
+
+for file in "$RUNNER_TEMP"/.buildx-cache/blobs/sha256/*; do
+    for name in "${files[@]}"; do
+        if [[ "${file##*/}" == "$name" ]]; then
+            rm -f "$file"
+            echo "deleted: $name"
+            break;
+        fi
+    done
+done

.github/workflows/release-proxy-init.yml

@@ -44,7 +44,6 @@ jobs:
     permissions:
       id-token: write # needed for signing the images with GitHub OIDC token
     steps:
-      - uses: extractions/setup-just@95b912dc5d3ed106a72907f2f9b91e76d60bdb76
       - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18
       - uses: docker/setup-buildx-action@95cb08cb2672c73d4ffd2f422e6d11953d2a9c70
 
@@ -61,7 +60,7 @@ jobs:
               --cache-from type=local,src="$RUNNER_TEMP/.buildx-cache" \
               --cache-to type=local,dest="$RUNNER_TEMP/.buildx-cache",mode=max \
               --platform="linux/amd64,linux/arm64,linux/arm/v7"
-      - run: just action-prune-docker
+      - run: .github/docker-prune.sh
 
       # Only publish images on release
       - if: needs.meta.outputs.mode == 'release'

.github/workflows/release-validator.yml

@@ -14,8 +14,9 @@ jobs:
   meta:
     timeout-minutes: 3
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v30-tools
+    container: ghcr.io/linkerd/dev:v30-rust
     steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - id: meta
         shell: bash
         run: |
@@ -28,18 +29,15 @@ jobs:
             ) >> "$GITHUB_OUTPUT"
           else
             sha="${{ github.sha }}"
-            ( echo version="test-${sha:0:7}"
+            ( echo version="v$(just validator --evaluate version)-${sha:0:7}"
               echo mode=test
             ) >> "$GITHUB_OUTPUT"
           fi
-      - if: steps.meta.outputs.mode == 'release'
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - if: steps.meta.outputs.mode == 'release'
         name: Check that validator version matches release version
         shell: bash
         run: |
-          version=$(cargo metadata --format-version=1 \
-              | jq -r '.packages[] | select(.name == "linkerd-network-validator") | .version')
+          version=$(just validator --evaluate version)
           # shellcheck disable=SC2193
           if [[ "v${version}" != '${{ steps.meta.outputs.version }}' ]]; then
             echo "::error ::Crate version v${version} does not match tag ${{ steps.meta.outputs.version }}"
@@ -78,7 +76,7 @@ jobs:
           ) >> "$GITHUB_ENV"
 
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
-      - run: just validator-package
+      - run: just validator package
         env:
           CARGO_RELEASE: "1"
           VALIDATOR_VERSION: ${{ needs.meta.outputs.version }}

justfile

@@ -16,8 +16,6 @@ default: lint test
 
 lint: sh-lint md-lint rs-clippy proxy-init-lint action-lint action-dev-check
 
-build: proxy-init-build validator-build
-
 test: rs-test proxy-init-test-unit proxy-init-test-integration
 
 # Check whether the Go code is formatted.
@@ -36,39 +34,6 @@ rs-toolchain := ""
 
 export RUST_BACKTRACE := env_var_or_default("RUST_BACKTRACE", "short")
 
-# The version name to use for packages.
-_validator-version := env_var_or_default("VALIDATOR_VERSION", ```
-    cargo metadata --format-version=1 \
-        | jq -r '.packages[] | select(.name == "linkerd-network-validator") | .version' \
-        | head -n 1
-    ```)
-
-# The architecture name to use for packages. Either 'amd64', 'arm64', or 'arm'.
-_arch := env_var_or_default("ARCH", "amd64")
-
-# If a `package_arch` is specified, then we change the default cargo `--target`
-# to support cross-compilation. Otherwise, we use `rustup` to find the default.
-_cargo-target := if _arch == "amd64" {
-        "x86_64-unknown-linux-musl"
-    } else if _arch == "arm64" {
-        "aarch64-unknown-linux-musl"
-    } else if _arch == "arm" {
-        "armv7-unknown-linux-musleabihf"
-    } else {
-        `rustup show | sed -n 's/^Default host: \(.*\)/\1/p'`
-    }
-
-# Support cross-compilation when `_arch` changes.
-export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER := "aarch64-linux-gnu-gcc"
-export CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER := "arm-linux-gnueabihf-gcc"
-_strip := if _arch == "arm64" { "aarch64-linux-gnu-strip" } else if _arch == "arm" { "arm-linux-gnueabihf-strip" } else { "strip" }
-
-_target-dir := "target" / _cargo-target / rs-build-type
-_validator-bin := _target-dir / "linkerd-network-validator"
-_validator-package-name := "linkerd-network-validator-" + _validator-version + "-" + _arch
-_validator-package-dir := "target/package" / _validator-package-name
-_shasum := "shasum -a 256"
-
 _cargo := env_var_or_default("CARGO", "cargo") + if rs-toolchain != "" { " +" + rs-toolchain } else { "" }
 
 # Fetch Rust dependencies
@@ -81,29 +46,26 @@ rs-fmt-check:
 
 # Lint Rust code
 rs-clippy:
-    {{ _cargo }} clippy --frozen --workspace --all-targets --no-deps {{ _cargo-fmt }}
+    {{ _cargo }} clippy {{ _cargo-build-flags }} --all-targets --no-deps {{ _cargo-fmt }}
 
 # Audit Rust dependencies
 rs-audit-deps:
     {{ _cargo }} deny check
 
 # Build Rust unit and integration tests
 rs-test-build:
-    {{ _cargo-test }} --no-run --frozen --workspace {{ _cargo-fmt }}
+    {{ _cargo-test }} {{ _cargo-build-flags }} --workspace --no-run {{ _cargo-fmt }}
 
 # Run unit tests in whole Rust workspace
 rs-test *flags:
-    {{ _cargo-test }} --frozen --workspace \
-        {{ if rs-build-type == "release" { "--release" } else { "" } }} \
-        {{ flags }}
+    {{ _cargo-test }} {{ _cargo-build-flags }} --workspace {{ flags }}
 
 # Check a specific Rust crate
 rs-check-dir dir *flags:
     cd {{ dir }} \
-        && {{ _cargo }} check --frozen \
-        {{ if rs-build-type == "release" { "--release" } else { "" } }} \
-        {{ flags }} \
-        {{ _cargo-fmt }}
+        && {{ _cargo }} check {{ _cargo-build-flags }} {{ flags }} {{ _cargo-fmt }}
+
+_cargo-build-flags := "--frozen" + if rs-build-type == "release" { " --release" } else { "" }
 
 # If recipe is run in github actions (and cargo-action-fmt is installed), then add a
 # command suffix that formats errors
@@ -115,10 +77,10 @@ _cargo-fmt := if env_var_or_default("GITHUB_ACTIONS", "") != "true" { "" } else
     ```
 }
 
-# When available, use cargo-nextest to run Rust tests; if the binary is not available,
-# use default test runner
+# Use cargo-nextest to run Rust tests if available locally. We use the default
+# runner in CI.
 _cargo-test := _cargo + ```
-    if command -v cargo-nextest >/dev/null 2>&1 ; then
+    if [ "${GITHUB_ACTIONS:-}" != "true" ] && command -v cargo-nextest >/dev/null 2>&1 ; then
         echo " nextest run"
     else
         echo " test"
@@ -129,18 +91,8 @@ _cargo-test := _cargo + ```
 ## validator
 ##
 
-# Build validator code
-validator-build *flags:
-    {{ _cargo }} build --workspace -p linkerd-network-validator \
-        --target={{ _cargo-target }} \
-        {{ if rs-build-type == "release" { "--release" } else { "" } }} \
-        {{ flags }}
-
-validator-package: rs-fetch validator-build
-    @-mkdir -p target/package
-    cp {{ _validator-bin }} target/package/{{ _validator-package-name }}
-    {{ _strip }} target/package/{{ _validator-package-name }}
-    {{ _shasum }} target/package/{{ _validator-package-name }} >target/package/{{ _validator-package-name }}.shasum
+validator *args:
+    {{ just_executable() }} --justfile=validator/.justfile {{ args }}
 
 ##
 ## proxy-init
@@ -172,7 +124,7 @@ proxy-init-image:
     docker buildx build . \
         --tag={{ _image }} \
         --platform={{ docker-arch }} \
-        --load \
+        --load
 
 # Build docker image for iptables-tester (Development)
 proxy-init-test-image:
@@ -287,25 +239,3 @@ sh-lint:
     done < <(find . -type f ! \( -path ./.git/\* -or -path \*/target/\* \)) | xargs)
     echo "shellcheck $files" >&2
     shellcheck $files
-
-# Prune Docker BuildKit cache (in CI)
-action-prune-docker:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-    # Delete all files under the buildkit blob directory that are not referred
-    # to any longer in the cache manifest file
-    manifest_sha=$(jq -r .manifests[0].digest < "$RUNNER_TEMP/.buildx-cache/index.json")
-    manifest=${manifest_sha#"sha256:"}
-    files=("$manifest")
-    while IFS= read -r f; do
-        files+=("$f")
-    done < <(jq -r '.manifests[].digest | sub("^sha256:"; "")' < "$RUNNER_TEMP/.buildx-cache/blobs/sha256/$manifest")
-    for file in "$RUNNER_TEMP"/.buildx-cache/blobs/sha256/*; do
-        for name in "${files[@]}"; do
-            if [[ "${file##*/}" == "$name" ]]; then
-                rm -f "$file"
-                echo "deleted: $name"
-                break;
-            fi
-        done
-    done

validator/.justfile

@@ -0,0 +1,65 @@
+# This justfile includes recipes for building and packaging the validator for
+# release. This file is separated so that we can invoke cargo, etc when building
+# defaults. If this logic were in the top-level justfile, then these tools would
+# be invoked (possibly updating Rust, etc), for unrelated targets.
+#
+# Users are expected to interact with this via the top-level Justfile.
+
+# The version name to use for packages.
+version := env_var_or_default("VALIDATOR_VERSION", ```
+    cd .. && cargo metadata --format-version=1 \
+        | jq -r '.packages[] | select(.name == "linkerd-network-validator") | .version' \
+        | head -n 1
+    ```)
+
+# The architecture name to use for packages. Either 'amd64', 'arm64', or 'arm'.
+_arch := env_var_or_default("ARCH", "amd64")
+
+# If an `_arch` is specified, then we change the default cargo `--target` to
+# support cross-compilation. Otherwise, we use `rustup` to find the default.
+_cargo-target := if _arch == "amd64" {
+        "x86_64-unknown-linux-musl"
+    } else if _arch == "arm64" {
+        "aarch64-unknown-linux-musl"
+    } else if _arch == "arm" {
+        "armv7-unknown-linux-musleabihf"
+    } else {
+        `rustup show | sed -n 's/^Default host: \(.*\)/\1/p'`
+    }
+
+_target-dir := "../target" / _cargo-target / "release"
+_bin := _target-dir / "linkerd-network-validator"
+_package-name := "linkerd-network-validator-" + version + "-" + _arch
+_package-dir := "../target/package"
+_shasum := "shasum -a 256"
+
+export RUST_BACKTRACE := env_var_or_default("RUST_BACKTRACE", "short")
+
+# Support cross-compilation when `_arch` changes.
+_strip := if _arch == "arm64" { "aarch64-linux-gnu-strip" } else if _arch == "arm" { "arm-linux-gnueabihf-strip" } else { "strip" }
+
+_cargo := env_var_or_default("CARGO", "cargo")
+
+# If recipe is run in github actions (and cargo-action-fmt is installed), then add a
+# command suffix that formats errors
+_cargo-fmt := if env_var_or_default("GITHUB_ACTIONS", "") != "true" { "" } else {
+    ```
+    if command -v cargo-action-fmt >/dev/null 2>&1 ; then
+        echo "--message-format=json | cargo-action-fmt"
+    fi
+    ```
+}
+
+package: build
+    @-mkdir -p {{ _package-dir }}
+    cp {{ _bin }} {{ _package-dir / _package-name }}
+    {{ _strip }} {{ _package-dir / _package-name }}
+    {{ _shasum }} {{ _package-dir / _package-name }} > {{ _package-dir / _package-name }}.shasum
+
+build *flags:
+    {{ _cargo }} fetch --locked
+    {{ _cargo }} build --workspace -p linkerd-network-validator \
+        --release \
+        --target={{ _cargo-target }} \
+        {{ flags }} \
+        {{ _cargo-fmt }}