From 53540f1a9bfae893fabf0ce0115d9ad842bb1779 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:45:56 +0100 Subject: [PATCH 01/27] Check that the checkout contains the agent work in the bare repo --- lib/preserve.sh | 148 +++++++++++++++++++++++++++ tests/test_preserve.sh | 224 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 372 insertions(+) create mode 100644 lib/preserve.sh create mode 100755 tests/test_preserve.sh diff --git a/lib/preserve.sh b/lib/preserve.sh new file mode 100644 index 0000000..93848d6 --- /dev/null +++ b/lib/preserve.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# Preservation checks: before anything deletes engine state, every +# piece of agent work in it has to be in the checkout. Sourced by +# launch.sh after lib/project.sh. + +# Refs of the bare repo that hold agent work. refs/swarm-runs/ is kept +# for per-run namespaces. +SWARM_PRESERVATION_REFS=( + refs/heads/agent-work + refs/heads/swarm/ + refs/heads/agent-parked/ + refs/swarm-runs/ +) + +# Git for the preservation checks. Replacement objects and grafts can +# give a commit parents it does not have, which would make work look +# contained, so both are turned off. Git warns about the graft file +# setting itself; that advice is silenced, because the checks treat +# any message on stderr as a failure. +swarm_preserve_git() { + GIT_NO_REPLACE_OBJECTS=1 GIT_GRAFT_FILE=/dev/null \ + git -c advice.graftFileDeprecated=false "$@" +} + +# Succeed when the checkout at ROOT is a repository with a commit at +# HEAD; say why on stderr otherwise. Callers run this before +# swarm_tip_contained, so that a failure there means a missing commit. +swarm_checkout_readable() { + local root="$1" + if ! swarm_preserve_git -C "$root" rev-parse --verify --quiet \ + 'HEAD^{commit}' >/dev/null 2>&1; then + echo "ERROR: cannot read the commit at HEAD in ${root}." >&2 + return 1 + fi +} + +# Succeed when commit SHA is in the history of the checkout at ROOT +# (its HEAD). Return 1 when it is not, including when the checkout does +# not have the object at all, and 2 when git cannot tell. The checkout +# must have passed swarm_checkout_readable. +swarm_tip_contained() { + local root="$1" sha="$2" errf rc=0 err type + errf=$(mktemp) || return 2 + # Git can report a broken object store on stderr and still exit 0, + # so every step checks stderr as well as the status. cat-file -e + # exits 1 without a word for a missing object. + swarm_preserve_git -C "$root" cat-file -e "$sha" 2>"$errf" || rc=$? + if [ "$rc" -eq 1 ] && [ ! -s "$errf" ]; then + rm -f "$errf" + return 1 + fi + if [ "$rc" -eq 0 ] && [ ! -s "$errf" ]; then + type=$(swarm_preserve_git -C "$root" cat-file -t "$sha" \ + 2>"$errf") || rc=$? + fi + if [ "$rc" -eq 0 ] && [ ! -s "$errf" ] && [ "$type" = commit ]; then + swarm_preserve_git -C "$root" merge-base --is-ancestor "$sha" \ + HEAD 2>"$errf" || rc=$? + if [ "$rc" -eq 1 ] && [ ! -s "$errf" ]; then + rm -f "$errf" + return 1 + fi + fi + err=$(cat "$errf") + rm -f "$errf" + if [ "$rc" -ne 0 ] || [ -n "$err" ]; then + echo "ERROR: cannot read ${sha} in ${root}: ${err}" >&2 + return 2 + fi + # Not a commit: nothing in HEAD's history. + [ "$type" = commit ] || return 1 + return 0 +} + +# Check the preservation refs of bare repo BARE against the checkout at +# ROOT. Return 0 when the checkout contains every tip, 1 when some are +# missing (printed as " " lines), and 2 when the check could +# not be made (said on stderr). Callers branch on the status. +swarm_preservation_check() { + local bare="$1" root="$2" tmp err sha type ref status rc=0 + if [ -L "$bare" ] || [ ! -d "$bare" ] \ + || [ "$(swarm_preserve_git -C "$bare" rev-parse \ + --is-bare-repository 2>/dev/null)" != true ]; then + echo "ERROR: ${bare} is not a bare repository." >&2 + return 2 + fi + swarm_checkout_readable "$root" || return 2 + tmp=$(mktemp) || return 2 + err=$(mktemp) || { rm -f "$tmp"; return 2; } + # for-each-ref skips a broken ref with only a warning and still + # succeeds, so anything on stderr counts as a failure too. + if ! swarm_preserve_git -C "$bare" for-each-ref \ + --format='%(objectname) %(objecttype) %(refname)' \ + "${SWARM_PRESERVATION_REFS[@]}" > "$tmp" 2> "$err" \ + || [ -s "$err" ]; then + echo "ERROR: cannot list the refs of ${bare}:" >&2 + sed 's/^/ /' "$err" >&2 + rm -f "$tmp" "$err" + return 2 + fi + rm -f "$err" + while read -r sha type ref; do + if [ "$type" != commit ]; then + echo "ERROR: ${ref} in ${bare} does not point to a commit." >&2 + rc=2 + break + fi + status=0 + swarm_tip_contained "$root" "$sha" || status=$? + case "$status" in + 0) ;; + 1) + printf '%s %s\n' "$ref" "$sha" + rc=1 ;; + *) + echo "ERROR: cannot compare ${ref} with ${root}." >&2 + rc=2 + break ;; + esac + done < "$tmp" + rm -f "$tmp" + return "$rc" +} + +# Explain the " " lines of swarm_preservation_check (on +# stdin) for bare repo BARE, on stderr. +swarm_preservation_explain() { + local bare="$1" ref sha quoted qref + quoted=$(printf '%q' "$bare") + echo "The checkout does not contain this agent work yet:" >&2 + while read -r ref sha; do + [ -n "$ref" ] || continue + qref=$(printf '%q' "$ref") + case "$ref" in + refs/heads/agent-work|refs/heads/swarm/*/interactive-*) + echo " ${ref#refs/heads/} ${sha:0:12}: run ./harvest.sh" \ + "to merge it" >&2 ;; + *) + echo " ${ref#refs/heads/} ${sha:0:12}: look at it with" \ + "git fetch ${quoted} ${qref}, then merge or" \ + "cherry-pick what you keep" >&2 ;; + esac + done + echo "A ref you have decided to drop can be deleted from the bare" \ + "repo with:" >&2 + echo " git --git-dir=${quoted} update-ref -d " >&2 +} diff --git a/tests/test_preserve.sh b/tests/test_preserve.sh new file mode 100755 index 0000000..955a324 --- /dev/null +++ b/tests/test_preserve.sh @@ -0,0 +1,224 @@ +#!/bin/bash +set -euo pipefail + +# shellcheck source=_test_env.sh +source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" + +# Unit tests for lib/preserve.sh: the checks that keep agent work from +# being deleted. Uses local git repos (no Docker or API key required). + +PASS=0 +FAIL=0 +TMPDIR=$(mktemp -d) +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +trap 'rm -rf "$TMPDIR"' EXIT + +# shellcheck source=../lib/preserve.sh +source "$REPO_ROOT/lib/preserve.sh" + +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + echo " PASS: ${label}" + PASS=$((PASS + 1)) + else + echo " FAIL: ${label}" + echo " expected: ${expected}" + echo " actual: ${actual}" + FAIL=$((FAIL + 1)) + fi +} + +g() { + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false "$@" +} + +# A checkout with two commits, and a bare repo cloned from it. +WORK="$TMPDIR/work" +BARE="$TMPDIR/bare.git" +AGENT="$TMPDIR/agent" +git init -q "$WORK" +g -C "$WORK" commit -q --allow-empty -m one +ONE=$(git -C "$WORK" rev-parse HEAD) +g -C "$WORK" commit -q --allow-empty -m two +git clone -q --bare "$WORK" "$BARE" +git -C "$BARE" update-ref refs/heads/agent-work HEAD +git clone -q "$BARE" "$AGENT" + +# Run the check and print "|". +check() { + local out rc=0 + out=$(swarm_preservation_check "$BARE" "$WORK" 2>/dev/null) || rc=$? + printf '%s|%s' "$rc" "$out" +} + +# Push a new agent commit from the agent clone to REF and print it. +agent_push() { + local ref="$1" + git -C "$AGENT" fetch -q origin + git -C "$AGENT" checkout -q --detach origin/agent-work + g -C "$AGENT" commit -q --allow-empty -m "agent work for ${ref}" + git -C "$AGENT" push -q origin "HEAD:${ref}" + git -C "$AGENT" rev-parse HEAD +} + +echo "=== 1. Every tip contained ===" +assert_eq "fresh bare repo: nothing to preserve" "0|" "$(check)" +git -C "$BARE" update-ref refs/heads/swarm/abc/interactive-x "$ONE" +assert_eq "an ancestor of HEAD counts as contained" "0|" "$(check)" +git -C "$BARE" update-ref -d refs/heads/swarm/abc/interactive-x + +echo "" +echo "=== 2. Uncontained tips in each namespace ===" +for ref in refs/heads/agent-work refs/heads/swarm/abc/interactive-x \ + refs/heads/agent-parked/1-20260101T000000Z refs/swarm-runs/r1/x; do + sha=$(agent_push "$ref") + assert_eq "${ref} blocks" "1|${ref} ${sha}" "$(check)" + # The checkout fetches the work but has not merged it: still missing. + git -C "$WORK" fetch -q "$BARE" "$ref" + assert_eq "${ref} fetched but not merged still blocks" \ + "1|${ref} ${sha}" "$(check)" + g -C "$WORK" merge -q --ff-only "$sha" + assert_eq "${ref} merged into HEAD no longer blocks" "0|" "$(check)" + git -C "$BARE" update-ref refs/heads/agent-work "$sha" +done + +echo "" +echo "=== 3. Every uncontained ref is listed ===" +a=$(agent_push refs/heads/agent-parked/1-a) +b=$(agent_push refs/heads/agent-parked/2-b) +out=$(check) +assert_eq "status 1 with two refs" "1" "${out%%|*}" +assert_eq "both refs are listed" "2" \ + "$(printf '%s\n' "${out#*|}" | grep -c 'agent-parked/' || true)" +git -C "$BARE" update-ref -d refs/heads/agent-parked/1-a "$a" +git -C "$BARE" update-ref -d refs/heads/agent-parked/2-b "$b" + +echo "" +echo "=== 4. Refs outside the preservation namespaces are ignored ===" +c=$(agent_push refs/heads/scratch) +assert_eq "an agent branch elsewhere does not block" "0|" "$(check)" +git -C "$BARE" update-ref -d refs/heads/scratch "$c" + +echo "" +echo "=== 5. A ref that is not a commit fails the check ===" +blob=$(printf 'x' | git -C "$BARE" hash-object -w --stdin) +git -C "$BARE" update-ref refs/swarm-runs/blob "$blob" +assert_eq "non-commit ref: status 2" "2" "$(check | cut -d'|' -f1)" +git -C "$BARE" update-ref -d refs/swarm-runs/blob + +echo "" +echo "=== 5b. Broken refs fail the check instead of being skipped ===" +# for-each-ref warns about these and leaves them out of its output. +mkdir -p "$BARE/refs/heads/agent-parked" +printf 'not a sha\n' > "$BARE/refs/heads/agent-parked/garbage" +assert_eq "malformed loose ref: status 2" "2" "$(check | cut -d'|' -f1)" +rm -f "$BARE/refs/heads/agent-parked/garbage" +printf '%s\n' "0123456789012345678901234567890123456789" \ + > "$BARE/refs/heads/agent-parked/missing" +assert_eq "ref to a missing object: status 2" "2" \ + "$(check | cut -d'|' -f1)" +rm -f "$BARE/refs/heads/agent-parked/missing" +assert_eq "clean again after removing them" "0|" "$(check)" + +echo "" +echo "=== 6. Not a bare repository ===" +rc=0 +swarm_preservation_check "$WORK" "$WORK" >/dev/null 2>&1 || rc=$? +assert_eq "a non-bare repo: status 2" "2" "$rc" +rc=0 +swarm_preservation_check "$TMPDIR/missing.git" "$WORK" >/dev/null 2>&1 \ + || rc=$? +assert_eq "a missing path: status 2" "2" "$rc" +ln -s "$BARE" "$TMPDIR/link.git" +rc=0 +swarm_preservation_check "$TMPDIR/link.git" "$WORK" >/dev/null 2>&1 \ + || rc=$? +assert_eq "a symlink: status 2" "2" "$rc" + +echo "" +echo "=== 7. A checkout git cannot read fails the check ===" +d=$(agent_push refs/heads/agent-parked/3-d) +rc=0 +swarm_preservation_check "$BARE" "$TMPDIR/not-a-repo" >/dev/null 2>&1 \ + || rc=$? +assert_eq "an unreadable checkout: status 2" "2" "$rc" +git -C "$BARE" update-ref -d refs/heads/agent-parked/3-d "$d" + +# HEAD is readable, but the object store behind the tips is not. +f=$(agent_push refs/heads/agent-parked/5-f) +# A broken alternates entry only in the checkout: HEAD still resolves. +printf '%s\n' "$TMPDIR/no-such-objects" > "$WORK/.git/objects/info/alternates" +rc=0 +swarm_preservation_check "$BARE" "$WORK" >/dev/null 2>"$TMPDIR/err" \ + || rc=$? +assert_eq "an object store git cannot read: status 2, not missing work" \ + "2" "$rc" +assert_eq "the git error is passed on" "1" \ + "$(grep -c 'cannot read' "$TMPDIR/err" || true)" +rm -f "$WORK/.git/objects/info/alternates" +# Same with a tip the checkout has: git reads it but still complains. +git -C "$WORK" fetch -q "$BARE" refs/heads/agent-parked/5-f +printf '%s\n' "$TMPDIR/no-such-objects" > "$WORK/.git/objects/info/alternates" +rc=0 +swarm_preservation_check "$BARE" "$WORK" >/dev/null 2>"$TMPDIR/err" \ + || rc=$? +assert_eq "a present tip in a broken object store: status 2" "2" "$rc" +rm -f "$WORK/.git/objects/info/alternates" +git -C "$BARE" update-ref -d refs/heads/agent-parked/5-f "$f" + +echo "" +echo "=== 7b. Grafts and replace objects cannot fake containment ===" +# An agent commit on a side line, not in HEAD's history. +e=$(agent_push refs/heads/agent-parked/4-e) +git -C "$WORK" fetch -q "$BARE" refs/heads/agent-parked/4-e +side=$(g -C "$WORK" commit-tree -p "$e" -m side "$(git -C "$WORK" \ + rev-parse 'HEAD^{tree}')") +assert_eq "side work blocks" "1" "$(check | cut -d'|' -f1)" +# A graft that makes HEAD a child of the agent commit. +head=$(git -C "$WORK" rev-parse HEAD) +printf '%s %s %s\n' "$head" "$(git -C "$WORK" rev-parse HEAD~1)" "$e" \ + > "$WORK/.git/info/grafts" +assert_eq "still blocks with a graft" "1" "$(check | cut -d'|' -f1)" +rm -f "$WORK/.git/info/grafts" +# A replace object that swaps HEAD for a commit with the agent parent. +fake=$(g -C "$WORK" commit-tree -p "$(git -C "$WORK" rev-parse HEAD~1)" \ + -p "$e" -m fake "$(git -C "$WORK" rev-parse 'HEAD^{tree}')") +git -C "$WORK" replace "$head" "$fake" +assert_eq "plain git is fooled by the replace object" "0" \ + "$(git -C "$WORK" merge-base --is-ancestor "$e" HEAD && echo 0 || echo 1)" +assert_eq "still blocks with a replace object" "1" \ + "$(check | cut -d'|' -f1)" +git -C "$WORK" replace -d "$head" +: "$side" +git -C "$BARE" update-ref -d refs/heads/agent-parked/4-e "$e" + +echo "" +echo "=== 8. The explanation says what to do with each ref ===" +msg=$(printf '%s\n' "refs/heads/agent-work 0123456789abcdef" \ + "refs/heads/swarm/r/interactive-x 0123456789abcdef" \ + "refs/heads/agent-parked/1-x 0123456789abcdef" \ + | swarm_preservation_explain "$BARE" 2>&1) +assert_eq "agent-work: harvest" "1" \ + "$(printf '%s\n' "$msg" | grep -c '^ agent-work .*harvest' || true)" +assert_eq "interactive branch: harvest" "1" \ + "$(printf '%s\n' "$msg" \ + | grep -c '^ swarm/r/interactive-x .*harvest' || true)" +assert_eq "parked branch: fetch and review" "1" \ + "$(printf '%s\n' "$msg" \ + | grep -c '^ agent-parked/1-x .*git fetch' || true)" +msg=$(printf '%s\n' 'refs/swarm-runs/$USER;x 0123456789abcdef' \ + | swarm_preservation_explain "$BARE" 2>&1) +assert_eq "a ref with shell characters is quoted" "1" \ + "$(printf '%s\n' "$msg" | grep -cF 'refs/swarm-runs/\$USER\;x' || true)" +msg=$(printf '%s\n' "refs/heads/agent-work 0123456789abcdef" \ + | swarm_preservation_explain "$BARE" 2>&1) +assert_eq "the explicit delete command is shown" "1" \ + "$(printf '%s\n' "$msg" | grep -c 'update-ref -d ' || true)" + +echo "" +echo "===============================" +echo " ${PASS} passed, ${FAIL} failed" +echo "===============================" + +[ "$FAIL" -eq 0 ] From 39e1a61b0625e52a1cbe12489ca9e88635b5eb6b Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:20:26 +0100 Subject: [PATCH 02/27] Start a new bare repo from the checkout's HEAD without old agent refs --- CHANGELOG.md | 9 ++++ USAGE.md | 6 +++ launch.sh | 66 ++++++++++++++++++----- tests/test_launch.sh | 121 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 189 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 538c388..869da88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- **Fix: a new bare repo starts from the checkout's HEAD.** The + clone copied every local branch, so an old local `agent-work`, + `swarm/*` or `agent-parked/*` branch in the checkout reached the new + engagement and harvest could take it for new agent work. + `create_bare_repo` now deletes the `swarm/*` and `agent-parked/*` + branches in the new bare repo, points `agent-work` and + `refs/swarm/base` at the checkout's HEAD, and records that commit as + `base_commit` in the state file. A checkout without a commit is + refused. - **Fix: a signal that ends costs.sh, progress.sh or dashboard.sh no longer prints "stopped before the end of its run" on Bash 4.1 and later.** Bash runs the EXIT trap with status 0 when a signal ends a diff --git a/USAGE.md b/USAGE.md index 28a4f76..196386c 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1071,6 +1071,12 @@ only when `SWARM_ENGAGEMENT_ID` is unset or equal to the recorded id, so a caller that names its engagement cannot act on another one. An empty value counts as set and is refused. +A new bare repo starts from the commit the checkout has: `agent-work` +and `refs/swarm/base` point at it, the state file records it as +`base_commit`, and the checkout's own `swarm/*` and `agent-parked/*` +branches are not copied, so branches from an earlier engagement never +look like new agent work. + ### Engagement lock `start`, `post-process`, `interactive`, `chat`, `shell` and diff --git a/launch.sh b/launch.sh index 25368f6..d90325b 100755 --- a/launch.sh +++ b/launch.sh @@ -271,33 +271,75 @@ write_state_file() { } create_bare_repo() { - local label="${1:-bare repo}" + local label="${1:-bare repo}" base new="${BARE_REPO}.new" echo "--- Creating ${label} ---" + # The engagement starts from the commit the checkout has now. + base=$(git -C "$REPO_ROOT" rev-parse --verify --quiet 'HEAD^{commit}') \ + || { echo "ERROR: ${REPO_ROOT} has no commit to start from." >&2 + exit 1; } rm_docker_dir "$BARE_REPO" + rm_docker_dir "$new" + if [ -e "$BARE_REPO" ] || [ -e "$new" ]; then + echo "ERROR: cannot remove the old ${BARE_REPO}." >&2 + exit 1 + fi # A local clone hardlinks object files by default, and the chmod # below would then make the checkout's own objects writable for # every user. Copy them instead, with any objects the checkout # borrows through alternates. - git clone --bare --no-hardlinks --dissociate "$REPO_ROOT" "$BARE_REPO" - git -C "$BARE_REPO" branch agent-work HEAD 2>/dev/null || true - git -C "$BARE_REPO" symbolic-ref HEAD refs/heads/agent-work + git clone --bare --no-hardlinks --dissociate "$REPO_ROOT" "$new" + # Later commands join any bare repo they find, so the clone gets its + # name only once it is ready and the engagement is recorded. A + # failure leaves at most the .new directory, which nothing joins. + if ! prepare_bare_repo "$new" "$base" \ + || ! begin_engagement "$base" \ + || ! mv "$new" "$BARE_REPO"; then + echo "ERROR: cannot prepare ${BARE_REPO}." >&2 + rm_docker_dir "$new" + exit 1 + fi +} + +# Set up the fresh clone DIR for an engagement that starts at BASE. +# Returns 1 at the first failure. +prepare_bare_repo() { + local dir="$1" base="$2" ref refs + # The clone copies every local branch. Branches that harvest or + # the preservation checks treat as agent work start empty, so an + # old local agent-work, interactive or parked branch is never + # taken for work of this engagement. + refs=$(git -C "$dir" for-each-ref --format='%(refname)' \ + refs/heads/swarm/ refs/heads/agent-parked/ refs/swarm-runs/) \ + || return 1 + # Ref names cannot contain whitespace or glob characters, so word + # splitting yields one name per word. + for ref in $refs; do + git -C "$dir" update-ref -d "$ref" || return 1 + done + git -C "$dir" update-ref refs/heads/agent-work "$base" || return 1 + git -C "$dir" update-ref refs/swarm/base "$base" || return 1 + git -C "$dir" symbolic-ref HEAD refs/heads/agent-work || return 1 # The container's agent user can have a different UID from the host # user, so the repo stays writable for everyone. The owner-only # runtime directory around it keeps other host users out. - git -C "$BARE_REPO" config core.sharedRepository world - chmod -R a+rwX "$BARE_REPO" - begin_engagement + git -C "$dir" config core.sharedRepository world || return 1 + chmod -R a+rwX "$dir" || return 1 } # A new bare repo starts a new engagement: record its id, which every -# later phase checks against SWARM_ENGAGEMENT_ID. +# later phase checks against SWARM_ENGAGEMENT_ID, and the commit it +# starts from (BASE). Agents can write to the bare repo, including its +# refs/swarm/base, so the state file keeps the copy that later checks +# of the agents' branches can trust. Returns 1 if either fails. begin_engagement() { - local id - id=$(swarm_engagement_new) || exit 1 + local base="$1" id + id=$(swarm_engagement_new) || return 1 swarm_state_update "$STATE_FILE" \ - '.engagement_id = $id | .engagement_started_at = $at' \ + '.engagement_id = $id | .engagement_started_at = $at + | .base_commit = $base' \ --arg id "$id" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - || exit 1 + --arg base "$base" \ + || return 1 echo "Engagement: ${id}" } diff --git a/tests/test_launch.sh b/tests/test_launch.sh index a005d82..c4ff655 100755 --- a/tests/test_launch.sh +++ b/tests/test_launch.sh @@ -1353,7 +1353,8 @@ _hl_sub_pack=${_hl_packs[0]} _hl_src_mode=$(_hl_mode "$_hl_src_pack") _hl_sub_mode=$(_hl_mode "$_hl_sub_pack") -for _hl_fn in create_bare_repo begin_engagement mirror_submodules; do +for _hl_fn in create_bare_repo prepare_bare_repo begin_engagement \ + mirror_submodules; do awk -v fn="$_hl_fn" '$0 ~ "^" fn "\\(\\)[[:space:]]*\\{" { p = 1 } p { print } p && /^\}[[:space:]]*$/ { exit }' "$_hl_launch" @@ -2814,6 +2815,124 @@ assert_eq "start --dashboard unlocks before exec" "1" \ || true)" # ============================================================ +echo "" +echo "=== 52. A new bare repo records its base and drops old agent refs ===" + +# The checkout has a stale local agent-work and branches in the +# namespaces harvest and the preservation checks read. None of them may +# reach the new bare repo; agent-work and refs/swarm/base start at HEAD, +# and the state file records HEAD as the base. +_br_dir="$TMPDIR/base-refs" +_br_src="$_br_dir/src" +mkdir -p "$_br_dir" +git init -q "$_br_src" +_br_git() { + git -C "$_br_src" -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false "$@" +} +_br_git commit -q --allow-empty -m one +_br_old=$(git -C "$_br_src" rev-parse HEAD) +_br_git commit -q --allow-empty -m two +_br_head=$(git -C "$_br_src" rev-parse HEAD) +for _br_ref in refs/heads/agent-work refs/heads/swarm/abc/interactive-x \ + refs/heads/agent-parked/1-20260101T000000Z refs/swarm-runs/r1 \ + refs/heads/feature; do + git -C "$_br_src" update-ref "$_br_ref" "$_br_old" +done +awk '$0 ~ /^(create_bare_repo|prepare_bare_repo|begin_engagement)\(\)[[:space:]]*\{/ { p = 1 } + p { print } + p && /^\}[[:space:]]*$/ { p = 0 }' "$_LAUNCH_SH" > "$_br_dir/funcs.sh" +_br_run() { + ( + # shellcheck source=/dev/null + source "$_br_dir/funcs.sh" + # The extracted functions call this stub. + # shellcheck disable=SC2329 + rm_docker_dir() { rm -rf "$1"; } + REPO_ROOT="$1" + BARE_REPO="$_br_dir/bare-$2.git" + STATE_FILE="$_br_dir/state-$2.json" + create_bare_repo + ) >/dev/null 2>&1 +} +_br_rc=0 +_br_run "$_br_src" 1 || _br_rc=$? +_br_bare="$_br_dir/bare-1.git" +assert_eq "bare repo is created" "0" "$_br_rc" +assert_eq "agent-work starts at HEAD, not the old local branch" \ + "$_br_head" "$(git -C "$_br_bare" rev-parse agent-work 2>/dev/null)" +assert_eq "refs/swarm/base is HEAD" "$_br_head" \ + "$(git -C "$_br_bare" rev-parse refs/swarm/base 2>/dev/null)" +assert_eq "the state file records HEAD as the base" "$_br_head" \ + "$(jq -r .base_commit "$_br_dir/state-1.json" 2>/dev/null)" +assert_eq "old interactive, parked and run refs are not copied" "" \ + "$(git -C "$_br_bare" for-each-ref --format='%(refname)' \ + refs/heads/swarm/ refs/heads/agent-parked/ refs/swarm-runs/)" +assert_eq "other branches are copied as before" "$_br_old" \ + "$(git -C "$_br_bare" rev-parse feature 2>/dev/null)" +assert_eq "the bare repo's HEAD is agent-work" "refs/heads/agent-work" \ + "$(git -C "$_br_bare" symbolic-ref HEAD 2>/dev/null)" + +# A failure while the clone is prepared must not leave a bare repo that +# later commands would join with old agent branches in it, even when +# removing the clone fails too: rm_docker_dir is a no-op here. FAIL +# names the step that fails. +_br_fail() { + local n="$1" fail="$2" + ( + # shellcheck source=/dev/null + source "$_br_dir/funcs.sh" + # The extracted functions call these stubs. + # shellcheck disable=SC2329 + rm_docker_dir() { :; } + # shellcheck disable=SC2329 + git() { + case "$fail $*" in + "list "*" for-each-ref "*) return 128 ;; + "delete "*" update-ref -d "*) return 128 ;; + esac + command git "$@" + } + # shellcheck disable=SC2329 + swarm_state_update() { + [ "$fail" != state ] + } + REPO_ROOT="$_br_src" + BARE_REPO="$_br_dir/bare-$n.git" + STATE_FILE="$_br_dir/state-$n.json" + create_bare_repo + ) >/dev/null 2>&1 +} +for _br_case in 3:list 4:delete 5:state; do + _br_n="${_br_case%%:*}" + _br_step="${_br_case#*:}" + _br_rc=0 + _br_fail "$_br_n" "$_br_step" || _br_rc=$? + assert_eq "$_br_step failure stops bare creation" "1" "$_br_rc" + assert_eq "$_br_step failure leaves no bare repo to join" "false" \ + "$([ -e "$_br_dir/bare-$_br_n.git" ] && echo true || echo false)" + assert_eq "$_br_step failure records no engagement" "false" \ + "$([ -e "$_br_dir/state-$_br_n.json" ] && echo true || echo false)" +done + +# An old bare repo that cannot be removed is refused, not cloned into. +mkdir -p "$_br_dir/bare-6.git" +touch "$_br_dir/bare-6.git/marker" +_br_rc=0 +_br_fail 6 none || _br_rc=$? +assert_eq "an old bare repo that stays is refused" "1" "$_br_rc" +assert_eq "the old bare repo is left as it was" "true" \ + "$([ -e "$_br_dir/bare-6.git/marker" ] && echo true || echo false)" + +# A checkout without a commit has nothing to start from. +git init -q "$_br_dir/empty" +_br_rc=0 +_br_run "$_br_dir/empty" 2 || _br_rc=$? +assert_eq "a checkout without commits is refused" "1" "$_br_rc" +assert_eq "no state is written for it" "false" \ + "$([ -e "$_br_dir/state-2.json" ] && echo true || echo false)" + + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" From ca32561863747790cfd999581be1d8e6f395985c Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:40:22 +0100 Subject: [PATCH 03/27] Keep commits that no push shipped instead of resetting them away --- CHANGELOG.md | 13 ++ USAGE.md | 5 + lib/harness.sh | 80 ++++++++++- tests/test_session_end_push.sh | 245 ++++++++++++++++++++++++++++++++- 4 files changed, 337 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 869da88..4d54dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +- **Fix: the harness keeps commits that no push shipped.** Each + loop iteration reset the workspace to `origin/agent-work`, so + commits left from a session whose push and park both failed, or + from a restart before the push, stayed only in the reflog. The loop + now fetches with `--prune` and checks HEAD against origin's + `agent-work`, `swarm/*` and `agent-parked/*`; commits on none of + them go through the session-end push first, and if they are still + unshipped the harness exits without resetting. After a scratch + worktree push, the branch moves to the pushed commits, since the + originals were shipped as copies, but only when the tree is clean + and `git cherry` finds every local commit on origin. The scratch + push also stopped reporting success when it could not list the + commits to transplant. - **Fix: a new bare repo starts from the checkout's HEAD.** The clone copied every local branch, so an old local `agent-work`, `swarm/*` or `agent-parked/*` branch in the checkout reached the new diff --git a/USAGE.md b/USAGE.md index 196386c..fcc3734 100644 --- a/USAGE.md +++ b/USAGE.md @@ -237,6 +237,11 @@ session-end push pipeline only runs at the bottom of each loop iteration and never gets to run on a mid-session SIGTERM. +Commits that no push shipped, because the push and the park both +failed or the container restarted first, are not reset away: at the +top of the next iteration the harness pushes them, and when that +fails it exits and leaves them on the workspace branch. + For larger swarms where 45+ agents race the same push lock, raise the grace: diff --git a/lib/harness.sh b/lib/harness.sh index 3337982..b93b816 100755 --- a/lib/harness.sh +++ b/lib/harness.sh @@ -155,7 +155,11 @@ _scratch_worktree_push() { git fetch --no-recurse-submodules origin agent-work 2>&1 \ | hlog_pipe || true - _shas=$(git rev-list --reverse origin/agent-work..HEAD 2>/dev/null) + if ! _shas=$(git rev-list --reverse origin/agent-work..HEAD 2>/dev/null) + then + hlog_err "scratch push: cannot list the unpushed commits" + return 1 + fi if [ -z "$_shas" ]; then hlog "scratch push: no unpushed commits (already in sync)" return 0 @@ -282,6 +286,23 @@ _scratch_worktree_push() { # Returns 0 on success or no-op (nothing to push, or pushed # cleanly), 1 if there were unpushed commits and every push # path failed. +# Succeed when resetting the branch to origin/agent-work after a +# scratch transplant loses nothing: `git status` shows no change and +# `git cherry` finds an equivalent on origin/agent-work for every +# local commit. Any failure counts as "not settled". +_transplant_settled() { + local _st _ch + _st=$(git status --porcelain 2>/dev/null) || return 1 + [ -z "$_st" ] || return 1 + _ch=$(git cherry origin/agent-work HEAD 2>/dev/null) || return 1 + # A pattern match, not a pipe into grep: under pipefail an early + # grep exit can fail printf with SIGPIPE. + case $'\n'"$_ch" in + *$'\n+'*) return 1 ;; + esac + return 0 +} + _session_end_push() { git fetch --no-recurse-submodules origin 2>&1 | hlog_pipe AFTER=$(git rev-parse origin/agent-work) @@ -412,16 +433,35 @@ _session_end_push() { # turns the 0.20.4 "100% data loss on push failure" state # into "push succeeds via transplant" for every failure # pattern we've observed in production. + local _transplanted=false if [ "$_push_ok" != true ]; then hlog "rebase path exhausted, trying scratch worktree fallback" if _scratch_worktree_push; then _push_ok=true + _transplanted=true fi fi if [ "$_push_ok" = true ]; then git fetch --no-recurse-submodules origin 2>&1 | hlog_pipe AFTER=$(git rev-parse origin/agent-work) + # The scratch push shipped copies of the local commits; the + # originals are still on this branch and on no origin ref. + # Move the branch to what was pushed, so the workspace does + # not look like it holds unshipped work, but only when that + # loses nothing: the tree is clean (the pre-push stash took + # every edit) and origin has an equivalent of every original + # commit. Otherwise the branch stays, and the loop top stops + # the harness with the commits in /workspace. + if [ "$_transplanted" = true ]; then + if _transplant_settled; then + git reset -q --hard origin/agent-work 2>&1 \ + | hlog_pipe || true + else + hlog_err "scratch push: keeping the local branch;" \ + "the tree is dirty or a commit has no copy on origin" + fi + fi return 0 fi @@ -677,9 +717,45 @@ rm -f "$RETRY_FILE" trap '_on_signal TERM' TERM trap '_on_signal INT' INT +# Print a commit of HEAD that no origin ref holds (agent-work, swarm/* +# or agent-parked/*, as last fetched), or nothing when every commit is +# on one. A missing origin/agent-work or a failing rev-list prints +# "unknown", so the caller treats it as unshipped. +_unshipped_commit() { + local out + if ! git rev-parse -q --verify refs/remotes/origin/agent-work \ + >/dev/null 2>&1; then + echo unknown + return 0 + fi + if ! out=$(git rev-list -n 1 HEAD --not refs/remotes/origin/agent-work \ + --remotes='origin/swarm/*' --remotes='origin/agent-parked/*' \ + 2>/dev/null); then + echo unknown + return 0 + fi + printf '%s' "$out" +} + while true; do # Reset to latest. Do not re-init submodules; setup changes would be lost. - git fetch --no-recurse-submodules origin 2>&1 | hlog_pipe + git fetch --prune --no-recurse-submodules origin 2>&1 | hlog_pipe + # Commits on no origin ref (an earlier push and its park both + # failed, or the container restarted before it could push) exist + # only on this branch, and the reset below would leave them in the + # reflog alone. Ship them first; if that fails, stop and keep + # them on the branch, where cleanup finds them. + if [ -n "$(_unshipped_commit)" ]; then + hlog "unshipped commits from an earlier session, pushing them" + _session_end_push || true + git fetch --prune --no-recurse-submodules origin 2>&1 \ + | hlog_pipe || true + if [ -n "$(_unshipped_commit)" ]; then + hlog_err "commits remain on no origin ref; keeping them" \ + "in /workspace and exiting" + exit 1 + fi + fi git reset --hard origin/agent-work 2>&1 | hlog_pipe BEFORE=$(git rev-parse origin/agent-work) diff --git a/tests/test_session_end_push.sh b/tests/test_session_end_push.sh index c5fef14..4287810 100755 --- a/tests/test_session_end_push.sh +++ b/tests/test_session_end_push.sh @@ -460,13 +460,14 @@ assert_eq "every fatal exit 1 is preceded by _session_end_push" \ "yes" "$PREORDER_OK" # Call-site accounting: -# 1 definition + 1 inline (happy path) + 3 fatal-exit blocks -# + 1 emergency-shutdown trap == 6 occurrences total. +# 1 definition + 1 loop top (commits left from an earlier session) +# + 1 inline (happy path) + 3 fatal-exit blocks +# + 1 emergency-shutdown trap == 7 occurrences total. # Pinning the total is a tripwire against an accidental extra # reference; each individual site is also asserted above and # (for the trap) in ยง7 below. -assert_eq "_session_end_push referenced exactly 6 times" \ - "6" \ +assert_eq "_session_end_push referenced exactly 7 times" \ + "7" \ "$(grep -cE '_session_end_push' "$HARNESS_FILE")" # ============================================================ @@ -527,6 +528,242 @@ assert_contains "_on_signal exits with TERM code 143" \ assert_contains "_on_signal exits with INT code 130" \ 'exit_code=130' "$ON_SIGNAL_BODY" +# ============================================================ +echo "" +echo "=== 8. Loop top keeps commits that are on no origin ref ===" + +# Run the real loop top (fetch, unshipped check, reset) against a +# real origin, with _session_end_push stubbed: "fail" ships nothing, +# "park" pushes HEAD to agent-parked/ as the real park path does. +awk '/^_unshipped_commit\(\) \{/,/^\}$/' "$HARNESS_FILE" \ + > "$TMPDIR/loop_top.sh" +awk ' + /^while true; do$/ { print "loop_top() {"; p = 1; next } + p { print } + p && /^ git reset --hard origin\/agent-work/ { print "}"; exit } +' "$HARNESS_FILE" >> "$TMPDIR/loop_top.sh" + +_lt_git() { + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false "$@" +} +# Prepare origin and a clone at origin/agent-work; print the clone. +_lt_setup() { + local d="$TMPDIR/lt-$1" + git init -q --bare "$d/origin.git" + git init -q "$d/seed" + _lt_git -C "$d/seed" commit -q --allow-empty -m base + git -C "$d/seed" push -q "$d/origin.git" HEAD:refs/heads/agent-work + git clone -q -b agent-work "$d/origin.git" "$d/ws" 2>/dev/null + : > "$d/pushes" + echo "$d/ws" +} +# Run loop_top in WS with push mode MODE; print "rc pushes". +_lt_run() { + local ws="$1" mode="$2" + ( + cd "$ws" || exit 99 + set -euo pipefail + # shellcheck source=/dev/null + source "$TMPDIR/loop_top.sh" + # Stubs for the harness helpers the loop top calls. + # shellcheck disable=SC2329 + hlog() { :; } + # shellcheck disable=SC2329 + hlog_err() { :; } + # shellcheck disable=SC2329 + hlog_pipe() { cat >/dev/null; } + # shellcheck disable=SC2329 + _session_end_push() { + echo push >> "$ws/../pushes" + [ "$mode" = park ] || return 1 + git push -q origin HEAD:refs/heads/agent-parked/1-x + } + loop_top + ) >/dev/null 2>&1 + echo "$? $(wc -l < "$ws/../pushes" | tr -d ' ')" +} + +# Nothing local: no push, reset to origin. +_lt_ws=$(_lt_setup clean) +assert_eq "clean workspace: loop goes on without a push" "0 0" \ + "$(_lt_run "$_lt_ws" fail)" + +# A local commit and a push that fails: stop, keep the commit on HEAD. +_lt_ws=$(_lt_setup stuck) +_lt_git -C "$_lt_ws" commit -q --allow-empty -m local +_lt_tip=$(git -C "$_lt_ws" rev-parse HEAD) +assert_eq "unshipped commit and failed push: exit 1 after one push" \ + "1 1" "$(_lt_run "$_lt_ws" fail)" +assert_eq "the commit stays on HEAD, no reset" "$_lt_tip" \ + "$(git -C "$_lt_ws" rev-parse HEAD)" + +# A local commit that the push parks: go on and reset to agent-work. +_lt_ws=$(_lt_setup parked) +_lt_git -C "$_lt_ws" commit -q --allow-empty -m local +_lt_tip=$(git -C "$_lt_ws" rev-parse HEAD) +assert_eq "unshipped commit parked by the push: loop goes on" "0 1" \ + "$(_lt_run "$_lt_ws" park)" +assert_eq "after parking, HEAD is reset to origin/agent-work" \ + "$(git -C "$_lt_ws" rev-parse origin/agent-work)" \ + "$(git -C "$_lt_ws" rev-parse HEAD)" +assert_eq "the parked branch on origin holds the commit" "$_lt_tip" \ + "$(git -C "$_lt_ws/../origin.git" rev-parse refs/heads/agent-parked/1-x)" + +# A commit already on origin's swarm/* counts as shipped. +_lt_ws=$(_lt_setup swarm) +_lt_git -C "$_lt_ws" commit -q --allow-empty -m local +git -C "$_lt_ws" push -q origin HEAD:refs/heads/swarm/a/interactive-b +assert_eq "commit on origin swarm/*: no push" "0 0" \ + "$(_lt_run "$_lt_ws" fail)" + +# A commit only on another origin branch is not one of the refs the +# preservation checks read, so it still counts as unshipped. +_lt_ws=$(_lt_setup other) +_lt_git -C "$_lt_ws" commit -q --allow-empty -m local +git -C "$_lt_ws" push -q origin HEAD:refs/heads/agent-workx +assert_eq "commit only on another origin branch: unshipped" "1 1" \ + "$(_lt_run "$_lt_ws" fail)" + +# A deleted origin branch no longer protects the commit (fetch --prune). +_lt_ws=$(_lt_setup pruned) +_lt_git -C "$_lt_ws" commit -q --allow-empty -m local +git -C "$_lt_ws" push -q origin HEAD:refs/heads/agent-parked/old +git -C "$_lt_ws" fetch -q origin +git -C "$_lt_ws/../origin.git" update-ref -d refs/heads/agent-parked/old +assert_eq "commit on a deleted origin branch: unshipped" "1 1" \ + "$(_lt_run "$_lt_ws" fail)" + +# Without origin/agent-work the check cannot decide: fail closed. +_lt_ws=$(_lt_setup noaw) +git -C "$_lt_ws/../origin.git" branch -q keep agent-work +git -C "$_lt_ws/../origin.git" symbolic-ref HEAD refs/heads/keep +git -C "$_lt_ws/../origin.git" update-ref -d refs/heads/agent-work +assert_eq "no origin/agent-work: treated as unshipped" "1 1" \ + "$(_lt_run "$_lt_ws" fail)" + +# A scratch transplant pushes copies of the local commits. The branch +# then moves to what was pushed, but only when nothing is lost. Run +# the real functions with the rebase path failing, so the scratch +# worktree fallback runs. STUB adds one more failing git command. +for _tp_fn in _scratch_worktree_push _transplant_settled _session_end_push +do + awk -v fn="$_tp_fn" '$0 ~ "^" fn "\\(\\) \\{" { p = 1 } + p { print } + p && /^\}$/ { exit }' "$HARNESS_FILE" +done > "$TMPDIR/transplant.sh" +_tp_run() { + local ws="$1" stub="$2" + ( + cd "$ws" || exit 99 + set -euo pipefail + # shellcheck source=/dev/null + source "$TMPDIR/transplant.sh" + # The scratch path names its worktree after the agent. + # shellcheck disable=SC2034 + AGENT_ID=1 + # Stubs for the harness helpers and the retry backoff. + # shellcheck disable=SC2329 + hlog() { :; } + # shellcheck disable=SC2329 + hlog_err() { :; } + # shellcheck disable=SC2329 + hlog_pipe() { cat >/dev/null; } + # shellcheck disable=SC2329 + sleep() { :; } + # shellcheck disable=SC2329 + git() { + case " $* " in + *" pull "*) return 1 ;; + *" $stub "*) return 1 ;; + esac + command git -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false "$@" + } + _session_end_push + ) >/dev/null 2>&1 +} +# Prepare a workspace with one local commit that changes a file, while +# origin moved on, so the transplanted copy is a different commit. +_tp_setup() { + local ws seed + ws=$(_lt_setup "tp-$1") + seed="$ws/../seed" + echo change > "$ws/a.txt" + git -C "$ws" add a.txt + _lt_git -C "$ws" commit -q -m local + echo other > "$seed/b.txt" + git -C "$seed" add b.txt + _lt_git -C "$seed" commit -q -m other + git -C "$seed" push -q "$ws/../origin.git" HEAD:refs/heads/agent-work + echo "$ws" +} + +# The check reads `git cherry` output: any "+" line, wherever it is, +# means a commit has no copy on origin. A long output must not hide it. +_ts_check() { + ( + set -euo pipefail + # shellcheck source=/dev/null + source "$TMPDIR/transplant.sh" + # shellcheck disable=SC2329 + git() { + case "$1" in + status) return 0 ;; + cherry) printf '%s' "$_ts_out"; return "$_ts_rc" ;; + esac + } + _transplant_settled + ) && echo settled || echo kept +} +_ts_rc=0 +_ts_out="- aaa"$'\n'"- bbb" +assert_eq "every commit has a copy: settled" "settled" "$(_ts_check)" +_ts_out="+ aaa"$'\n'"$(printf -- '- x\n%.0s' $(seq 1 10000))" +assert_eq "first commit unmatched before a long list: kept" "kept" \ + "$(_ts_check)" +_ts_out="- aaa"$'\n'"+ bbb" +assert_eq "last commit unmatched: kept" "kept" "$(_ts_check)" +_ts_out="" +_ts_rc=1 +assert_eq "git cherry fails: kept" "kept" "$(_ts_check)" + +_tp_ws=$(_tp_setup clean) +_tp_rc=0 +_tp_run "$_tp_ws" none-such || _tp_rc=$? +assert_eq "transplant with a clean tree succeeds" "0" "$_tp_rc" +assert_eq "transplant with a clean tree resets to the pushed tip" \ + "$(git -C "$_tp_ws/../origin.git" rev-parse agent-work)" \ + "$(git -C "$_tp_ws" rev-parse HEAD)" +assert_eq "the pushed tip carries the change" "change" \ + "$(git -C "$_tp_ws/../origin.git" show agent-work:a.txt 2>/dev/null)" +assert_eq "the pushed tip also keeps origin's commit" "other" \ + "$(git -C "$_tp_ws/../origin.git" show agent-work:b.txt 2>/dev/null)" + +# The stash fails, so an untracked file stays in the tree: no reset. +_tp_ws=$(_tp_setup dirty) +_tp_tip=$(git -C "$_tp_ws" rev-parse HEAD) +echo wip > "$_tp_ws/untracked.txt" +_tp_run "$_tp_ws" "stash push" || true +assert_eq "dirty tree after a failed stash: branch kept" "$_tp_tip" \ + "$(git -C "$_tp_ws" rev-parse HEAD)" +assert_eq "dirty tree after a failed stash: the file stays" "wip" \ + "$(cat "$_tp_ws/untracked.txt" 2>/dev/null)" +assert_eq "dirty tree after a failed stash: the copy is still pushed" \ + "change" \ + "$(git -C "$_tp_ws/../origin.git" show agent-work:a.txt 2>/dev/null)" + +# Listing the commits to transplant fails: report failure, keep all. +_tp_ws=$(_tp_setup enum) +_tp_tip=$(git -C "$_tp_ws" rev-parse HEAD) +_tp_origin=$(git -C "$_tp_ws/../origin.git" rev-parse agent-work) +_tp_rc=0 +_tp_run "$_tp_ws" "rev-list --reverse" || _tp_rc=$? +assert_eq "failed commit listing: the push reports failure" "1" "$_tp_rc" +assert_eq "failed commit listing: branch kept" "$_tp_tip" \ + "$(git -C "$_tp_ws" rev-parse HEAD)" +assert_eq "failed commit listing: origin unchanged" "$_tp_origin" \ + "$(git -C "$_tp_ws/../origin.git" rev-parse agent-work)" + # ============================================================ echo "" echo "===============================" From e37cd6e9cc43f36621eb8a3678768ae8dba15303 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Tue, 3 Nov 2026 16:58:41 +0100 Subject: [PATCH 04/27] Keep the bare repo and mirrors while an interactive session is open --- CHANGELOG.md | 12 ++ USAGE.md | 22 ++- launch.sh | 99 ++++++----- lib/project.sh | 205 +++++++++++++++++++++ tests/test_launch.sh | 415 ++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 681 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d54dc5..be4fb34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## Unreleased +- **Fix: open interactive sessions keep the bare repo and the + mirrors.** `interactive` held the lock in a helper process until the + container ran, and `start` could still replace the bare repo and the + mirrors under a running session. `interactive` now records the + session under the lock with its pid, that process's start time and + the Docker daemon, releases the lock and execs `docker run`. While + that process runs, or the container runs after a detach, `start` + fails, and so do `post-process` and `interactive` when they would + create the bare repo. A joining session no longer replaces the + mirrors; it only adds missing ones. `post-process` now mirrors the + submodules when it creates the bare repo, as `start` and + `interactive` do. - **Fix: the harness keeps commits that no push shipped.** Each loop iteration reset the workspace to `origin/agent-work`, so commits left from a session whose push and park both failed, or diff --git a/USAGE.md b/USAGE.md index fcc3734..7618b41 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1088,11 +1088,23 @@ look like new agent work. `harvest.sh` take `engagement.lock` in the runtime directory with `flock -n` before they change anything, and fail at once while another operation holds it. `start` and `post-process` keep it until they exit -(`harvest.sh` started by them shares it). `interactive` keeps it until -the session's container runs, so its mounts are in place before -another command could replace them, and then lets it go for the rest -of the session; `start --dashboard` lets it go before the dashboard -opens. `stop`, `status`, `logs`, `progress.sh`, `costs.sh` and the +(`harvest.sh` started by them shares it). `interactive` keeps it while +it records the session in `interactive-.pending` in the +runtime directory (its pid, that process's start time and the Docker +daemon), then lets it go and execs `docker run`, which keeps the pid. +Docker mounts the bare repo and the mirrors when the container starts, +so while that process runs, or the container runs after a detach +or was created and may still start (found by its `swarm-core.session` +label, which a rename keeps), +`start`, and `post-process` or `interactive` when they would create +the bare repo, fail instead of replacing them. Every command that +creates the bare repo mirrors the submodules; a session that joins an +existing bare repo keeps the mirrors already there and only adds +missing ones. Once the session's process has ended, a check from +another Docker daemon (another Docker context or `DOCKER_HOST`) fails +instead of guessing, and each check and launch pins one Docker context +for all its Docker commands. `start --dashboard` lets the lock go before +the dashboard opens. `stop`, `status`, `logs`, `progress.sh`, `costs.sh` and the dashboard never take it, and containers push through git without it. A caller that runs several commands as one unit can hold the lock diff --git a/launch.sh b/launch.sh index d90325b..ea22e21 100755 --- a/launch.sh +++ b/launch.sh @@ -207,22 +207,21 @@ own_container_state() { || printf 'not found' } -# Hold the lock descriptors this process inherited until container NAME -# has started (running, or already stopped), file STOP exists, or -# process PARENT is gone. cmd_interactive creates STOP as soon as -# docker run returns; the parent check covers a launch.sh that was -# killed. There is no timeout: a slow start keeps the lock, to protect -# what the container is about to mount. -hold_lock_until_running() { - local name="$1" parent="$2" stop="$3" state - while [ ! -e "$stop" ] && kill -0 "$parent" 2>/dev/null; do - state=$(docker inspect -f '{{.State.Status}}' "$name" \ - 2>/dev/null || true) - case "$state" in - running|exited|dead) return 0 ;; - esac - sleep 0.1 - done +# Stop when an interactive session may still be about to start its +# container: Docker makes the mounts at start, so the bare repo and the +# mirrors must stay as they are while such a session lives. +refuse_while_interactive_pending() { + local live + live=$(swarm_pending_live "$RUNTIME_DIR") || exit 1 + [ -n "$live" ] || return 0 + echo "ERROR: interactive sessions of this checkout are open" \ + "(containers below); the bare repo and the mirrors stay" \ + "until they end:" >&2 + printf '%s\n' "$live" | sed 's/^/ /' >&2 + echo "A container that was created but never ran also counts; once" \ + "you are sure it will not start, remove it with" \ + "docker rm ." >&2 + exit 1 } # Lifecycle commands call this before they touch the bare repo or the @@ -230,8 +229,9 @@ hold_lock_until_running() { # the directory, as they did), create or check the private runtime # directory, take its engagement lock, and record this checkout as its # owner. The lock stays held until the command exits, except that -# interactive releases it once its container runs and start --dashboard -# before the dashboard opens; children such as harvest.sh inherit it. +# interactive releases it once its session is recorded, before docker +# run, and start --dashboard before the dashboard opens; children such +# as harvest.sh inherit it. prepare_runtime() { swarm_refuse_legacy_state "$(swarm_project_id "$PROJECT_RAW")" \ || exit 1 @@ -343,9 +343,14 @@ begin_engagement() { echo "Engagement: ${id}" } +# A session that creates the bare repo mirrors the submodules too; one +# that joins keeps the mirrors the engagement has, which other sessions +# may be about to mount, and only adds the ones that are missing. ensure_bare_repo_for_interactive() { if [ ! -d "$BARE_REPO" ]; then + refuse_while_interactive_pending create_bare_repo "bare repo for interactive session" + mirror_submodules return fi swarm_engagement_join "$STATE_FILE" || exit 1 @@ -363,9 +368,14 @@ ensure_bare_repo_for_interactive() { echo " $(swarm_bare_rm_command "$RUNTIME_DIR")" >&2 exit 1 fi + mirror_submodules missing } +# Mirror every submodule into MIRROR_DIR, replacing old mirrors. With +# "missing", only create the mirrors that do not exist yet and leave the +# others, which a session may be about to mount. mirror_submodules() { + local only_missing="${1:-}" cd "$REPO_ROOT" mkdir -p "$MIRROR_DIR" git submodule foreach --quiet 'echo "$name|$toplevel/.git/modules/$sm_path"' | \ @@ -373,6 +383,9 @@ mirror_submodules() { local safe_name mirror safe_name="${name//\//_}" mirror="${MIRROR_DIR}/${safe_name}.git" + if [ "$only_missing" = missing ] && [ -d "$mirror" ]; then + continue + fi rm_docker_dir "$mirror" echo "--- Mirroring submodule: ${name} ---" # Copy objects, as create_bare_repo does, so the chmod below @@ -660,7 +673,6 @@ HELP prepare_runtime ensure_bare_repo_for_interactive - mirror_submodules build_image MIRROR_ARGS=() @@ -687,29 +699,35 @@ HELP remove_own_container "$name" # The session can last hours and only pushes through git, so it runs - # without the lock, but the lock has to last until the container - # runs: its mounts are made at start, and a concurrent start could - # otherwise replace the bare repo or the mirrors first. A helper - # that inherits the lock descriptor holds it until then; this shell - # drops its own copy so that docker run, which keeps the terminal - # for the whole session, does not inherit it. - # The helper gets no stdio, so it never keeps a caller's output pipe - # open. Its stop file lives in the private runtime directory. - local helper stop rc=0 - stop="${RUNTIME_DIR}/.interactive-$$.stop" - rm -f "$stop" - hold_lock_until_running "$name" "$$" "$stop" \ - /dev/null 2>&1 & - helper=$! + # without the lock. It is recorded under the lock with this + # process's pid; then the lock goes and exec keeps that pid for + # docker run, which makes the container and its mounts. Until this + # process ends, or its container after a detach, start and + # post-process leave the bare repo and the mirrors alone + # (refuse_while_interactive_pending). One Docker context serves + # the daemon id and docker run, even if another terminal switches + # it meanwhile. + local daemon + if ! swarm_docker_pin || ! daemon=$(swarm_docker_daemon_id); then + echo "ERROR: cannot read the Docker daemon id." >&2 + exit 1 + fi + if ! swarm_pending_write "$RUNTIME_DIR" "$name" "$$" "$daemon"; then + echo "ERROR: cannot record the interactive session." >&2 + exit 1 + fi swarm_unlock echo "--- Starting interactive ${profile_label} (${agent_model}) ---" echo "Branch: ${branch}" - docker run -it \ - --name "$name" \ + # --name and the session label come after docker_args: the last + # one wins, and the record must find the container Docker makes. + exec docker run -it \ -v "${BARE_REPO}:/upstream:rw" \ "${MIRROR_ARGS[@]+"${MIRROR_ARGS[@]}"}" \ "${SIGNING_KEY_ARGS[@]+"${SIGNING_KEY_ARGS[@]}"}" \ "${DOCKER_EXTRA_ARGS[@]+"${DOCKER_EXTRA_ARGS[@]}"}" \ + --name "$name" \ + --label "${SWARM_SESSION_LABEL}=${name}" \ "${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"}" \ --label "$CHECKOUT_LABEL" \ --label "$PROJECT_LABEL" \ @@ -732,13 +750,7 @@ HELP -e "SWARM_INTERACTIVE_PROFILE=${profile_label}" \ -e "SWARM_INTERACTIVE_MODE=${mode}" \ --entrypoint /interactive.sh \ - "$IMAGE_NAME" || rc=$? - # docker run has returned: the container ran or never will. wait - # only reaps this shell's own child, so a reused PID is never hit. - : > "$stop" - wait "$helper" 2>/dev/null || true - rm -f "$stop" - return "$rc" + "$IMAGE_NAME" } cmd_start() { @@ -806,6 +818,7 @@ cmd_start() { fi fi + refuse_while_interactive_pending create_bare_repo "bare repo" # Mirror each submodule so containers can init without network. @@ -1089,7 +1102,9 @@ cmd_post_process() { prepare_runtime if [ ! -d "$BARE_REPO" ]; then + refuse_while_interactive_pending create_bare_repo "bare repo for post-process" + mirror_submodules else swarm_engagement_join "$STATE_FILE" || exit 1 fi diff --git a/lib/project.sh b/lib/project.sh index 40024c4..1f7344c 100644 --- a/lib/project.sh +++ b/lib/project.sh @@ -397,6 +397,9 @@ swarm_runtime_verify() { # checkout that created it in these labels. SWARM_PROJECT_LABEL="swarm-core.project" SWARM_CHECKOUT_LABEL="swarm-core.checkout" +# An interactive session's container carries its record's name here: +# a running container can be renamed, but not relabeled. +SWARM_SESSION_LABEL="swarm-core.session" # docker ps format and jq definitions for the project's containers. A # container is the project's when its project label is the id, @@ -691,3 +694,205 @@ swarm_unlock() { eval "exec ${fd}>&-" unset SWARM_LOCK_FD } + +# An interactive session runs for hours without the lock. launch.sh +# records it under the lock in DIR/interactive-.pending +# with its own pid, that process's start time and the Docker daemon, +# releases the lock and execs `docker run`, which keeps the pid and +# makes the container and its mounts. While that process lives, or the +# container runs after a detach, the bare repo and the mirrors the +# container mounts must stay as they are. + +# Print the start time of process PID, or nothing when ps cannot tell. +# The format follows the locale and the time zone, so both are fixed: +# the reader may run with other settings than the session. +swarm_proc_started() { + LC_ALL=C TZ=UTC0 ps -o lstart= -p "$1" 2>/dev/null \ + | sed 's/^ *//; s/ *$//' || true +} + +# Pin the Docker daemon for the rest of the calling shell: each docker +# command resolves the current context again, and another terminal can +# switch it (docker context use) between two commands. DOCKER_HOST or +# DOCKER_CONTEXT already pin it; otherwise export DOCKER_CONTEXT as the +# current context. Callers that must not leak it declare +# DOCKER_CONTEXT local first. +swarm_docker_pin() { + local ctx + if [ -n "${DOCKER_HOST:-}" ] || [ -n "${DOCKER_CONTEXT:-}" ]; then + return 0 + fi + ctx=$(docker context show 2>/dev/null) || return 1 + # Docker allows letters, digits and "_.+-" in context names. + case "$ctx" in + ''|*[!"$SWARM_ALNUM"._+-]*) return 1 ;; + esac + DOCKER_CONTEXT="$ctx" + export DOCKER_CONTEXT +} + +# Print the id of the Docker daemon the CLI talks to. A container id +# means something only on its daemon. +swarm_docker_daemon_id() { + local id + id=$(docker info -f '{{.ID}}' 2>/dev/null) || return 1 + case "$id" in + ''|*[!"$SWARM_ALNUM":-]*) return 1 ;; + esac + printf '%s' "$id" +} + +# Succeed when NAME is a container name Docker accepts. +swarm_container_name_valid() { + case "$1" in + ["$SWARM_ALNUM"]*) ;; + *) return 1 ;; + esac + case "$1" in + *[!"$SWARM_ALNUM"._-]*) return 1 ;; + esac +} + +# Record that the session in process PID runs container NAME on the +# daemon with id DAEMON. +swarm_pending_write() { + local dir="$1" name="$2" pid="$3" daemon="$4" file + if ! swarm_container_name_valid "$name"; then + echo "ERROR: unexpected container name: ${name}" >&2 + return 1 + fi + case "$daemon" in + ''|*[!"$SWARM_ALNUM":-]*) + echo "ERROR: unexpected Docker daemon id: ${daemon}" >&2 + return 1 ;; + esac + file="$dir/interactive-${name}.pending" + { + printf 'pid=%s\n' "$pid" + printf 'started=%s\n' "$(swarm_proc_started "$pid")" + printf 'daemon=%s\n' "$daemon" + } > "$file.tmp" && mv "$file.tmp" "$file" +} + +# Succeed when no container of session NAME runs or can still start: +# Docker lists none with its session label, or only exited or dead +# ones. A created container counts as running, since the daemon can +# finish starting it after the client is gone. Returns 1 while one +# runs, and 2 when Docker cannot tell. +swarm_container_settled() { + local name="$1" states state + states=$(docker ps -a \ + --filter "label=${SWARM_SESSION_LABEL}=${name}" \ + --format '{{.State}}' 2>/dev/null) || return 2 + for state in $states; do + case "$state" in + exited|dead) ;; + created|running|paused|restarting|removing) return 1 ;; + *) return 2 ;; + esac + done + return 0 +} + +# Print the container names of the sessions in DIR that are open, one +# per line, and remove the records of the others. A session is open +# while its process runs (a process whose start time ps cannot read +# counts as running), and after that while its container runs: a +# session can detach and leave it running. The container is found by +# its session label, which a rename does not change. A record must be exactly +# "pid=N\nstarted=T\ndaemon=D\n" (T empty or a C-locale ps start time, +# D the daemon that made the container); anything else, a record that +# cannot be read or removed, a record from another daemon, or a +# container state Docker cannot give, fails the check. +swarm_pending_live() { + local dir="$1" f name content rest l1 l2 l3 pid started daemon now rc + local current="" + # The daemon id and the container states below must come from one + # daemon; the pin ends with this function. + local DOCKER_CONTEXT="${DOCKER_CONTEXT:-}" + for f in "$dir"/interactive-*.pending; do + [ -e "$f" ] || [ -L "$f" ] || continue + name="${f##*/interactive-}" + name="${name%.pending}" + # The x keeps trailing newlines from command substitution. + if ! swarm_container_name_valid "$name"; then + echo "ERROR: ${f} does not name a container." >&2 + return 1 + fi + if [ -L "$f" ] || [ ! -f "$f" ] \ + || ! content=$(cat "$f" 2>/dev/null && echo x); then + echo "ERROR: cannot read ${f}." >&2 + return 1 + fi + content="${content%x}" + pid="" + started="" + daemon="" + case "$content" in + pid=*$'\n'started=*$'\n'daemon=*$'\n') + l1="${content%%$'\n'*}" + rest="${content#*$'\n'}" + l2="${rest%%$'\n'*}" + rest="${rest#*$'\n'}" + l3="${rest%$'\n'}" + pid="${l1#pid=}" + started="${l2#started=}" + daemon="${l3#daemon=}" + if [ "$started" = "$l2" ] || [ "$daemon" = "$l3" ]; then + pid="" + fi + case "$daemon" in + ''|*[!"$SWARM_ALNUM":-]*) pid="" ;; + esac ;; + esac + case "$pid" in + ''|*[!0123456789]*) + echo "ERROR: ${f} is not a session record." >&2 + return 1 ;; + esac + case "$started" in + ''|[A-Z][a-z][a-z]' '[A-Z][a-z][a-z]' '[' '0123][0-9]' '[0-9][0-9]:[0-9][0-9]:[0-9][0-9]' '[0-9][0-9][0-9][0-9]) + ;; + *) + echo "ERROR: ${f} is not a session record." >&2 + return 1 ;; + esac + if kill -0 "$pid" 2>/dev/null; then + now=$(swarm_proc_started "$pid") + if [ -z "$started" ] || [ -z "$now" ] \ + || [ "$now" = "$started" ]; then + printf '%s\n' "$name" + continue + fi + fi + # The process is gone: only the daemon that made the container + # can say whether it still runs. + if [ -z "$current" ]; then + if ! swarm_docker_pin \ + || ! current=$(swarm_docker_daemon_id); then + echo "ERROR: cannot read the Docker daemon id." >&2 + return 1 + fi + fi + if [ "$daemon" != "$current" ]; then + echo "ERROR: ${f} belongs to another Docker daemon" \ + "(${daemon}); switch back to it to check the session." >&2 + return 1 + fi + rc=0 + swarm_container_settled "$name" || rc=$? + case "$rc" in + 0) ;; + 1) + printf '%s\n' "$name" + continue ;; + *) + echo "ERROR: cannot read the state of container ${name}." >&2 + return 1 ;; + esac + rm -f "$f" || { + echo "ERROR: cannot remove ${f}." >&2 + return 1 + } + done +} diff --git a/tests/test_launch.sh b/tests/test_launch.sh index c4ff655..988c7b9 100755 --- a/tests/test_launch.sh +++ b/tests/test_launch.sh @@ -2109,9 +2109,9 @@ CMD_INTERACTIVE_BODY=$(awk ' p && /^\}[[:space:]]*$/ { exit } ' "$LAUNCH_FILE") -assert_eq "interactive uses docker run -it" "1" \ +assert_eq "interactive execs docker run -it" "1" \ "$(printf '%s\n' "$CMD_INTERACTIVE_BODY" \ - | grep -cF 'docker run -it')" + | grep -cF 'exec docker run -it')" assert_eq "interactive uses dedicated entrypoint" "1" \ "$(printf '%s\n' "$CMD_INTERACTIVE_BODY" \ | grep -cF -- '--entrypoint /interactive.sh')" @@ -2414,9 +2414,24 @@ _st_bin="$TMPDIR/start-bin" _st_repo="$TMPDIR/start-repo" _st_rt="$TMPDIR/start-runtime" mkdir -p "$_st_bin" "$_st_repo/sub/configs" -# Like the real CLI, inspecting a missing container fails. -printf '#!/bin/bash\n[ "$1 $2" = "container inspect" ] && exit 1\nexit 0\n' \ - > "$_st_bin/docker" +# Like the real CLI, inspecting a missing container fails, info and +# context show answer, and the alpine rm -rf that rm_docker_dir runs +# removes its last argument. +cat > "$_st_bin/docker" <<'EOF' +#!/bin/bash +if [ "$1 $2" = "container inspect" ]; then + echo "Error: No such container: ${*: -1}" >&2 + exit 1 +fi +[ "$1" = info ] && echo d1 +[ "$1 $2" = "context show" ] && echo default +case " $* " in + " run --rm "*" alpine rm -rf "*) + for _a in "$@"; do _last="$_a"; done + rm -rf "$_last" ;; +esac +exit 0 +EOF chmod +x "$_st_bin/docker" git init -q "$_st_repo" echo "Prompt." > "$_st_repo/prompt.md" @@ -2706,20 +2721,24 @@ cat > "$_lk_bin/docker" <> "$_lk_log" [ "\$1 \$2" = "container inspect" ] && exit 1 -# The interactive container reports "created" until the session marks -# itself running. The session checks the lock before that and again -# while it runs, prints a line, and exits with FAKE_EXIT. With -# FAKE_NEVER_RUNS the container never starts. -if [ "\$1" = inspect ]; then - if [ -e "$_lk_rt.running" ]; then echo running; else echo created; fi - exit 0 -fi +# docker run -it checks that the session record for its --name names +# its own pid (exec keeps launch.sh's) and logs its arguments, checks +# the lock while the session runs, prints a line and exits with +# FAKE_EXIT. With FAKE_NEVER_RUNS the container never starts. +[ "\$1" = info ] && { echo d1; exit 0; } +[ "\$1 \$2" = "context show" ] && { echo default; exit 0; } if [ "\$1 \$2" = "run -it" ]; then - if flock -n 6 6>>"$_lk_rt/engagement.lock"; then - echo "lock free before the container runs" >> "$_lk_log" + _name="" + _prev="" + for _a in "\$@"; do + [ "\$_prev" = --name ] && _name="\$_a" + _prev="\$_a" + done + if grep -qx "pid=\$\$" "$_lk_rt/interactive-\$_name.pending" 2>/dev/null + then + echo "record names the session" >> "$_lk_log" fi [ -n "\${FAKE_NEVER_RUNS:-}" ] && exit 3 - : > "$_lk_rt.running" for _i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do if flock -n 6 6>>"$_lk_rt/engagement.lock"; then echo "lock free during session" >> "$_lk_log" @@ -2728,7 +2747,6 @@ if [ "\$1 \$2" = "run -it" ]; then sleep 0.1 done echo "SESSION OUTPUT" - rm -f "$_lk_rt.running" exit "\${FAKE_EXIT:-0}" fi exit 0 @@ -2757,23 +2775,45 @@ _lk_rc=0 assert_eq "start with the holder's SWARM_LOCK_FD runs" "0" "$_lk_rc" exec 8>&- -# An interactive session keeps the lock until its container runs (its -# mounts are then in place) and gives it up for the rest of the -# session; docker run keeps the terminal, so output and exit status -# come straight from the container. +# An interactive session records itself under the lock, gives the lock +# up and execs docker run, which keeps the terminal and the exit status. : > "$_lk_log" _lk_rc=0 _lk_out=$(FAKE_EXIT=4 run_lock interactive --agent-index 1) || _lk_rc=$? -assert_eq "the lock is held until the container runs" "0" \ - "$(grep -c 'lock free before the container runs' "$_lk_log" || true)" +assert_eq "the session is recorded before docker run" "1" \ + "$(grep -c 'record names the session' "$_lk_log" || true)" assert_eq "the lock is free once the container runs" "1" \ "$(grep -c 'lock free during session' "$_lk_log" || true)" assert_eq "the session's output reaches the terminal" "true" \ "$(grep -q 'SESSION OUTPUT' <<< "$_lk_out" \ && echo true || echo false)" assert_eq "interactive returns the container's exit status" "4" "$_lk_rc" -# A container that never starts: docker run fails, launch.sh exits, and -# the helper lets the lock go once it sees that. +# docker_args reach docker run unchanged, run-only options included, +# and a --name among them does not replace the recorded name: the fake +# takes the last --name, as Docker does, and checks its record. +_lk_cfg="$_st_repo/sub/configs/docker-args.json" +jq '. + {docker_args: ["--detach-keys=ctrl-x", "--label", "-itd", + "--name", "custom-session"]}' \ + "$_st_repo/sub/configs/swarm.json" > "$_lk_cfg" +: > "$_lk_log" +(cd "$_st_repo" && PATH="$_lk_bin:$PATH" SWARM_RUNTIME_DIR="$_lk_rt" \ + SWARM_CONFIG=sub/configs/docker-args.json SWARM_SKIP_DEP_CHECK=1 \ + "$BASH" "$_LAUNCH_SH" interactive --agent-index 1 >/dev/null 2>&1) \ + || true +assert_eq "docker_args reach docker run unchanged" "1" \ + "$(grep '^run -it ' "$_lk_log" \ + | grep -c -- ' --detach-keys=ctrl-x --label -itd --name custom' \ + || true)" +assert_eq "the container gets the recorded name" "1" \ + "$(grep -c 'record names the session' "$_lk_log" || true)" +assert_eq "the session label follows docker_args too" "1" \ + "$(grep '^run -it ' "$_lk_log" \ + | grep -c -- ' --name custom-session --name [^ ]* --label swarm-core.session=' \ + || true)" +rm -f "$_lk_cfg" + +# A container that never starts: docker run fails, and the lock is +# already free. : > "$_lk_log" _lk_rc=0 FAKE_NEVER_RUNS=1 run_lock interactive --agent-index 1 >/dev/null \ @@ -2933,6 +2973,331 @@ assert_eq "no state is written for it" "false" \ "$([ -e "$_br_dir/state-2.json" ] && echo true || echo false)" +# ============================================================ +echo "" +echo "=== 53. Open interactive sessions keep the bare repo and the mirrors ===" + +# The session record names a process and its start time. It is live +# while that process runs; a record whose process is gone, or whose +# pid now belongs to a newer process, is removed. +_pd_dir="$TMPDIR/pending" +mkdir -p "$_pd_dir" +# The caller's Docker selectors would steer the stub below; the cases +# that set them come last. +unset DOCKER_CONTEXT DOCKER_HOST +# Docker's answers once a session's process is gone. Two contexts: A +# (daemon PD_DAEMON) and B (daemon d2, where no session container +# exists). The current context is in a file, as docker context use +# keeps it; DOCKER_CONTEXT overrides it. With PD_SWITCH set, docker +# info switches the current context to B, as another terminal could. +# PD_STATE is the state of the container with the session's label on +# A, "gone" for none, "error" for a failed listing. A lookup by name +# finds nothing, as after a rename of the detached container. +PD_STATE=gone +PD_DAEMON=d1 +PD_SWITCH="" +_pd_ctx="$TMPDIR/pending-context" +echo A > "$_pd_ctx" +# shellcheck disable=SC2329 +docker() { + local ctx + ctx="${DOCKER_CONTEXT:-$(cat "$_pd_ctx")}" + if [ "$1 $2" = "context show" ]; then + cat "$_pd_ctx" + return 0 + fi + if [ "$1" = info ]; then + [ -z "$PD_SWITCH" ] || echo B > "$_pd_ctx" + if [ "$ctx" = A ]; then echo "$PD_DAEMON"; else echo d2; fi + return 0 + fi + if [ "$1" != ps ]; then + echo "Error: No such container: ${*: -1}" >&2 + return 1 + fi + [ "$ctx" = A ] || return 0 + case "$PD_STATE" in + gone) ;; + error) echo "Cannot connect to the Docker daemon" >&2; return 1 ;; + *) + case "$*" in + *" label=swarm-core.session="*) echo "$PD_STATE" ;; + esac ;; + esac +} +_pd_id_a="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +_pd_id_b="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +sleep 60 & +_pd_live=$! +sh -c 'exit 0' & +_pd_dead=$! +wait "$_pd_dead" 2>/dev/null || true +swarm_pending_write "$_pd_dir" "$_pd_id_a" "$_pd_live" d1 +swarm_pending_write "$_pd_dir" "$_pd_id_b" "$_pd_dead" d1 +assert_eq "the record names the pid" "pid=$_pd_live" \ + "$(grep '^pid=' "$_pd_dir/interactive-$_pd_id_a.pending")" +assert_eq "only the running session is live" "$_pd_id_a" \ + "$(swarm_pending_live "$_pd_dir")" +assert_eq "the record of the ended session is removed" "false" \ + "$([ -e "$_pd_dir/interactive-$_pd_id_b.pending" ] \ + && echo true || echo false)" +printf 'pid=%s\nstarted=Thu Jan 1 00:00:00 1970\ndaemon=d1\n' \ + "$_pd_live" > "$_pd_dir/interactive-$_pd_id_b.pending" +assert_eq "a pid with another start time is not the session" \ + "$_pd_id_a" "$(swarm_pending_live "$_pd_dir")" +assert_eq "its record is removed" "false" \ + "$([ -e "$_pd_dir/interactive-$_pd_id_b.pending" ] \ + && echo true || echo false)" +printf 'pid=x\n' > "$_pd_dir/interactive-$_pd_id_b.pending" +_pd_rc=0 +swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || _pd_rc=$? +assert_eq "a record without a pid fails the check" "1" "$_pd_rc" +rm -f "$_pd_dir/interactive-$_pd_id_b.pending" +ln -s "$_pd_dir/interactive-$_pd_id_a.pending" \ + "$_pd_dir/interactive-$_pd_id_b.pending" +_pd_rc=0 +swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || _pd_rc=$? +assert_eq "a symlinked record fails the check" "1" "$_pd_rc" +rm -f "$_pd_dir/interactive-$_pd_id_b.pending" +_pd_rc=0 +swarm_pending_write "$_pd_dir" "-bad name" "$_pd_live" d1 2>/dev/null \ + || _pd_rc=$? +assert_eq "a malformed container name is not recorded" "1" "$_pd_rc" +printf 'pid=%s\nstarted=\ndaemon=d1\n' "$_pd_live" \ + > "$_pd_dir/interactive--x.pending" +_pd_rc=0 +swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || _pd_rc=$? +assert_eq "a record whose file name is no container name fails" "1" \ + "$_pd_rc" +rm -f "$_pd_dir/interactive--x.pending" +_pd_rc=0 +swarm_pending_write "$_pd_dir" "$_pd_id_b" "$_pd_live" "" 2>/dev/null \ + || _pd_rc=$? +assert_eq "a record without a daemon id is not written" "1" "$_pd_rc" + +# The start time is read the same way whatever the caller's time zone +# and locale, so a live session is not taken for a reused pid. +TZ=UTC LC_ALL=C swarm_pending_write "$_pd_dir" "$_pd_id_b" "$_pd_live" d1 +assert_eq "a session recorded in UTC is live from another zone" \ + "$_pd_id_a $_pd_id_b" \ + "$(TZ=Pacific/Kiritimati LC_TIME=fr_FR.UTF-8 \ + swarm_pending_live "$_pd_dir" | sort | tr '\n' ' ' | sed 's/ $//')" +rm -f "$_pd_dir/interactive-$_pd_id_b.pending" + +# A session can detach and leave the container running, and the +# container can be renamed: the session is still open after its +# process ended, found by its label. +swarm_pending_write "$_pd_dir" "$_pd_id_b" "$_pd_dead" d1 +PD_STATE=running +assert_eq "a detached session whose container runs stays open" \ + "$_pd_id_a $_pd_id_b" \ + "$(swarm_pending_live "$_pd_dir" | sort | tr '\n' ' ' | sed 's/ $//')" +PD_STATE=error +_pd_rc=0 +swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || _pd_rc=$? +assert_eq "a container state Docker cannot give fails the check" "1" \ + "$_pd_rc" +assert_eq "and keeps the record" "true" \ + "$([ -e "$_pd_dir/interactive-$_pd_id_b.pending" ] \ + && echo true || echo false)" +# Container ids mean something only on their daemon: after a switch to +# another daemon, "no such container" proves nothing. +PD_STATE=gone +echo B > "$_pd_ctx" +_pd_rc=0 +swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || _pd_rc=$? +assert_eq "a record from another Docker daemon fails the check" "1" \ + "$_pd_rc" +assert_eq "and stays" "true" \ + "$([ -e "$_pd_dir/interactive-$_pd_id_b.pending" ] \ + && echo true || echo false)" +# A switch between reading the daemon id and inspecting the container +# must not send the inspect to the other daemon. +echo A > "$_pd_ctx" +PD_STATE=running +PD_SWITCH=1 +assert_eq "a context switch during the check keeps the session open" \ + "$_pd_id_a $_pd_id_b" \ + "$(swarm_pending_live "$_pd_dir" 2>/dev/null | sort | tr '\n' ' ' \ + | sed 's/ $//')" +PD_SWITCH="" +echo A > "$_pd_ctx" +assert_eq "the pin does not outlive the check" "unset" \ + "${DOCKER_CONTEXT-unset}" +# A created container can still start after its client died. +PD_STATE=created +assert_eq "a session whose container is only created stays open" \ + "$_pd_id_a $_pd_id_b" \ + "$(swarm_pending_live "$_pd_dir" | sort | tr '\n' ' ' | sed 's/ $//')" +PD_STATE=exited +assert_eq "an ended session whose container exited is closed" \ + "$_pd_id_a" "$(swarm_pending_live "$_pd_dir")" +assert_eq "its record is removed" "false" \ + "$([ -e "$_pd_dir/interactive-$_pd_id_b.pending" ] \ + && echo true || echo false)" +PD_STATE=gone + +# Records that are not exactly "pid=N" and "started=T" fail the check +# and stay, even with a live pid. +_pd_bad() { + local label="$1" content="$2" rc=0 + printf '%s' "$content" > "$_pd_dir/interactive-$_pd_id_b.pending" + swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || rc=$? + assert_eq "$label fails the check" "1" "$rc" + assert_eq "$label is kept" "true" \ + "$([ -e "$_pd_dir/interactive-$_pd_id_b.pending" ] \ + && echo true || echo false)" + rm -f "$_pd_dir/interactive-$_pd_id_b.pending" +} +_pd_nl=$'\n' +_pd_ok_tail="started=${_pd_nl}daemon=d1${_pd_nl}" +_pd_bad "a garbage start time" \ + "pid=$_pd_live${_pd_nl}started=garbage${_pd_nl}daemon=d1${_pd_nl}" +_pd_bad "a garbage day" "pid=$_pd_live${_pd_nl}started=Thu Jan garbage\ + 00:00:00 1970${_pd_nl}daemon=d1${_pd_nl}" +_pd_bad "a second start time" "pid=$_pd_live${_pd_nl}started=${_pd_nl}\ +started=Thu Jan 1 00:00:00 1970${_pd_nl}daemon=d1${_pd_nl}" +_pd_bad "an extra line" "pid=$_pd_live${_pd_nl}${_pd_ok_tail}other=1${_pd_nl}" +_pd_bad "a missing start line" "pid=$_pd_live${_pd_nl}daemon=d1${_pd_nl}" +_pd_bad "a missing daemon line" "pid=$_pd_live${_pd_nl}started=${_pd_nl}" +_pd_bad "a garbage daemon id" \ + "pid=$_pd_live${_pd_nl}started=${_pd_nl}daemon=a b${_pd_nl}" +_pd_bad "a second pid" "pid=$_pd_live${_pd_nl}pid=1${_pd_nl}${_pd_ok_tail}" +_pd_bad "trailing blank lines" \ + "pid=$_pd_live${_pd_nl}${_pd_ok_tail}${_pd_nl}${_pd_nl}" +_pd_bad "no final newline" "pid=$_pd_live${_pd_nl}started=${_pd_nl}daemon=d1" +# Root reads any file, so the unreadable case needs another user. +if [ "$(id -u)" != 0 ]; then + printf 'pid=%s\n%s' "$_pd_live" "$_pd_ok_tail" \ + > "$_pd_dir/interactive-$_pd_id_b.pending" + chmod 000 "$_pd_dir/interactive-$_pd_id_b.pending" + _pd_rc=0 + swarm_pending_live "$_pd_dir" >/dev/null 2>&1 || _pd_rc=$? + assert_eq "an unreadable record fails the check" "1" "$_pd_rc" + chmod 600 "$_pd_dir/interactive-$_pd_id_b.pending" + rm -f "$_pd_dir/interactive-$_pd_id_b.pending" +fi + +# A selector the caller set already pins the daemon: DOCKER_CONTEXT is +# kept, DOCKER_HOST is left alone. Docker allows "+" in context names. +# Print the context swarm_docker_pin leaves, or "unset". +_pd_pinned() { + swarm_docker_pin && printf '%s' "${DOCKER_CONTEXT-unset}" +} +echo dev+local > "$_pd_ctx" +assert_eq "a context name with + is pinned" "dev+local" "$(_pd_pinned)" +echo B > "$_pd_ctx" +assert_eq "a caller's DOCKER_CONTEXT is kept" "A" \ + "$(DOCKER_CONTEXT=A _pd_pinned)" +assert_eq "with DOCKER_HOST no context is pinned" "unset" \ + "$(DOCKER_HOST=unix:///x _pd_pinned)" +# With the caller's DOCKER_CONTEXT=A, a context switch of the current +# context changes nothing either. +echo A > "$_pd_ctx" +swarm_pending_write "$_pd_dir" "$_pd_id_b" "$_pd_dead" d1 +PD_STATE=running +PD_SWITCH=1 +assert_eq "with the caller's DOCKER_CONTEXT a switch keeps it open" \ + "$_pd_id_a $_pd_id_b" \ + "$(DOCKER_CONTEXT=A swarm_pending_live "$_pd_dir" 2>/dev/null \ + | sort | tr '\n' ' ' | sed 's/ $//')" +PD_SWITCH="" +PD_STATE=gone +echo A > "$_pd_ctx" +rm -f "$_pd_dir/interactive-$_pd_id_b.pending" + +unset -f docker + +# Every place that creates a bare repo checks for open sessions first, +# and interactive mirrors only when it creates the bare repo. +assert_eq "each bare repo creation checks for open sessions" "3" \ + "$(grep -B1 '^ *create_bare_repo "' "$_LAUNCH_SH" \ + | grep -c 'refuse_while_interactive_pending')" +assert_eq "cmd_interactive mirrors only through the bare repo step" "0" \ + "$(printf '%s\n' "$CMD_INTERACTIVE_BODY" \ + | grep -c 'mirror_submodules' || true)" + +# Every bare repo creation mirrors the submodules. A joining session +# adds missing mirrors and leaves existing ones alone. +assert_eq "each bare repo creation mirrors the submodules" "3" \ + "$(grep -A3 '^ *create_bare_repo "' "$_LAUNCH_SH" \ + | grep -c '^ *mirror_submodules$')" +_mm_dir="$TMPDIR/mirror-missing" +mkdir -p "$_mm_dir" +git init -q "$_mm_dir/sub" +git -C "$_mm_dir/sub" -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false commit -q --allow-empty -m sub +git init -q "$_mm_dir/src" +git -C "$_mm_dir/src" -c protocol.file.allow=always submodule add -q \ + "$_mm_dir/sub" lib >/dev/null 2>&1 +git -C "$_mm_dir/src" -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false commit -q -m add-sub +awk '$0 ~ /^mirror_submodules\(\)[[:space:]]*\{/ { p = 1 } + p { print } + p && /^\}[[:space:]]*$/ { exit }' "$_LAUNCH_SH" > "$_mm_dir/funcs.sh" +_mm_run() { + ( + # shellcheck source=/dev/null + source "$_mm_dir/funcs.sh" + # The extracted function calls this stub. + # shellcheck disable=SC2329 + rm_docker_dir() { rm -rf "$1"; } + REPO_ROOT="$_mm_dir/src" + MIRROR_DIR="$_mm_dir/mirrors" + mirror_submodules "$@" + ) >/dev/null 2>&1 +} +_mm_run +assert_eq "the submodule is mirrored" "true" \ + "$([ -d "$_mm_dir/mirrors/lib.git" ] && echo true || echo false)" +touch "$_mm_dir/mirrors/lib.git/keep" +_mm_run missing +assert_eq "a joining session leaves an existing mirror alone" "true" \ + "$([ -e "$_mm_dir/mirrors/lib.git/keep" ] && echo true || echo false)" +rm -rf "$_mm_dir/mirrors/lib.git" +_mm_run missing +assert_eq "a joining session adds a missing mirror" "true" \ + "$(git -C "$_mm_dir/mirrors/lib.git" rev-parse --is-bare-repository \ + 2>/dev/null || echo false)" +touch "$_mm_dir/mirrors/lib.git/keep" +_mm_run +assert_eq "a new engagement replaces the mirrors" "false" \ + "$([ -e "$_mm_dir/mirrors/lib.git/keep" ] && echo true || echo false)" + +# Through launch.sh: with a session open, start leaves the bare repo +# alone, and another interactive session still joins it. Once the +# session ends, start replaces the bare repo as before. +_pd_rt="$TMPDIR/pending-runtime" +run_pd() { + (cd "$_st_repo" && PATH="$_st_bin:$PATH" SWARM_RUNTIME_DIR="$_pd_rt" \ + SWARM_CONFIG=sub/configs/swarm.json SWARM_SKIP_DEP_CHECK=1 \ + "$BASH" "$_LAUNCH_SH" "$@" 2>&1) +} +run_pd start >/dev/null +touch "$_pd_rt/upstream.git/keep" +swarm_pending_write "$_pd_rt" "$_pd_id_a" "$_pd_live" d1 +_pd_rc=0 +_pd_out=$(run_pd start) || _pd_rc=$? +assert_eq "start refuses while a session is open" "1" "$_pd_rc" +assert_eq "start names the open session" "true" \ + "$(grep -q "$_pd_id_a" <<< "$_pd_out" \ + && echo true || echo false)" +assert_eq "the bare repo stays" "true" \ + "$([ -e "$_pd_rt/upstream.git/keep" ] && echo true || echo false)" +_pd_rc=0 +run_pd interactive --agent-index 1 >/dev/null || _pd_rc=$? +assert_eq "another interactive session joins meanwhile" "0" "$_pd_rc" +kill "$_pd_live" 2>/dev/null || true +wait "$_pd_live" 2>/dev/null || true +_pd_rc=0 +run_pd start >/dev/null || _pd_rc=$? +assert_eq "after the session ends start goes ahead" "0" "$_pd_rc" +assert_eq "and replaces the bare repo" "false" \ + "$([ -e "$_pd_rt/upstream.git/keep" ] && echo true || echo false)" +assert_eq "the old record is gone" "false" \ + "$([ -e "$_pd_rt/interactive-$_pd_id_a.pending" ] \ + && echo true || echo false)" + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" From 85aad19497132ae2d873e5a9020c5205ce251abe Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:55:05 +0100 Subject: [PATCH 05/27] Preserve every object and git-dir file of a repository location as a recovery bundle --- CHANGELOG.md | 9 + lib/preserve-scan.sh | 577 +++++++++++++++++++++++++++++++++ lib/preserve.sh | 308 ++++++++++++++++++ tests/test_preserve.sh | 716 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1610 insertions(+) create mode 100644 lib/preserve-scan.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index be4fb34..cd351ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +- **Feature: recovery snapshots (building block).** + `lib/preserve-scan.sh` packs everything Git keeps in a repository + location (every ref, reflog entry, stash, operation state, object, + and every file of its git dirs) into a bundle and reports work-tree + content outside Git's history; `swarm_recovery_import` copies such a + bundle into the checkout under `refs/swarm-recovery// + /` and checks it, and `swarm_recovery_verify_checkout` + re-hashes the checkout's objects. Cleanup uses them in a later + change; no command calls them yet. - **Fix: open interactive sessions keep the bare repo and the mirrors.** `interactive` held the lock in a helper process until the container ran, and `start` could still replace the bare repo and the diff --git a/lib/preserve-scan.sh b/lib/preserve-scan.sh new file mode 100644 index 0000000..7f5c8bf --- /dev/null +++ b/lib/preserve-scan.sh @@ -0,0 +1,577 @@ +#!/bin/bash +# Inventory one repository location and pack everything Git keeps there +# into a bundle, so cleanup can copy it into the checkout before it +# deletes the location. Runs on the host for the bare repo and the +# mirrors, and inside a throwaway container (volume mounted read-only) +# for a workspace. It only reads ROOT. +# +# Usage: preserve-scan.sh worktree ROOT CONTEXT OUT +# preserve-scan.sh gitdir ROOT - OUT +# +# worktree: ROOT is a checkout (a workspace) whose repository is +# ROOT/.git; CONTEXT (full, slim or none) says which deletions under +# .claude the engine made itself. +# gitdir: ROOT is a repository directory (a bare repo or a mirror). +# OUT is an empty directory. On success the scan writes: +# OUT/report one line per finding (see below) +# OUT/bundle a bundle whose refs are refs/v//t/ (one per +# object to keep), refs/v//f/ (one per file of the +# git dir) and refs/v//manifest, refs/v//files +# OUT/objects every object id the bundle must carry, one per line +# Report lines: +# empty ROOT has no files at all +# gitdir a git dir, relative to ROOT +# block content the bundle cannot carry +# uncommitted content outside Git's history that +# only an explicit discard may drop +# Exit status: 0 when the report and bundle are complete, 3 when the +# scan cannot tell (message on stderr). Compatible with Bash 3.2. + +set -euo pipefail + +# Git's own environment must not redirect anything this scan reads. +for _v in $(env | sed -n 's/^\(GIT_[A-Za-z0-9_]*\)=.*/\1/p'); do + unset "$_v" +done +# Replacement objects and grafts would make Git walk parents a commit +# does not have, so objects could look reached that the bundle lacks. +export GIT_NO_REPLACE_OBJECTS=1 GIT_GRAFT_FILE=/dev/null +export GIT_NO_LAZY_FETCH=1 GIT_TERMINAL_PROMPT=0 +export LC_ALL=C + +MODE="${1:-}" +ROOT="${2:-}" +CONTEXT="${3:-}" +OUT="${4:-}" + +# A single file larger than this, or more than TOTAL bytes of files per +# git dir tree, is reported as uncommitted instead of being stored. +# The tests lower them through SWARM_SCAN_FILE_LIMIT/_TOTAL_LIMIT. +FILE_LIMIT="${SWARM_SCAN_FILE_LIMIT:-$((256 * 1024 * 1024))}" +TOTAL_LIMIT="${SWARM_SCAN_TOTAL_LIMIT:-$((2 * 1024 * 1024 * 1024))}" + +fail() { + echo "preserve-scan: $*" >&2 + exit 3 +} + +case "$MODE" in + worktree|gitdir) ;; + *) fail "unknown mode: ${MODE}" ;; +esac +[ -d "$ROOT" ] || fail "not a directory: ${ROOT}" +[ -d "$OUT" ] || fail "not a directory: ${OUT}" +# Print the resolved form of directory $1. Command substitution would +# drop trailing newlines and turn "repo" into "repo", so the x keeps +# them; a path with a newline anywhere fails (the scan's lists are +# line-based). +canon() { + local r + r=$(cd -P "$1" 2>/dev/null && pwd -P && printf x) || return 1 + r="${r%x}" + r="${r%$'\n'}" + case "$r" in + *$'\n'*) return 1 ;; + esac + printf '%s' "$r" +} + +# Git records worktree paths resolved; compare against the same form. +ROOT=$(canon "$ROOT") || fail "cannot resolve ${2:-}, or it holds a newline" +_entries=$(ls -A "$OUT") || fail "cannot list ${OUT}" +[ -z "$_entries" ] || fail "output directory is not empty: ${OUT}" + +ERR="$OUT/.err" +REPORT="$OUT/report" +SCRATCH="$OUT/scratch.git" +: > "$REPORT" + +# Run git on git dir $1 with the settings every read here needs, and +# fail the scan on a non-zero exit or on anything it writes to stderr. +g() { + local dir="$1" + shift + if ! git --git-dir="$dir" -c safe.directory='*' \ + -c core.fsmonitor=false -c core.untrackedCache=false \ + -c advice.graftFileDeprecated=false --no-optional-locks \ + "$@" 2>"$ERR"; then + fail "git $* in ${dir} failed: $(cat "$ERR")" + fi + [ ! -s "$ERR" ] || fail "git $* in ${dir}: $(cat "$ERR")" +} + +# The same for a work tree $1 whose git dir Git finds itself. +gw() { + local dir="$1" + shift + if ! git -C "$dir" -c safe.directory='*' \ + -c core.fsmonitor=false -c core.untrackedCache=false \ + -c advice.graftFileDeprecated=false --no-optional-locks \ + "$@" 2>"$ERR"; then + fail "git $* in ${dir} failed: $(cat "$ERR")" + fi + [ ! -s "$ERR" ] || fail "git $* in ${dir}: $(cat "$ERR")" +} + +report() { + printf '%s\n' "$*" >> "$REPORT" +} + +# Print the size of file $1 in bytes. +fsize() { + wc -c < "$1" | tr -d ' ' +} + +# ---- Find the git dirs -------------------------------------------------- + +# A git dir is a directory with HEAD and either an objects/ directory +# (a repository) or a commondir file (a linked worktree's git dir). +# Every one under TOP is listed, at any depth, whatever its name (a +# submodule can be called logs or refs): module repos under modules/, +# worktree dirs under worktrees/, and modules of worktrees. HEAD may be +# a symlink (core.preferSymlinkRefs). The HEAD files Git keeps +# elsewhere (logs/HEAD) sit in directories that have neither objects/ +# nor commondir. +list_gitdirs() { + local top="$1" head d + find "$top" \( -type f -o -type l \) -name HEAD -print0 2>"$ERR" \ + > "$OUT/.heads" || fail "cannot list ${top}" + [ ! -s "$ERR" ] || fail "cannot list ${top}: $(cat "$ERR")" + while IFS= read -r -d '' head; do + d="${head%/HEAD}" + [ -d "$d/objects" ] || [ -f "$d/commondir" ] || continue + # The git dir list is line-based; a git dir whose path holds a + # newline is reported instead of listed (its name is shown with + # the newlines as \n). + case "$d" in + *$'\n'*) + report "block newline-path $(printf '%s' "${d#"$ROOT"/}" \ + | awk 'BEGIN { ORS = "\\n" } { print }')" + continue ;; + esac + printf '%s\n' "$d" + done < "$OUT/.heads" +} + +# ---- Worktree-mode checks ----------------------------------------------- + +TOP="" +if [ "$MODE" = worktree ]; then + # find prints one x per entry, whatever bytes the names hold. + _entries=$(find "$ROOT" -mindepth 1 -maxdepth 1 -exec printf x \;) \ + || fail "cannot list ${ROOT}" + if [ -z "$_entries" ]; then + report "empty" + exit 0 + fi + if [ -L "$ROOT/.git" ] || [ ! -e "$ROOT/.git" ]; then + report "block nogit files without a repository in ${ROOT}" + exit 0 + fi + [ -d "$ROOT/.git" ] || { + report "block gitfile ${ROOT}/.git is not a directory" + exit 0 + } + TOP="$ROOT/.git" +else + TOP="$ROOT" + [ -f "$TOP/HEAD" ] || fail "not a git dir: ${TOP}" +fi + +# TOP as a find -path pattern: *, ?, [ and \ in it match only themselves. +TOP_PAT=$(printf '%s' "$TOP" | sed 's/[][*?\\]/\\&/g') + +GITDIRS="$OUT/.gitdirs" +list_gitdirs "$TOP" > "$GITDIRS" +grep -qx "$TOP" "$GITDIRS" || printf '%s\n' "$TOP" >> "$GITDIRS" + +# Unsupported layouts that keep history outside these git dirs. +while IFS= read -r d; do + [ ! -f "$d/shallow" ] || report "block shallow ${d#"$ROOT"/}" + _lfs=$(git --git-dir="$d" -c safe.directory='*' config \ + --get lfs.storage 2>/dev/null || true) + [ -z "$_lfs" ] || report "block lfs-storage ${d#"$ROOT"/} ${_lfs}" +done < "$GITDIRS" + +# Print path $1 on one line, newlines shown as \n. +oneline() { + printf '%s' "$1" | awk 'BEGIN { ORS = "\\n" } { print }' \ + | sed 's/\\n$//' +} + +# Engine files and the deletions the context mode makes: in the +# workspace's own checkout ($2 = "."), where the engine writes those +# paths, entry XY PATH ($1, $3) is not work. In submodules and +# worktrees it is. +engine_entry() { + local xy="$1" rel="$2" path="$3" + [ "$rel" = "." ] || return 1 + case "$path" in + agent_logs|agent_logs/*|.claude/settings.local.json) return 0 ;; + esac + if [ "$xy" = " D" ]; then + case "$path" in + .claude/*) + [ "$CONTEXT" != none ] || return 0 + if [ "$CONTEXT" = slim ] && [ "$path" != .claude/CLAUDE.md ] + then + return 0 + fi ;; + esac + fi + return 1 +} + +# Work-tree content outside Git's history, for the checkout at $1 with +# git dir $3 ($2 is its path in the report). Both are passed +# explicitly, so a core.worktree setting cannot point status elsewhere. +# Status is read NUL-separated (-z), so paths come unquoted and whole. +check_checkout() { + local wt="$1" rel="$2" gd="$3" rec xy path + # No rename detection: a rename record would hide its new path from + # the filters below. + gw "$wt" --git-dir="$gd" --work-tree="$wt" status --porcelain=v1 -z \ + --no-renames --untracked-files=all --ignore-submodules=none \ + > "$OUT/.st" + while IFS= read -r -d '' rec; do + xy="${rec:0:2}" + path="${rec:3}" + engine_entry "$xy" "$rel" "$path" && continue + report "uncommitted dirty ${rel}: ${xy} $(oneline "$path")" + done < "$OUT/.st" + gw "$wt" --git-dir="$gd" --work-tree="$wt" status --porcelain=v1 -z \ + --no-renames --untracked-files=all --ignored=matching \ + --ignore-submodules=all > "$OUT/.st" + while IFS= read -r -d '' rec; do + [ "${rec:0:3}" = "!! " ] || continue + path="${rec:3}" + engine_entry "!!" "$rel" "$path" && continue + report "uncommitted ignored ${rel}: $(oneline "$path")" + done < "$OUT/.st" + gw "$wt" --git-dir="$gd" --work-tree="$wt" ls-files -v -z > "$OUT/.ls" + while IFS= read -r -d '' rec; do + case "${rec:0:1}" in + [a-z]|S) report "uncommitted hidden ${rel}: ${rec:0:1} $(oneline "${rec:2}")" ;; + esac + done < "$OUT/.ls" +} + +if [ "$MODE" = worktree ]; then + check_checkout "$ROOT" "." "$TOP" + # Every other .git below ROOT must be a submodule checkout whose git + # dir lives in ROOT/.git; anything else is a repository the bundle + # does not see. + find "$ROOT" -path "$TOP_PAT" -prune -o -name .git -print0 2>"$ERR" \ + > "$OUT/.embedded" || fail "cannot list ${ROOT}" + [ ! -s "$ERR" ] || fail "cannot list ${ROOT}: $(cat "$ERR")" + while IFS= read -r -d '' e; do + wt="${e%/.git}" + rel="${wt#"$ROOT"/}" + case "$rel" in + *$'\n'*) + report "block newline-path $(printf '%s' "$rel" \ + | awk 'BEGIN { ORS = "\\n" } { print }')" + continue ;; + esac + if [ -f "$e" ] && [ ! -L "$e" ]; then + target=$(sed -n 's/^gitdir: //p' "$e") + case "$target" in + /*) ;; + *) target="$wt/$target" ;; + esac + target=$(canon "$target") || target="" + case "$target" in + "$TOP"/worktrees/*) + # A linked worktree's own admin dir (one level below + # worktrees/, with its gitdir back-pointer) is + # checked with the others below. Anything deeper, + # such as a module repo of a linked worktree, is a + # submodule checkout and is checked here. + _rest="${target#"$TOP"/worktrees/}" + case "$_rest" in + */*) ;; + *) + if [ -f "$target/gitdir" ] \ + && grep -qxF "$target" "$GITDIRS"; then + # Only when its back-pointer names this + # checkout; a copied worktree dir points + # at another one and is not checked + # there. + bp=$(cat "$target/gitdir") + case "$bp" in + /*) ;; + *) bp="$target/$bp" ;; + esac + bpwt=$(canon "${bp%/.git}") || bpwt="" + wtc=$(canon "$wt") || wtc="-" + if [ "$bpwt" = "$wtc" ]; then + continue + fi + report "block mismatched-worktree ${rel}" + continue + fi ;; + esac + case "$_rest" in + */*) + if grep -qxF "$target" "$GITDIRS"; then + check_checkout "$wt" "$rel" "$target" + continue + fi ;; + esac ;; + "$TOP"/*) + check_checkout "$wt" "$rel" "$target" + continue ;; + esac + fi + report "block embedded ${rel}" + done < "$OUT/.embedded" + # A repository directory anywhere else below ROOT (a bare clone, a + # copied .git under another name) holds history the bundle does not + # carry: it blocks, whatever filters would apply to its files. + find "$ROOT" -path "$TOP_PAT" -prune -o \( -type f -o -type l \) \ + -name HEAD -print0 2>"$ERR" > "$OUT/.otherheads" \ + || fail "cannot list ${ROOT}" + [ ! -s "$ERR" ] || fail "cannot list ${ROOT}: $(cat "$ERR")" + while IFS= read -r -d '' h; do + r="${h%/HEAD}" + [ -d "$r/objects" ] || [ -f "$r/commondir" ] || continue + [ "${r##*/}" != .git ] || continue + rel="${r#"$ROOT"/}" + case "$rel" in + *$'\n'*) + report "block newline-path $(printf '%s' "$rel" \ + | awk 'BEGIN { ORS = "\\n" } { print }')" ;; + *) report "block embedded ${rel}" ;; + esac + done < "$OUT/.otherheads" + + # Linked worktrees: inside ROOT they are checked like any checkout; + # outside it (a container's /tmp) their files are not in the volume. + while IFS= read -r d; do + [ -f "$d/gitdir" ] || continue + wtgit=$(cat "$d/gitdir") + # The back-pointer is relative to this admin dir when the + # worktree was added with --relative-paths. + case "$wtgit" in + /*) ;; + *) wtgit="$d/$wtgit" ;; + esac + wt="${wtgit%/.git}" + if ! wt=$(canon "$wt"); then + report "block missing-worktree ${wtgit%/.git}" + continue + fi + case "$wt" in + "$ROOT"/*) check_checkout "$wt" "${wt#"$ROOT"/}" "$d" ;; + *) report "block external ${wt}" ;; + esac + done < "$GITDIRS" +fi + +# Unfinished operations, in every git dir. +while IFS= read -r d; do + for f in MERGE_HEAD CHERRY_PICK_HEAD REVERT_HEAD BISECT_LOG \ + rebase-merge rebase-apply sequencer; do + [ ! -e "$d/$f" ] || report "uncommitted operation ${d#"$ROOT"/}/${f}" + done + for f in "$d"/*AUTOSTASH*; do + [ ! -e "$f" ] || report "uncommitted operation ${f#"$ROOT"/}" + done +done < "$GITDIRS" + +# ---- Objects to keep ---------------------------------------------------- + +# The scratch repo uses the source's hash; every git dir must share it. +FORMAT=$(git --git-dir="$TOP" -c safe.directory='*' rev-parse \ + --show-object-format 2>/dev/null) || fail "cannot read the hash of ${TOP}" +while IFS= read -r d; do + f=$(git --git-dir="$d" -c safe.directory='*' rev-parse \ + --show-object-format 2>/dev/null) || fail "cannot read the hash of ${d}" + [ "$f" = "$FORMAT" ] || fail "${d} uses ${f}, ${TOP} uses ${FORMAT}" +done < "$GITDIRS" +git init -q --bare --object-format="$FORMAT" "$SCRATCH" \ + || fail "cannot create ${SCRATCH}" +: > "$SCRATCH/objects/info/alternates" +OBJECTS="$OUT/objects" +: > "$OBJECTS" +n=0 +while IFS= read -r d; do + n=$((n + 1)) + rel="${d#"$ROOT"/}" + [ "$d" != "$ROOT" ] || rel="." + report "gitdir ${n} ${rel}" + tips="$OUT/.tips.$n" + : > "$tips" + # The object store this git dir uses. + if [ -d "$d/objects" ]; then + store="$d" + else + common=$(cat "$d/commondir") + case "$common" in + /*) store="$common" ;; + *) store="$d/$common" ;; + esac + fi + store=$(canon "$store") || fail "no object store for ${d}" + grep -qxF "$store/objects" "$SCRATCH/objects/info/alternates" \ + || printf '%s\n' "$store/objects" >> "$SCRATCH/objects/info/alternates" + # Refs (only a repository has its own). + if [ -d "$d/objects" ]; then + g "$d" for-each-ref --format='%(objectname)' >> "$tips" + fi + # Every object id in HEAD, the reflogs, and the other files Git + # keeps directly in the git dir or in an operation's directory + # (ORIG_HEAD, MERGE_AUTOSTASH, rebase-merge/autostash, ...). + for f in "$d"/* "$d"/rebase-merge/* "$d"/rebase-apply/* \ + "$d"/sequencer/*; do + [ -f "$f" ] || continue + case "${f##*/}" in + index|packed-refs|config|description) continue ;; + esac + grep -aowE '[0-9a-f]{40}([0-9a-f]{24})?' "$f" >> "$tips" || true + done + if [ -d "$d/logs" ]; then + find "$d/logs" -type f -print0 2>"$ERR" > "$OUT/.logs" \ + || fail "cannot list ${d}/logs" + [ ! -s "$ERR" ] || fail "cannot list ${d}/logs: $(cat "$ERR")" + while IFS= read -r -d '' f; do + awk '{ print $1; print $2 }' "$f" >> "$tips" \ + || fail "cannot read ${f}" + done < "$OUT/.logs" + fi + # Keep the ids that name an object in this store. + sort -u "$tips" | grep -vE '^0+$' > "$tips.u" || true + g "$d" cat-file --batch-check='%(objectname) %(objecttype)' \ + < "$tips.u" > "$tips.c" + grep -v ' missing$' "$tips.c" | awk '{ print $1 }' > "$tips" || true + # Every object of a repository's own store that those tips do not + # reach becomes a tip too: objects held only by the index or by + # nothing at all. + if [ -d "$d/objects" ]; then + g "$d" cat-file --batch-all-objects --batch-check='%(objectname)' \ + > "$OUT/.all.$n" + sort -u "$OUT/.all.$n" > "$OUT/.all.$n.s" + g "$d" rev-list --objects --no-object-names --stdin < "$tips" \ + > "$OUT/.reach.$n" + sort -u "$OUT/.reach.$n" > "$OUT/.reach.$n.s" + comm -23 "$OUT/.all.$n.s" "$OUT/.reach.$n.s" >> "$tips" + cat "$OUT/.all.$n.s" >> "$OBJECTS" + fi + sort -u "$tips" -o "$tips" + k=0 + while IFS= read -r oid; do + k=$((k + 1)) + printf 'update refs/v/%s/t/%s %s\n' "$n" "$k" "$oid" + done < "$tips" > "$OUT/.upd.$n" + git --git-dir="$SCRATCH" update-ref --stdin < "$OUT/.upd.$n" 2>"$ERR" \ + || fail "cannot record the objects of ${d}: $(cat "$ERR")" + manifest=$(git --git-dir="$SCRATCH" hash-object -w --stdin \ + < "$tips") || fail "cannot write the manifest of ${d}" + git --git-dir="$SCRATCH" update-ref "refs/v/$n/manifest" "$manifest" + printf '%s\n' "$d" > "$OUT/.gitdir.$n" +done < "$GITDIRS" +sort -u "$OBJECTS" -o "$OBJECTS" + +# ---- Files of the git dirs ---------------------------------------------- + +# Every file and symlink under TOP is stored as a blob, except the +# .pack and .idx files of a pack whose index Git verifies and whose +# objects are all in the inventory: verify-pack recomputes every +# object's hash and checks both files' trailing checksums, so their +# bytes are exactly what the bundle carries. Loose object files are +# stored like any other file (a loose file can hold bytes after its +# zlib stream that no reader returns), and so are a pack's .rev, +# .mtimes and .bitmap files. +EXEMPT="$OUT/.exempt" +: > "$EXEMPT" +regular() { + [ -f "$1" ] && [ ! -L "$1" ] +} +while IFS= read -r d; do + [ -d "$d/objects/pack" ] && [ ! -L "$d/objects/pack" ] || continue + for p in "$d"/objects/pack/pack-*.pack; do + regular "$p" || continue + base="${p%.pack}" + hex="${base##*/pack-}" + case "$hex" in + *[!0-9a-f]*) continue ;; + esac + [ "${#hex}" -eq 40 ] || [ "${#hex}" -eq 64 ] || continue + regular "$base.idx" || continue + # Run in the source repo, so the pack is read with its hash. + git --git-dir="$d" -c safe.directory='*' verify-pack "$base.idx" \ + >/dev/null 2>&1 || continue + git show-index --object-format="$FORMAT" < "$base.idx" \ + > "$OUT/.pidx.raw" 2>/dev/null || continue + awk '{ print $2 }' "$OUT/.pidx.raw" | sort -u > "$OUT/.pidx" + [ -s "$OUT/.pidx" ] || continue + comm -23 "$OUT/.pidx" "$OBJECTS" > "$OUT/.pmiss" + [ ! -s "$OUT/.pmiss" ] || continue + printf '%s\n%s\n' "$base.pack" "$base.idx" >> "$EXEMPT" + done +done < "$GITDIRS" +sort -u "$EXEMPT" -o "$EXEMPT" + +# Paths are read NUL-separated, so any byte in a name survives. A path +# with a newline in it is never exempt (exempt names are hex). +: > "$OUT/.flist" +: > "$OUT/.fnl" +find "$TOP" ! -type d -print0 2>"$ERR" > "$OUT/.flist0" \ + || fail "cannot list ${TOP}" +[ ! -s "$ERR" ] || fail "cannot list ${TOP}: $(cat "$ERR")" +while IFS= read -r -d '' f; do + case "$f" in + *$'\n'*) printf '%s\0' "$f" >> "$OUT/.fnl" ;; + *) printf '%s\n' "$f" >> "$OUT/.flist" ;; + esac +done < "$OUT/.flist0" +sort "$OUT/.flist" -o "$OUT/.flist" +comm -23 "$OUT/.flist" "$EXEMPT" | tr '\n' '\0' > "$OUT/.fkeep" +cat "$OUT/.fnl" >> "$OUT/.fkeep" +: > "$OUT/.files" +: > "$OUT/.fupd" +k=0 +total=0 +while IFS= read -r -d '' f; do + rel="${f#"$TOP"/}" + if [ -L "$f" ]; then + # readlink -n prints the target as is; the x keeps a trailing + # newline of the target from command substitution. + target=$(readlink -n "$f" && printf x) || fail "cannot read ${f}" + target="${target%x}" + k=$((k + 1)) + blob=$(printf '%s' "$target" \ + | git --git-dir="$SCRATCH" hash-object -w --stdin) \ + || fail "cannot store ${f}" + printf 'update refs/v/0/f/%s %s\n' "$k" "$blob" >> "$OUT/.fupd" + printf '%s 120000 0 %s\0' "$k" "$rel" >> "$OUT/.files" + continue + fi + [ -f "$f" ] || fail "special file in a git dir: ${f}" + size=$(fsize "$f") || fail "cannot read ${f}" + if [ "$size" -gt "$FILE_LIMIT" ] \ + || [ $((total + size)) -gt "$TOTAL_LIMIT" ]; then + report "uncommitted large-file ${f#"$ROOT"/} ${size}" + continue + fi + total=$((total + size)) + k=$((k + 1)) + blob=$(git --git-dir="$SCRATCH" hash-object -w --no-filters --stdin \ + < "$f") || fail "cannot store ${f}" + mode=100644 + [ ! -x "$f" ] || mode=100755 + printf 'update refs/v/0/f/%s %s\n' "$k" "$blob" >> "$OUT/.fupd" + printf '%s %s %s %s\0' "$k" "$mode" "$size" "$rel" >> "$OUT/.files" +done < "$OUT/.fkeep" +git --git-dir="$SCRATCH" update-ref --stdin < "$OUT/.fupd" 2>"$ERR" \ + || fail "cannot record the files of ${TOP}: $(cat "$ERR")" +files=$(git --git-dir="$SCRATCH" hash-object -w --stdin < "$OUT/.files") \ + || fail "cannot write the file list of ${TOP}" +git --git-dir="$SCRATCH" update-ref refs/v/0/files "$files" + +# ---- Bundle ------------------------------------------------------------- + +git --git-dir="$SCRATCH" -c safe.directory='*' bundle create \ + "$OUT/bundle" --all >/dev/null 2>"$ERR" \ + || fail "cannot create the bundle: $(cat "$ERR")" +exit 0 diff --git a/lib/preserve.sh b/lib/preserve.sh index 93848d6..e5dfb8e 100644 --- a/lib/preserve.sh +++ b/lib/preserve.sh @@ -146,3 +146,311 @@ swarm_preservation_explain() { "repo with:" >&2 echo " git --git-dir=${quoted} update-ref -d " >&2 } + +# ---- Recovery snapshots ------------------------------------------------- +# +# Before cleanup deletes a repository location (the bare repo, a +# mirror, a workspace volume), lib/preserve-scan.sh packs everything Git +# keeps there into a bundle, and swarm_recovery_import copies that +# bundle into the checkout under +# refs/swarm-recovery//// +# so no commit, object or git-dir file of the location has its only +# copy in what is deleted. The engine never deletes recovery refs. + +SWARM_RECOVERY_NS="refs/swarm-recovery" + +# Print a new snapshot id: UTC time and 8 random hex digits. +swarm_recovery_snapshot_id() { + local rand + rand=$(od -An -N4 -tx1 /dev/urandom 2>/dev/null | tr -d ' \n') \ + || return 1 + [ "${#rand}" -eq 8 ] || return 1 + printf '%s-%s' "$(date -u +%Y%m%dT%H%M%SZ)" "$rand" +} + +# Git in the checkout for recovery work: no environment redirection, +# no hooks or fsmonitor, no lazy fetches from promisor remotes. +swarm_recovery_git() { + env -u GIT_DIR -u GIT_WORK_TREE -u GIT_COMMON_DIR -u GIT_INDEX_FILE \ + -u GIT_OBJECT_DIRECTORY -u GIT_ALTERNATE_OBJECT_DIRECTORIES \ + -u GIT_NAMESPACE -u GIT_CONFIG_PARAMETERS -u GIT_CONFIG_COUNT \ + GIT_NO_LAZY_FETCH=1 GIT_NO_REPLACE_OBJECTS=1 GIT_TERMINAL_PROMPT=0 \ + GIT_GRAFT_FILE=/dev/null \ + git -c core.fsmonitor=false -c core.hooksPath=/dev/null \ + -c fetch.prune=false -c fetch.pruneTags=false \ + -c advice.graftFileDeprecated=false "$@" +} + +# Print the device:inode of PATH. GNU stat takes -c; BSD stat takes +# -f, and GNU stat -f means something else, so the form is chosen once. +swarm_dev_ino() { + if [ -z "${SWARM_STAT_FORM:-}" ]; then + if stat -c '%d:%i' / >/dev/null 2>&1; then + SWARM_STAT_FORM=-c + else + SWARM_STAT_FORM=-f + fi + fi + stat "$SWARM_STAT_FORM" '%d:%i' "$1" +} + +# Succeed when the checkout at ROOT keeps every object in its own +# store, so objects imported there survive the deletion of TARGETS +# (directories cleanup is about to delete; may be empty). Refuses a +# shallow or partial clone, alternates, symlinks in the object store, +# an object store inside RUNTIME or a target, and a target that shares +# a directory or pack file with the object store (bind mounts). +swarm_checkout_storage_ok() { + local root="$1" runtime="$2" + shift 2 + local common objects canon t tcanon + common=$(swarm_recovery_git -C "$root" rev-parse --git-common-dir \ + 2>/dev/null) || { + echo "ERROR: cannot read the git dir of ${root}." >&2 + return 1 + } + case "$common" in + /*) ;; + *) common="$root/$common" ;; + esac + objects="$common/objects" + if [ "$(swarm_recovery_git -C "$root" rev-parse \ + --is-shallow-repository 2>/dev/null)" != false ]; then + echo "ERROR: ${root} is a shallow clone; recovery needs full" \ + "history (git fetch --unshallow)." >&2 + return 1 + fi + local names name val partial="" + partial=$(swarm_recovery_git -C "$root" config --get \ + extensions.partialClone 2>/dev/null) || partial="" + names=$(swarm_recovery_git -C "$root" config --name-only \ + --get-regexp '^remote\..*\.promisor$' 2>/dev/null) || names="" + for name in $names; do + val=$(swarm_recovery_git -C "$root" config --type=bool --get \ + "$name" 2>/dev/null) || val=true + [ "$val" = false ] || partial=yes + done + if [ -n "$partial" ]; then + echo "ERROR: ${root} is a partial clone; recovery needs every" \ + "object locally." >&2 + return 1 + fi + if [ -s "$objects/info/alternates" ]; then + echo "ERROR: ${root} borrows objects (${objects}/info/alternates)." \ + "Run git repack -a -d there, then remove that file." >&2 + return 1 + fi + # Recovery refs live in the same git dir; they must not be stored + # through a link into what cleanup deletes either. + local links + links=$(find "$objects" "$common/refs" -type l -print) || { + echo "ERROR: cannot list ${objects} and ${common}/refs." >&2 + return 1 + } + if [ -n "$links" ] || [ -L "$objects" ] || [ -L "$common" ] \ + || [ -L "$common/refs" ] || [ -L "$common/packed-refs" ] \ + || [ -e "$common/reftable" ]; then + echo "ERROR: the object store or refs of ${root} contain" \ + "symlinks (or use reftable); make them plain directories" \ + "first." >&2 + return 1 + fi + canon=$(cd -P "$objects" 2>/dev/null && pwd -P) || { + echo "ERROR: cannot resolve ${objects}." >&2 + return 1 + } + for t in "$runtime" "$@"; do + [ -e "$t" ] || continue + tcanon=$(cd -P "$t" 2>/dev/null && pwd -P) || { + echo "ERROR: cannot resolve ${t}." >&2 + return 1 + } + case "$canon/" in + "$tcanon"/*) + echo "ERROR: the object store of ${root} is inside ${t}." >&2 + return 1 ;; + esac + case "$tcanon/" in + "$canon"/*) + echo "ERROR: ${t} is inside the object store of ${root}." >&2 + return 1 ;; + esac + done + [ "$#" -gt 0 ] || return 0 + # The same directory or pack file seen through a bind mount has the + # same device and inode under both paths. + local tmp p rc=0 + tmp=$(mktemp -d) || return 1 + { + find "$objects" "$common/refs" -type d -print || rc=1 + for p in "$objects/pack" "$objects/info"; do + [ ! -d "$p" ] || find "$p" -type f -print || rc=1 + done + [ ! -f "$common/packed-refs" ] || printf '%s\n' "$common/packed-refs" + } > "$tmp/mine.paths" + for t in "$@"; do + [ -d "$t" ] || continue + find "$t" \( -type d -o -path '*/objects/pack/*' \ + -o -path '*/objects/info/*' \) -print >> "$tmp/theirs.paths" \ + || rc=1 + done + : >> "$tmp/theirs.paths" + for p in mine theirs; do + while IFS= read -r t; do + swarm_dev_ino "$t" || rc=1 + done < "$tmp/$p.paths" > "$tmp/$p.ids" || rc=1 + sort -u "$tmp/$p.ids" -o "$tmp/$p.ids" || rc=1 + done + if [ "$rc" -ne 0 ]; then + rm -rf "$tmp" + echo "ERROR: cannot list the object store of ${root} or what" \ + "cleanup deletes." >&2 + return 1 + fi + comm -12 "$tmp/mine.ids" "$tmp/theirs.ids" > "$tmp/both" || rc=1 + if [ "$rc" -ne 0 ] || [ -s "$tmp/both" ]; then + rm -rf "$tmp" + echo "ERROR: the object store of ${root} shares files with what" \ + "cleanup deletes (a bind mount?)." >&2 + return 1 + fi + rm -rf "$tmp" +} + +# Copy the scan output in DIR (from lib/preserve-scan.sh) into the +# checkout at ROOT under PREFIX/ and verify it: every ref of the bundle +# exists under PREFIX/ with the same object, and every object of the +# inventory, and everything the new refs reach, is in the checkout. +# PREFIX must hold no refs yet. Fails without changing anything else. +swarm_recovery_import() { + local root="$1" dir="$2" prefix="$3" heads got + case "$prefix" in + "$SWARM_RECOVERY_NS"/*) ;; + *) echo "ERROR: not a recovery prefix: ${prefix}" >&2; return 1 ;; + esac + if ! got=$(swarm_recovery_git -C "$root" for-each-ref \ + --format='%(refname)' "$prefix/" 2>/dev/null); then + echo "ERROR: cannot read the refs of ${root}." >&2 + return 1 + fi + if [ -n "$got" ]; then + echo "ERROR: ${prefix} already holds refs in ${root}." >&2 + return 1 + fi + if ! swarm_recovery_git -C "$root" bundle verify "$dir/bundle" \ + >/dev/null 2>&1; then + echo "ERROR: the bundle in ${dir} does not verify." >&2 + return 1 + fi + heads=$(swarm_recovery_git -C "$root" bundle list-heads "$dir/bundle" \ + 2>/dev/null) || { + echo "ERROR: cannot list the bundle in ${dir}." >&2 + return 1 + } + if ! swarm_recovery_git -C "$root" fetch --quiet --no-tags --no-prune \ + --no-write-fetch-head --no-recurse-submodules --no-auto-gc \ + "$dir/bundle" "refs/v/*:${prefix}/*" >/dev/null 2>&1; then + echo "ERROR: cannot import the bundle in ${dir} into ${root}." >&2 + return 1 + fi + got=$(swarm_recovery_git -C "$root" for-each-ref \ + --format='%(objectname) %(refname)' "$prefix/" 2>/dev/null \ + | sed "s|^\([0-9a-f]*\) ${prefix}/|\1 refs/v/|" | sort) || got="" + if [ "$got" != "$(printf '%s\n' "$heads" | sort)" ]; then + echo "ERROR: the refs imported into ${prefix} do not match the" \ + "bundle." >&2 + return 1 + fi + # Every object of the inventory must be in the checkout. The + # output goes to a file first, so no early exit hides a failure. + local tmp + tmp=$(mktemp -d) || return 1 + if [ ! -f "$dir/objects" ] \ + || ! swarm_recovery_git -C "$root" cat-file --batch-check \ + < "$dir/objects" > "$tmp/check" 2>/dev/null \ + || [ "$(wc -l < "$tmp/check")" -ne "$(wc -l < "$dir/objects")" ] \ + || grep ' missing$' "$tmp/check" > "$tmp/missing"; then + rm -rf "$tmp" + echo "ERROR: objects of the inventory are missing in ${root}." >&2 + return 1 + fi + if ! swarm_recovery_git -C "$root" for-each-ref --format='%(objectname)' \ + "$prefix/" > "$tmp/tips" 2>/dev/null \ + || ! swarm_recovery_git -C "$root" rev-list --objects \ + --no-object-names --missing=error --stdin \ + < "$tmp/tips" > "$tmp/reach" 2>/dev/null; then + rm -rf "$tmp" + echo "ERROR: history under ${prefix} is incomplete in ${root}." >&2 + return 1 + fi + # Every inventoried object must hang off the new refs. + if ! sort -u "$tmp/reach" > "$tmp/want" \ + || ! comm -23 "$dir/objects" "$tmp/want" > "$tmp/outside"; then + rm -rf "$tmp" + return 1 + fi + if [ -s "$tmp/outside" ]; then + rm -rf "$tmp" + echo "ERROR: objects of the inventory are not under ${prefix}." >&2 + return 1 + fi + # Presence is checked here; content is checked once for all imports + # by swarm_recovery_verify_checkout, before anything is deleted. + rm -rf "$tmp" +} + +# Succeed when every object in the checkout at ROOT has the content its +# name says. Fetch skips objects the checkout already has, and Git does +# not re-hash objects it reads, so an object stored under the wrong name +# (or two objects with swapped contents) would pass every presence +# check. git fsck --full re-hashes every loose and packed object. +# Cleanup runs this once after its imports and before it deletes +# anything. It reads the whole object store, so it takes a while on a +# large repository. +swarm_recovery_verify_checkout() { + local root="$1" err + err=$(mktemp) || return 1 + if ! swarm_recovery_git -C "$root" fsck --full --no-dangling \ + --no-progress >/dev/null 2>"$err"; then + echo "ERROR: git fsck found problems in ${root}; recovery refs" \ + "there cannot be trusted:" >&2 + sed 's/^/ /' "$err" | head -20 >&2 + rm -f "$err" + return 1 + fi + rm -f "$err" +} + +# Scan the host directory PATH (a bare repo or a mirror: MODE gitdir) +# into the empty directory OUT. Returns the scanner's status. +swarm_scan_host() { + local path="$1" out="$2" + bash "$(dirname "${BASH_SOURCE[0]}")/preserve-scan.sh" gitdir "$path" - \ + "$out" +} + +# Scan workspace volume VOL with image IMAGE (context CONTEXT) into the +# empty directory OUT: the scanner runs in a throwaway container with +# the volume read-only and no network, as root so it can read every +# file, and streams its results back as a tar archive. +swarm_scan_volume() { + local vol="$1" image="$2" context="$3" out="$4" rc=0 + docker run --rm -i --pull never --network none --user 0 \ + --entrypoint bash \ + --mount "type=volume,src=${vol},dst=/workspace,readonly,volume-nocopy" \ + "$image" -c ' + cat > /tmp/scan.sh || exit 3 + mkdir /tmp/out || exit 3 + bash /tmp/scan.sh worktree /workspace "$1" /tmp/out >&2 || exit $? + cd /tmp/out || exit 3 + set -- report + [ ! -f bundle ] || set -- "$@" bundle + [ ! -f objects ] || set -- "$@" objects + tar -cf - "$@" || exit 3 + ' scan "$context" \ + < "$(dirname "${BASH_SOURCE[0]}")/preserve-scan.sh" \ + > "$out/scan.tar" || rc=$? + [ "$rc" -eq 0 ] || return 3 + tar -C "$out" -xf "$out/scan.tar" 2>/dev/null || return 3 + rm -f "$out/scan.tar" +} diff --git a/tests/test_preserve.sh b/tests/test_preserve.sh index 955a324..c8cc5af 100755 --- a/tests/test_preserve.sh +++ b/tests/test_preserve.sh @@ -216,6 +216,722 @@ msg=$(printf '%s\n' "refs/heads/agent-work 0123456789abcdef" \ assert_eq "the explicit delete command is shown" "1" \ "$(printf '%s\n' "$msg" | grep -c 'update-ref -d ' || true)" +# ============================================================ +echo "" +echo "=== 9. The scanner keeps every object and git-dir file ===" + +SCAN="$REPO_ROOT/lib/preserve-scan.sh" +# Scan into a new directory; print its path. Extra args: env settings. +scan() { + local mode="$1" root="$2" ctx="$3" out + out=$(mktemp -d "$TMPDIR/scan.XXXXXX") + shift 3 + env "$@" bash "$SCAN" "$mode" "$root" "$ctx" "$out" >/dev/null 2>&1 \ + || echo "rc=$?" > "$out/.rc" + printf '%s' "$out" +} +# Import scan output into a fresh repository; print its path. +imported() { + local dst + dst=$(mktemp -d "$TMPDIR/dst.XXXXXX") + git init -q "$dst" + git -C "$dst" fetch -q --no-tags "$1/bundle" 'refs/v/*:refs/r/*' + printf '%s' "$dst" +} +has_line() { + grep -cF -- "$2" "$1/report" || true +} + +W="$TMPDIR/ws" +git init -q "$W" +echo a > "$W/a" +echo 'ign*' > "$W/.gitignore" +g -C "$W" add . +g -C "$W" commit -q -m one +echo b > "$W/b" +g -C "$W" add b +g -C "$W" commit -q -m two +LOST=$(git -C "$W" rev-parse HEAD) +git -C "$W" reset -q --hard HEAD~1 +echo wip > "$W/a" +STASHED=$(g -C "$W" stash create) +git -C "$W" checkout -q -- a +git -C "$W" reflog expire --expire=now --all +echo blob-only | git -C "$W" hash-object -w --stdin > "$TMPDIR/blobid" +echo note > "$W/.git/objects/info/rescue.patch" +printf '#!/bin/sh\necho hi\n' > "$W/.git/hooks/pre-commit" +chmod +x "$W/.git/hooks/pre-commit" +out=$(scan worktree "$W" full) +assert_eq "a clean workspace scans" "" "$(cat "$out/.rc" 2>/dev/null)" +assert_eq "a clean workspace reports nothing to keep or discard" "0" \ + "$(grep -cE '^(block|uncommitted) ' "$out/report" || true)" +dst=$(imported "$out") +assert_eq "a commit only the expired reflog knew is kept" "commit" \ + "$(git -C "$dst" cat-file -t "$LOST" 2>/dev/null)" +assert_eq "a stash create result no ref names is kept" "commit" \ + "$(git -C "$dst" cat-file -t "$STASHED" 2>/dev/null)" +assert_eq "a blob nothing references is kept" "blob" \ + "$(git -C "$dst" cat-file -t "$(cat "$TMPDIR/blobid")" 2>/dev/null)" +assert_eq "every object of the inventory is in the bundle" "0" \ + "$(git -C "$dst" cat-file --batch-check < "$out/objects" \ + | grep -c ' missing$' || true)" +_files=$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n') +for _f in objects/info/rescue.patch hooks/pre-commit config index \ + ORIG_HEAD; do + assert_eq "git-dir file ${_f} is stored" "1" \ + "$(printf '%s\n' "$_files" | awk -v f="$_f" '$4 == f' | wc -l \ + | tr -d ' ')" +done +rm -f "$W/.git/hooks/pre-commit" +_n=$(printf '%s\n' "$_files" \ + | awk '$4 == "objects/info/rescue.patch" { print $1 }') +assert_eq "a stored file has its content" "note" \ + "$(git -C "$dst" cat-file -p "refs/r/0/f/$_n")" +assert_eq "loose object files are stored as files too" "true" \ + "$([ "$(printf '%s\n' "$_files" \ + | awk '$4 ~ /^objects\/[0-9a-f][0-9a-f]\//' | wc -l)" -gt 0 ] \ + && echo true || echo false)" + +# Content outside Git's history is reported, not packed. +echo dirty >> "$W/a" +echo x > "$W/ignored.txt" +echo u > "$W/untracked" +git -C "$W" update-index --skip-worktree .gitignore +mkdir -p "$W/agent_logs" "$W/.claude" +echo log > "$W/agent_logs/x.log" +out=$(scan worktree "$W" full) +assert_eq "a modified file is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted dirty .: M a')" +assert_eq "an untracked file is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted dirty .: ?? untracked')" +assert_eq "an ignored file is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted ignored .: ignored.txt')" +assert_eq "a skip-worktree entry is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted hidden .: S .gitignore')" +assert_eq "agent logs are engine files" "0" \ + "$(has_line "$out" 'agent_logs')" +git -C "$W" update-index --no-skip-worktree .gitignore +git -C "$W" checkout -q -- a +rm -rf "$W/ignored.txt" "$W/untracked" "$W/agent_logs" + +# Paths Git would quote (spaces) are matched whole against the engine +# paths; other quoted paths are reported whole. +mkdir -p "$W/agent_logs" +echo l > "$W/agent_logs/a b.log" +echo w > "$W/my file.txt" +out=$(scan worktree "$W" full) +assert_eq "an agent log with a space in its name is an engine file" "0" \ + "$(has_line "$out" 'a b.log')" +assert_eq "a file with a space in its name is reported unquoted" "1" \ + "$(has_line "$out" 'uncommitted dirty .: ?? my file.txt')" +rm -rf "$W/agent_logs" "$W/my file.txt" + +# Context modes: the deletions under .claude the engine makes are not +# work; anything else there is. +mkdir -p "$W/.claude" +echo c > "$W/.claude/CLAUDE.md" +echo s > "$W/.claude/skill.md" +g -C "$W" add .claude +g -C "$W" commit -q -m claude +echo q > "$W/.claude/a b.md" +g -C "$W" add ".claude/a b.md" +g -C "$W" commit -q -m spaced +rm "$W/.claude/CLAUDE.md" "$W/.claude/skill.md" "$W/.claude/a b.md" +assert_eq "context none: deletions under .claude are not work" "0" \ + "$(has_line "$(scan worktree "$W" none)" 'uncommitted dirty')" +assert_eq "context slim: deleting CLAUDE.md is work" "1" \ + "$(has_line "$(scan worktree "$W" slim)" '.claude/CLAUDE.md')" +assert_eq "context full: every deletion is work" "3" \ + "$(has_line "$(scan worktree "$W" full)" 'uncommitted dirty .: D')" +git -C "$W" checkout -q -- .claude + +# Unfinished operations, wherever Git keeps their state. +mkdir "$W/.git/rebase-merge" +echo "$LOST" > "$W/.git/rebase-merge/autostash" +out=$(scan worktree "$W" full) +assert_eq "a paused rebase is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted operation .git/rebase-merge')" +rm -rf "$W/.git/rebase-merge" +echo "$STASHED" > "$W/.git/MERGE_AUTOSTASH" +out=$(scan worktree "$W" full) +assert_eq "a merge autostash is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted operation .git/MERGE_AUTOSTASH')" +rm -f "$W/.git/MERGE_AUTOSTASH" + +# A file above the size limit is uncommitted instead of stored. +head -c 2048 /dev/zero > "$W/.git/big.bin" +out=$(scan worktree "$W" full SWARM_SCAN_FILE_LIMIT=1024) +assert_eq "a large git-dir file is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted large-file .git/big.bin 2048')" +rm -f "$W/.git/big.bin" + +# A pack without its index cannot be proven to hold known objects, so +# its bytes are stored. +mkdir -p "$W/.git/objects/pack" +_hex=0123456789abcdef0123456789abcdef01234567 +echo stray > "$W/.git/objects/pack/pack-$_hex.pack" +out=$(scan worktree "$W" full) +dst=$(imported "$out") +assert_eq "a pack without an index is stored as a file" "1" \ + "$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/pack/pack-$_hex.pack" '$4 == f' | wc -l \ + | tr -d ' ')" +rm -f "$W/.git/objects/pack/pack-$_hex.pack" + +# Repository layouts the bundle cannot carry block. +E="$TMPDIR/empty-ws" +mkdir -p "$E" +assert_eq "an empty volume is empty" "empty" \ + "$(cat "$(scan worktree "$E" full)/report")" +echo f > "$E/file" +assert_eq "files without a repository block" "1" \ + "$(has_line "$(scan worktree "$E" full)" 'block nogit')" +E2="$TMPDIR/newline-only" +mkdir -p "$E2" +: > "$E2/"$'\n' +assert_eq "a volume holding only a file named by a newline is not empty" \ + "1" "$(has_line "$(scan worktree "$E2" full)" 'block nogit')" +# A path ending in a newline is not mistaken for its neighbour without +# one: the scan fails instead. +NLR="$TMPDIR/nlpair" +mkdir -p "$NLR" +git init -q "$NLR/repo" +g -C "$NLR/repo" commit -q --allow-empty -m plain +git init -q "$NLR/repo"$'\n' +_nlout=$(scan worktree "$NLR/repo"$'\n' full) +assert_eq "a root ending in a newline fails the scan" "rc=3" \ + "$(cat "$_nlout/.rc" 2>/dev/null)" + +# A root whose path holds find pattern characters still finds nested +# repositories (the prune of its .git matches only that .git). +GL="$TMPDIR/glob*root" +git init -q "$GL" +g -C "$GL" commit -q --allow-empty -m g +mkdir -p "$GL/agent_logs" +# Unescaped, ".../glob*root/.git" would also match this repo's .git. +git init -q "$GL/agent_logs/root" +assert_eq "a nested repo under a root with * in its path blocks" "1" \ + "$(has_line "$(scan worktree "$GL" full)" 'block embedded agent_logs/root')" + +# Repositories under other names inside the workspace block, also under +# engine paths. +git init -q --bare "$W/cache.git" +mkdir -p "$W/agent_logs" +git init -q --bare "$W/agent_logs/x.git" +out=$(scan worktree "$W" full) +assert_eq "a bare repo in the workspace blocks" "1" \ + "$(has_line "$out" 'block embedded cache.git')" +assert_eq "a bare repo under agent_logs blocks" "1" \ + "$(has_line "$out" 'block embedded agent_logs/x.git')" +rm -rf "$W/cache.git" "$W/agent_logs" + +git init -q "$W/inner" +assert_eq "an embedded repository blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block embedded inner')" +rm -rf "$W/inner" +git -C "$W" worktree add -q "$TMPDIR/outside-wt" 2>/dev/null +assert_eq "a linked worktree outside the volume blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" \ + "block external $(cd -P "$TMPDIR" && pwd -P)/outside-wt")" +git -C "$W" worktree remove --force "$TMPDIR/outside-wt" +git -C "$W" config lfs.storage /elsewhere +assert_eq "an lfs.storage setting blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block lfs-storage .git /elsewhere')" +git -C "$W" config --unset lfs.storage + +# Module repositories are found at any depth, initialized or not. +SUB="$TMPDIR/subsrc" +git init -q "$SUB" +g -C "$SUB" commit -q --allow-empty -m sub +g -C "$W" -c protocol.file.allow=always submodule add -q "$SUB" lib \ + >/dev/null 2>&1 +g -C "$W" commit -q -m add-sub +g -C "$W/lib" commit -q --allow-empty -m "detached sub work" +SUBWORK=$(git -C "$W/lib" rev-parse HEAD) +git -C "$W/lib" checkout -q HEAD~1 +git -C "$W" submodule deinit -q -f lib +out=$(scan worktree "$W" full) +assert_eq "a deinitialized module repo is a git dir" "1" \ + "$(grep -cE '^gitdir [0-9]+ \.git/modules/lib$' "$out/report" || true)" +assert_eq "its detached commit is kept" "commit" \ + "$(git -C "$(imported "$out")" cat-file -t "$SUBWORK" 2>/dev/null)" + +# A core.worktree setting cannot point status at another directory. +mkdir -p "$TMPDIR/clean-elsewhere" +echo dirty-again >> "$W/.gitmodules" +git -C "$W" config core.worktree "$TMPDIR/clean-elsewhere" +assert_eq "core.worktree does not hide dirty files" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'uncommitted dirty .: M .gitmodules')" +git -C "$W" config --unset core.worktree +git -C "$W" checkout -q -- .gitmodules + +# A graft that makes an unreached commit look reached does not drop it. +GRAFTED=$(g -C "$W" commit-tree -m grafted "$(git -C "$W" rev-parse 'HEAD^{tree}')") +echo "$(git -C "$W" rev-parse HEAD) $GRAFTED" > "$W/.git/info/grafts" +out=$(scan worktree "$W" full) +assert_eq "a grafted parent is still kept" "commit" \ + "$(git -C "$(imported "$out")" cat-file -t "$GRAFTED" 2>/dev/null)" +rm -f "$W/.git/info/grafts" + +# A loose file shadowed by a packed copy of the same object is never +# read by Git, so its bytes are stored as a file. +git -C "$W" repack -q -a -d +_shadow=$(git -C "$W" rev-parse HEAD) +_sdir="$W/.git/objects/${_shadow:0:2}" +mkdir -p "$_sdir" +echo "not the packed bytes" > "$_sdir/${_shadow:2}" +out=$(scan worktree "$W" full) +assert_eq "a loose file shadowed by a pack is stored" "1" \ + "$(git -C "$(imported "$out")" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/${_shadow:0:2}/${_shadow:2}" '$4 == f' | wc -l \ + | tr -d ' ')" +rm -f "$_sdir/${_shadow:2}" + +# The same, when the other copy is in an alternate's pack. +ALT="$TMPDIR/alt-src" +git init -q "$ALT" +echo alt-content > "$ALT/f" +g -C "$ALT" add f +g -C "$ALT" commit -q -m alt +git -C "$ALT" repack -q -a -d +_alt=$(git -C "$ALT" rev-parse HEAD) +echo "$ALT/.git/objects" > "$W/.git/objects/info/alternates" +mkdir -p "$W/.git/objects/${_alt:0:2}" +echo "other bytes" > "$W/.git/objects/${_alt:0:2}/${_alt:2}" +out=$(scan worktree "$W" full) +assert_eq "a loose file shadowed by an alternate's pack is stored" "1" \ + "$(git -C "$(imported "$out")" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/${_alt:0:2}/${_alt:2}" '$4 == f' | wc -l \ + | tr -d ' ')" +rm -f "$W/.git/objects/${_alt:0:2}/${_alt:2}" "$W/.git/objects/info/alternates" + +# Two loose files with swapped contents: each name exists, but neither +# file holds its own object, so both are stored as files. +_ba=$(printf 'swap a\n' | git -C "$W" hash-object -w --stdin) +_bb=$(printf 'swap b\n' | git -C "$W" hash-object -w --stdin) +git -C "$W" update-ref refs/keep/a "$_ba" +git -C "$W" update-ref refs/keep/b "$_bb" +git -C "$W" repack -q -a -d +mkdir -p "$W/.git/objects/${_ba:0:2}" "$W/.git/objects/${_bb:0:2}" +printf 'swap b\n' | git -C "$TMPDIR/subsrc" hash-object --stdin -w >/dev/null +cp "$TMPDIR/subsrc/.git/objects/${_bb:0:2}/${_bb:2}" "$TMPDIR/swap.b" +printf 'swap a\n' | git -C "$TMPDIR/subsrc" hash-object --stdin -w >/dev/null +cp "$TMPDIR/subsrc/.git/objects/${_ba:0:2}/${_ba:2}" "$TMPDIR/swap.a" +cp "$TMPDIR/swap.b" "$W/.git/objects/${_ba:0:2}/${_ba:2}" +cp "$TMPDIR/swap.a" "$W/.git/objects/${_bb:0:2}/${_bb:2}" +out=$(scan worktree "$W" full) +assert_eq "loose files with swapped contents are both stored" "2" \ + "$(git -C "$(imported "$out")" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v a="objects/${_ba:0:2}/${_ba:2}" \ + -v b="objects/${_bb:0:2}/${_bb:2}" '$4 == a || $4 == b' \ + | wc -l | tr -d ' ')" +rm -f "$W/.git/objects/${_ba:0:2}/${_ba:2}" "$W/.git/objects/${_bb:0:2}/${_bb:2}" +git -C "$W" update-ref -d refs/keep/a +git -C "$W" update-ref -d refs/keep/b + +# Verified pack data is not stored twice; a loose copy with extra bytes +# after its zlib stream is stored with them. +git -C "$W" repack -q -a -d +_pk=$(find "$W/.git/objects/pack" -name 'pack-*.pack' | head -1) +_head=$(git -C "$W" rev-parse HEAD) +mkdir -p "$W/.git/objects/${_head:0:2}" +git -C "$W" cat-file commit "$_head" \ + | git -C "$TMPDIR/subsrc" hash-object -t commit --stdin -w >/dev/null +cp "$TMPDIR/subsrc/.git/objects/${_head:0:2}/${_head:2}" \ + "$W/.git/objects/${_head:0:2}/${_head:2}" +chmod u+w "$W/.git/objects/${_head:0:2}/${_head:2}" +printf 'TRAILING' >> "$W/.git/objects/${_head:0:2}/${_head:2}" +out=$(scan worktree "$W" full) +dst=$(imported "$out") +assert_eq "a verified pack is not stored as a file" "0" \ + "$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/pack/${_pk##*/}" '$4 == f' | wc -l | tr -d ' ')" +_lf=$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/${_head:0:2}/${_head:2}" '$4 == f { print $1 }') +assert_eq "a loose file with trailing bytes keeps them" "TRAILING" \ + "$(git -C "$dst" cat-file -p "refs/r/0/f/$_lf" | tail -c 8)" +rm -f "$W/.git/objects/${_head:0:2}/${_head:2}" + +# A pack file that is a symlink is stored as a symlink, not exempted. +_pk=$(find "$W/.git/objects/pack" -name 'pack-*.pack' | head -1) +_pb="${_pk%.pack}" +ln -s /elsewhere/sidecar "$_pb.bitmap" +out=$(scan worktree "$W" full) +assert_eq "a symlinked pack sidecar is stored as a symlink" "1" \ + "$(git -C "$(imported "$out")" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/pack/${_pb##*/}.bitmap" '$2 == "120000" && $4 == f' \ + | wc -l | tr -d ' ')" +rm -f "$_pb.bitmap" + +# Names and symlink targets keep every byte, newlines included. +_nl="odd"$'\n'"name" +echo payload > "$W/.git/$_nl" +ln -s "target"$'\n'"line" "$W/.git/nl-link" +out=$(scan worktree "$W" full) +dst=$(imported "$out") +# The manifest is NUL-separated; turn NULs into "|" to keep the name's +# own newline visible as one entry. +assert_eq "a file name with a newline is stored whole" "1" \ + "$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0\n' '|~' \ + | tr '|' '\n' | grep -c '^[0-9]* 100644 8 odd~name$' || true)" +_ln=$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk '$4 == "nl-link" { print $1 }') +assert_eq "a symlink target keeps its newline" \ + "target"$'\n'"line" "$(git -C "$dst" cat-file -p "refs/r/0/f/$_ln")" +rm -f "$W/.git/$_nl" "$W/.git/nl-link" +ln -s "trail"$'\n' "$W/.git/trail-link" +out=$(scan worktree "$W" full) +dst=$(imported "$out") +_ln=$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk '$4 == "trail-link" { print $1 }') +assert_eq "a symlink target keeps a trailing newline" "74 72 61 69 6c 0a" \ + "$(git -C "$dst" cat-file -p "refs/r/0/f/$_ln" | od -An -tx1 \ + | tr -s ' \n' ' ' | sed 's/^ //; s/ $//')" +rm -f "$W/.git/trail-link" + +# lfs.storage set only for a linked worktree still blocks. +git -C "$W" worktree add -q "$W/wt2" 2>/dev/null +git -C "$W" config extensions.worktreeConfig true +git -C "$W/wt2" config --worktree lfs.storage /elsewhere +assert_eq "a worktree's lfs.storage blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block lfs-storage .git/worktrees/wt2 /elsewhere')" +git -C "$W" worktree remove --force "$W/wt2" +git -C "$W" config --unset extensions.worktreeConfig + +# A rename out of an engine path does not hide the new file's edits. +mkdir -p "$W/agent_logs" +echo old > "$W/agent_logs/old" +g -C "$W" add -f agent_logs/old +g -C "$W" commit -q -m old-log +git -C "$W" mv agent_logs/old valuable.txt +echo edited >> "$W/valuable.txt" +assert_eq "a file renamed from agent_logs is reported" "1" \ + "$(grep -c 'valuable.txt' "$(scan worktree "$W" full)/report" || true)" +git -C "$W" reset -q --hard HEAD +git -C "$W" rm -q agent_logs/old +g -C "$W" commit -q -m drop-old-log + +# A submodule may be called logs; its worktrees are still found. +LSUB="$TMPDIR/logsrc" +git init -q "$LSUB" +g -C "$LSUB" commit -q --allow-empty -m logs +g -C "$W" -c protocol.file.allow=always submodule add -q "$LSUB" logs \ + >/dev/null 2>&1 +git -C "$W/logs" worktree add -q --detach "$TMPDIR/logs-outside" \ + 2>/dev/null +assert_eq "a worktree of a submodule called logs is found" "1" \ + "$(has_line "$(scan worktree "$W" full)" \ + "block external $(cd -P "$TMPDIR" && pwd -P)/logs-outside")" +git -C "$W/logs" worktree remove --force "$TMPDIR/logs-outside" +git -C "$W" rm -q -f logs +rm -rf "$W/.git/modules/logs" +git -C "$W" checkout -q HEAD -- .gitmodules + +# Engine paths are only engine files in the workspace's own checkout; +# in a submodule they are work. +git -C "$W" -c protocol.file.allow=always submodule update -q --init \ + >/dev/null 2>&1 +mkdir -p "$W/lib/agent_logs" +echo notes > "$W/lib/agent_logs/notes" +assert_eq "agent_logs inside a submodule is work" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'uncommitted dirty lib: ?? agent_logs/notes')" +rm -rf "$W/lib/agent_logs" + +# A submodule of a linked worktree keeps its git dir under +# .git/worktrees//modules/; its checkout is still checked. +git -C "$W" worktree add -q --detach "$W/wt-sub" 2>/dev/null +g -C "$W/wt-sub" -c protocol.file.allow=always submodule update -q --init \ + >/dev/null 2>&1 +echo extra > "$W/wt-sub/lib/newfile" +assert_eq "a linked worktree's submodule checkout is checked" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'uncommitted dirty wt-sub/lib: ?? newfile')" +git -C "$W" worktree remove --force "$W/wt-sub" + +# A linked worktree whose HEAD is a symlink is still found. +git -C "$W" worktree add -q --detach "$TMPDIR/symhead-outside" 2>/dev/null +_wtg=$(find "$W/.git/worktrees" -maxdepth 1 -mindepth 1 -type d | head -1) +_wth=$(cat "$_wtg/HEAD") +printf '%s\n' "$_wth" > "$TMPDIR/symhead-target" +rm "$_wtg/HEAD" +ln -s "$TMPDIR/symhead-target" "$_wtg/HEAD" +assert_eq "a worktree with a symlinked HEAD is found" "1" \ + "$(has_line "$(scan worktree "$W" full)" \ + "block external $(cd -P "$TMPDIR" && pwd -P)/symhead-outside")" +rm "$_wtg/HEAD" +git -C "$W" worktree remove --force "$TMPDIR/symhead-outside" 2>/dev/null \ + || { rm -rf "$_wtg" "$TMPDIR/symhead-outside"; git -C "$W" worktree prune; } + +# A git dir whose path holds a newline cannot be listed; it blocks. +git init -q --bare "$W/.git/modules/a"$'\n'"b" +assert_eq "a git dir with a newline in its path blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block newline-path .git/modules/a\nb')" +rm -rf "$W/.git/modules/a"$'\n'"b" + +# Worktrees added with --relative-paths point back with a relative +# path; one inside the volume is checked, one outside blocks. +# git ... -h exits 129, so its help is read without pipefail. +_wth=$(git -C "$W" worktree add -h 2>&1 || true) +if grep -q 'relative-paths' <<< "$_wth"; then + git -C "$W" worktree add -q --detach --relative-paths "$W/rel-in" \ + 2>/dev/null + git -C "$W" worktree add -q --detach --relative-paths \ + "$TMPDIR/rel-out" 2>/dev/null + echo inside-edit > "$W/rel-in/new-file" + out=$(scan worktree "$W" full) + assert_eq "a relative worktree inside the volume is checked" "1" \ + "$(has_line "$out" 'uncommitted dirty rel-in: ?? new-file')" + assert_eq "a relative worktree inside the volume is not external" "0" \ + "$(has_line "$out" "block external $(cd -P "$W" && pwd -P)/rel-in")" + git -C "$W" worktree remove --force "$W/rel-in" + out=$(scan worktree "$W" full) + assert_eq "a relative worktree outside the volume blocks" "1" \ + "$(has_line "$out" "block external $(cd -P "$TMPDIR" && pwd -P)/rel-out")" + git -C "$W" worktree remove --force "$TMPDIR/rel-out" +else + echo " SKIP: git worktree add --relative-paths not supported" +fi + +# A copy of a linked worktree points at an admin dir whose back-pointer +# names the original: the copy blocks instead of going unchecked. +mkdir -p "$W/agent_logs" +git -C "$W" worktree add -q --detach "$W/agent_logs/wt" 2>/dev/null +cp -R "$W/agent_logs/wt" "$W/agent_logs/copy" +echo edit > "$W/agent_logs/copy/edited" +assert_eq "a copied linked worktree blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block mismatched-worktree agent_logs/copy')" +git -C "$W" worktree remove --force "$W/agent_logs/wt" +rm -rf "$W/agent_logs" + +# A reflog file whose name holds a newline is read whole. +_rl=$(g -C "$W" commit-tree -m only-in-odd-reflog "$(git -C "$W" rev-parse 'HEAD^{tree}')") +printf '%s %s t 0 +0000\tnote\n' "$(git -C "$W" rev-parse HEAD)" "$_rl" \ + > "$W/.git/logs/notes"$'\n'"saved" +out=$(scan worktree "$W" full) +assert_eq "a reflog file with a newline in its name scans" "" \ + "$(cat "$out/.rc" 2>/dev/null)" +assert_eq "its entries are kept" "commit" \ + "$(git -C "$(imported "$out")" cat-file -t "$_rl" 2>/dev/null)" +rm -f "$W/.git/logs/notes"$'\n'"saved" + +# SHA-256 repositories scan and import into a SHA-256 checkout. +S256="$TMPDIR/sha256" +git init -q --object-format=sha256 "$S256" +g -C "$S256" commit -q --allow-empty -m one +g -C "$S256" commit -q --allow-empty -m two +_lost256=$(git -C "$S256" rev-parse HEAD) +git -C "$S256" reset -q --hard HEAD~1 +git -C "$S256" repack -q -a -d +_p256=$(find "$S256/.git/objects/pack" -name 'pack-*.pack' | head -1) +# The test runs from a SHA-1 repository; the pack is still read as +# SHA-256 and exempted, not stored as a file. +out=$(cd "$REPO_ROOT" && scan worktree "$S256" full) +assert_eq "a verified SHA-256 pack is not stored as a file" "0" \ + "$(git -C "$(cd "$TMPDIR" && git init -q --object-format=sha256 i256 \ + && git -C i256 fetch -q --no-tags "$out/bundle" 'refs/v/*:refs/r/*' \ + && echo "$TMPDIR/i256")" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk -v f="objects/pack/${_p256##*/}" '$4 == f' | wc -l | tr -d ' ')" +assert_eq "a SHA-256 workspace scans" "" "$(cat "$out/.rc" 2>/dev/null)" +C256="$TMPDIR/sha256-checkout" +git init -q --object-format=sha256 "$C256" +g -C "$C256" commit -q --allow-empty -m base +rc=0 +swarm_recovery_import "$C256" "$out" "refs/swarm-recovery/e/s/v" \ + 2>/dev/null || rc=$? +assert_eq "it imports into a SHA-256 checkout" "0" "$rc" +assert_eq "its lost commit is there" "commit" \ + "$(git -C "$C256" cat-file -t "$_lost256" 2>/dev/null)" + +# ============================================================ +echo "" +echo "=== 10. Bare repos: every ref, HEAD and file ===" + +B="$TMPDIR/scan-bare.git" +git clone -q --bare "$WORK" "$B" +git -C "$B" update-ref refs/custom/x "$ONE" +git -C "$B" update-ref --no-deref HEAD "$ONE" +git -C "$B" config transfer.hideRefs refs/custom +out=$(scan gitdir "$B" -) +dst=$(imported "$out") +assert_eq "a bare repo scans" "" "$(cat "$out/.rc" 2>/dev/null)" +assert_eq "every object of the bare is in the bundle" "0" \ + "$(git -C "$dst" cat-file --batch-check < "$out/objects" \ + | grep -c ' missing$' || true)" +assert_eq "its config file is stored" "1" \ + "$(git -C "$dst" cat-file -p refs/r/0/files | tr '\0' '\n' \ + | awk '$4 == "config"' | wc -l | tr -d ' ')" + +# ============================================================ +echo "" +echo "=== 11. Importing into the checkout ===" + +IMP="$TMPDIR/import-checkout" +git clone -q "$WORK" "$IMP" +git -C "$IMP" config fetch.prune true +out=$(scan worktree "$W" full) +P1="refs/swarm-recovery/e1/s1/vol" +rc=0 +swarm_recovery_import "$IMP" "$out" "$P1" 2>/dev/null || rc=$? +assert_eq "an import succeeds" "0" "$rc" +assert_eq "the lost commit is in the checkout" "commit" \ + "$(git -C "$IMP" cat-file -t "$LOST" 2>/dev/null)" +assert_eq "its refs are under the prefix" "true" \ + "$([ -n "$(git -C "$IMP" for-each-ref "$P1/")" ] && echo true || echo false)" +rc=0 +swarm_recovery_import "$IMP" "$out" "$P1" 2>/dev/null || rc=$? +assert_eq "a prefix that holds refs is not written again" "1" "$rc" +_before=$(git -C "$IMP" for-each-ref "$P1/" | wc -l) +swarm_recovery_import "$IMP" "$out" "refs/swarm-recovery/e1/s2/vol" \ + 2>/dev/null || true +assert_eq "a second snapshot leaves the first alone" "$_before" \ + "$(git -C "$IMP" for-each-ref "$P1/" | wc -l)" +rc=0 +swarm_recovery_import "$IMP" "$out" "refs/heads/x" 2>/dev/null || rc=$? +assert_eq "only the recovery namespace is written" "1" "$rc" +_bad=$(mktemp -d "$TMPDIR/bad.XXXXXX") +echo garbage > "$_bad/bundle" +: > "$_bad/objects" +rc=0 +swarm_recovery_import "$IMP" "$_bad" "refs/swarm-recovery/e1/s3/v" \ + 2>/dev/null || rc=$? +assert_eq "a bundle that does not verify is refused" "1" "$rc" +# A missing inventory, or objects of it not in the checkout, fail the +# import, however many objects are missing. +_mo=$(scan worktree "$W" full) +rm -f "$_mo/objects" +rc=0 +swarm_recovery_import "$IMP" "$_mo" "refs/swarm-recovery/e1/s4/v" \ + 2>/dev/null || rc=$? +assert_eq "an import without an inventory fails" "1" "$rc" +_mo=$(scan worktree "$W" full) +for _i in $(seq 1 3000); do + printf '%040x\n' "$_i" +done >> "$_mo/objects" +rc=0 +swarm_recovery_import "$IMP" "$_mo" "refs/swarm-recovery/e1/s5/v" \ + 2>/dev/null || rc=$? +assert_eq "an import with many missing objects fails" "1" "$rc" +# An object already in the checkout with the right name but other +# content is caught: fetch skips it, the content check does not. +CR="$TMPDIR/corrupt-src" +git init -q "$CR" +printf 'aaaa\n' > "$CR/f" +g -C "$CR" add f +g -C "$CR" commit -q -m f +_blob=$(git -C "$CR" rev-parse HEAD:f) +CC="$TMPDIR/corrupt-checkout" +git init -q "$CC" +g -C "$CC" commit -q --allow-empty -m base +out=$(scan worktree "$CR" full) +swarm_recovery_import "$CC" "$out" "refs/swarm-recovery/c/s1/v" 2>/dev/null +# Turn the imported pack into loose objects, so one can be replaced. +for _p in "$CC"/.git/objects/pack/pack-*.pack; do + mv "$_p" "$TMPDIR/cc.pack" + rm -f "${_p%.pack}".* + git -C "$CC" unpack-objects -q < "$TMPDIR/cc.pack" +done +_other=$(printf 'bbbb\n' | git -C "$TMPDIR/subsrc" hash-object -w --stdin) +_bf="$CC/.git/objects/${_blob:0:2}/${_blob:2}" +chmod u+w "$_bf" +cp "$TMPDIR/subsrc/.git/objects/${_other:0:2}/${_other:2}" "$_bf" +rc=0 +swarm_recovery_verify_checkout "$CC" 2>/dev/null || rc=$? +assert_eq "a checkout with a corrupt object fails verification" "1" "$rc" +# Two objects with each other's contents: every name is present, only +# the content check sees it. +printf 'cccc\n' > "$CR/g" +g -C "$CR" add g +g -C "$CR" commit -q -m g +_bg=$(git -C "$CR" rev-parse HEAD:g) +rm -rf "$CC" +git init -q "$CC" +g -C "$CC" commit -q --allow-empty -m base +out=$(scan worktree "$CR" full) +swarm_recovery_import "$CC" "$out" "refs/swarm-recovery/c/s1/v" 2>/dev/null +for _p in "$CC"/.git/objects/pack/pack-*.pack; do + mv "$_p" "$TMPDIR/cc.pack" + rm -f "${_p%.pack}".* + git -C "$CC" unpack-objects -q < "$TMPDIR/cc.pack" +done +rc=0 +swarm_recovery_verify_checkout "$CC" 2>/dev/null || rc=$? +assert_eq "a sound checkout passes verification" "0" "$rc" +_fa="$CC/.git/objects/${_blob:0:2}/${_blob:2}" +_fb="$CC/.git/objects/${_bg:0:2}/${_bg:2}" +chmod u+w "$_fa" "$_fb" +cp "$_fa" "$TMPDIR/swap.tmp" +cp "$_fb" "$_fa" +cp "$TMPDIR/swap.tmp" "$_fb" +rc=0 +swarm_recovery_import "$CC" "$out" "refs/swarm-recovery/c/s2/v" \ + 2>/dev/null || rc=$? +assert_eq "presence checks alone pass with swapped objects" "0" "$rc" +rc=0 +swarm_recovery_verify_checkout "$CC" 2>/dev/null || rc=$? +assert_eq "swapped objects fail verification" "1" "$rc" +_id=$(swarm_recovery_snapshot_id) +assert_eq "snapshot ids are time and random hex" "true" \ + "$(grep -qE '^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$' <<< "$_id" \ + && echo true || echo false)" + +# ============================================================ +echo "" +echo "=== 12. The checkout must keep its objects itself ===" + +ST="$TMPDIR/storage" +RT="$TMPDIR/storage-runtime" +mkdir -p "$RT" +git clone -q "$WORK" "$ST" +storage() { + local rc=0 + swarm_checkout_storage_ok "$ST" "$RT" "$@" 2>/dev/null || rc=$? + printf '%s' "$rc" +} +assert_eq "a plain clone is fine" "0" "$(storage)" +echo "$BARE/objects" > "$ST/.git/objects/info/alternates" +assert_eq "alternates are refused" "1" "$(storage)" +rm "$ST/.git/objects/info/alternates" +ln -s "$RT" "$ST/.git/objects/linked" +assert_eq "a symlink in the object store is refused" "1" "$(storage)" +rm "$ST/.git/objects/linked" +git clone -q --depth 1 "file://$WORK" "$TMPDIR/shallow" 2>/dev/null +rc=0 +swarm_checkout_storage_ok "$TMPDIR/shallow" "$RT" 2>/dev/null || rc=$? +assert_eq "a shallow clone is refused" "1" "$rc" +# A failure of the pack/info listing alone fails the check. Called +# in a command substitution, so the find override ends with it. (Bash +# 3.2 cannot parse a case statement written inside $(...).) +storage_failing_pack_find() { + # shellcheck disable=SC2329 + find() { + case "$1" in + */objects/pack) return 1 ;; + esac + command find "$@" + } + storage "$@" +} +mkdir -p "$RT/target-x" +assert_eq "a failed pack listing fails the check" "1" \ + "$(storage_failing_pack_find "$RT/target-x")" +rm -rf "$RT/target-x" +ln -s "$RT" "$ST/.git/refs/swarm-recovery" +assert_eq "recovery refs stored through a symlink are refused" "1" \ + "$(storage)" +rm "$ST/.git/refs/swarm-recovery" +git -C "$ST" config remote.origin.promisor 1 +assert_eq "a promisor remote set to 1 is refused" "1" "$(storage)" +git -C "$ST" config --unset remote.origin.promisor +git -C "$ST" repack -q -a -d +_pack=$(find "$ST/.git/objects/pack" -name '*.pack' | head -1) +mkdir -p "$RT/upstream.git/objects/pack" +ln "$_pack" "$RT/upstream.git/objects/pack/shared.pack" +assert_eq "a pack file shared with a deletion target is refused" "1" \ + "$(storage "$RT/upstream.git")" +rm -rf "$RT/upstream.git" +mkdir -p "$RT/upstream.git" +assert_eq "an unrelated deletion target is fine" "0" \ + "$(storage "$RT/upstream.git")" + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" From 2081c402dc3c66d277cb038c6d9b67ef0cb419ba Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:13:40 +0100 Subject: [PATCH 06/27] Add launch.sh retire for stopped containers whose files live in their volumes --- CHANGELOG.md | 21 + USAGE.md | 51 ++ launch.sh | 18 + lib/preserve-scan.sh | 520 ++++++++++++++- lib/preserve.sh | 48 +- lib/project.sh | 16 + lib/retire.sh | 859 ++++++++++++++++++++++++ tests/runtime_retire.sh | 520 +++++++++++++++ tests/test.sh | 23 +- tests/test_preserve.sh | 88 ++- tests/test_retire.sh | 1378 +++++++++++++++++++++++++++++++++++++++ 11 files changed, 3515 insertions(+), 27 deletions(-) create mode 100644 lib/retire.sh create mode 100755 tests/runtime_retire.sh create mode 100755 tests/test_retire.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index cd351ea..ef85d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ ## Unreleased +- **Feature: `launch.sh retire [--post-process]`.** Removes this + checkout's stopped engine containers (or only the post-process one) by + ID, without `-f`, after checking every selected container first: + exited or created, no restart policy or auto-remove, exactly one mount + at `/workspace` that is its own workspace volume and nothing mounted + or on tmpfs below it, no open interactive session, no git config in + the container that keeps repository data elsewhere (the global and + system files its git names with `git var`, so Git 2.42 or later in the + image), no repository data outside the volume (worktrees, links, + alternates, config), no symlink in the volume to data the container + wrote (compared with `docker diff` and with the original image; a name + `docker diff` cannot report exactly, not being UTF-8, refuses), and no + Git repository anywhere outside the volume, made by the container or + shipped by its image, changed or not (`docker diff` cannot show every + change). The checks run in a `docker commit` snapshot of the + container. A container that changed since its check is left. Workspace + volumes stay. It needs no swarmfile, takes the engagement lock and + checks the engagement's Docker daemon. Until containers get workspace + volumes, it refuses every container. `tests/runtime_retire.sh` (in + `tests/test.sh --runtime` and `--all`) runs it on real stopped + containers. - **Feature: recovery snapshots (building block).** `lib/preserve-scan.sh` packs everything Git keeps in a repository location (every ref, reflog entry, stash, operation state, object, diff --git a/USAGE.md b/USAGE.md index 7618b41..6c183b2 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1165,6 +1165,57 @@ rm -rf /tmp/-swarm.env find -H /tmp -maxdepth 1 -name '-mirror-*.git' -exec rm -rf {} + ``` +## Retiring containers + +`./launch.sh retire` removes this checkout's stopped engine containers; +`./launch.sh retire --post-process` removes only `-agent-post`, +so `post-process` can run again. A container's `/workspace` is a +Docker volume that stays; what goes is the container's own layer, so +retire checks every selected container before it removes any, and +refuses when one + +- is running, paused or restarting, or has a restart policy or + auto-remove (Docker could start or remove it on its own); +- has anything but its own workspace volume at `/workspace`, or + another mount or a tmpfs below it; +- belongs to an open interactive session; +- has global or system git config (the files the container's git + names with `git var`, which needs Git 2.42 or later in the image, + read as the container's user with its `HOME` and `XDG_CONFIG_HOME`) + that sets `lfs.storage`, `core.worktree` or `core.hooksPath`, + includes other files, cannot be parsed, or lies on a mount; or has + Git environment variables of its own, or a `HOME` that is empty, + relative or not normalized; +- has repository data outside `/workspace`: a worktree registered + elsewhere, a `.git` that is a link or a gitfile, repository config + that includes files or sets `core.worktree` outside, `core.hooksPath` + or `lfs.storage`, alternates outside, or a nested repository; +- has a symlink in the volume that leads to data the container wrote + itself, or to an image file it changed (links into the unchanged + image, such as a virtualenv's `python`, are fine); +- has a Git repository anywhere outside `/workspace`, one it made (a + clone in `/tmp`, a bare repository; package caches that keep + repositories, such as Go's, count too) or one its image ships, + changed or not: `docker diff` goes by metadata, so it cannot show + every change to a repository (the engine image ships none). + +The checks run in a throwaway container started from a snapshot of the +stopped container (`docker commit`, removed afterwards, also when +retire is interrupted; one left by a retire that was killed outright +is removed by the next), with the volume read-only and no network, so +links resolve as they did in the container and every repository +outside the volume is found. For links that lead out of the volume, +`docker diff` and a comparison with the original image tell whether +the container wrote what they lead to (a name that is not valid UTF-8 +reaches `docker diff` altered, so such a name refuses too). + +The error says what to copy out first. Removal goes by ID, without +`-f`, and leaves any container that changed since it was checked. Run +`costs.sh` before retire if you need the containers' stats. retire +needs no swarmfile; it takes the engagement lock and refuses when +Docker talks to another daemon than the engagement's (an engagement +that recorded none needs `SWARM_DOCKER_DAEMON=`). + ## Cleanup After a swarm run, the agent containers (`-agent-N`, diff --git a/launch.sh b/launch.sh index ea22e21..9a0acb7 100755 --- a/launch.sh +++ b/launch.sh @@ -28,6 +28,9 @@ Commands: chat PROFILE Alias for interactive PROFILE. shell PROFILE Start an interactive shell from a named agent profile. + retire [--post-process] + Remove stopped containers whose files all live + in their workspace volumes. Start options: --dashboard Open the TUI dashboard after launch. @@ -420,6 +423,21 @@ available_drivers() { -exec basename {} .sh \; | tr '\n' ' ' } +# retire needs no swarmfile: it acts on what the runtime directory and +# Docker already hold. +case "${1:-}" in + retire) + shift + # shellcheck source=lib/preserve.sh + source "$SWARM_DIR/lib/preserve.sh" + # shellcheck source=lib/retire.sh + source "$SWARM_DIR/lib/retire.sh" + rc=0 + cmd_retire "$@" || rc=$? + exit "$rc" + ;; +esac + CONFIG_FILE="${SWARM_CONFIG:-}" if [ -z "$CONFIG_FILE" ] && [ -f "$REPO_ROOT/swarm.json" ]; then CONFIG_FILE="$REPO_ROOT/swarm.json" diff --git a/lib/preserve-scan.sh b/lib/preserve-scan.sh index 7f5c8bf..5a3984d 100644 --- a/lib/preserve-scan.sh +++ b/lib/preserve-scan.sh @@ -7,11 +7,16 @@ # # Usage: preserve-scan.sh worktree ROOT CONTEXT OUT # preserve-scan.sh gitdir ROOT - OUT +# preserve-scan.sh userconfig - - OUT # # worktree: ROOT is a checkout (a workspace) whose repository is # ROOT/.git; CONTEXT (full, slim or none) says which deletions under # .claude the engine made itself. # gitdir: ROOT is a repository directory (a bare repo or a mirror). +# userconfig: run as a container's user, with its HOME and +# XDG_CONFIG_HOME, in a snapshot of that container: reports the global +# and system git config settings that keep repository data outside a +# workspace, and config files that lie on a mount the snapshot lacks. # OUT is an empty directory. On success the scan writes: # OUT/report one line per finding (see below) # OUT/bundle a bundle whose refs are refs/v//t/ (one per @@ -24,6 +29,14 @@ # block content the bundle cannot carry # uncommitted content outside Git's history that # only an explicit discard may drop +# layer-link where a symlink in the volume leads +# out of it (worktree mode, resolved): +# in the image or in the container's +# own layer +# layer-via a symlink outside the volume that a +# link in it passes +# layer-path any path outside the volume such a +# link visits # Exit status: 0 when the report and bundle are complete, 3 when the # scan cannot tell (message on stderr). Compatible with Bash 3.2. @@ -56,10 +69,10 @@ fail() { } case "$MODE" in - worktree|gitdir) ;; + worktree|gitdir|userconfig) ;; *) fail "unknown mode: ${MODE}" ;; esac -[ -d "$ROOT" ] || fail "not a directory: ${ROOT}" +[ "$MODE" = userconfig ] || [ -d "$ROOT" ] || fail "not a directory: ${ROOT}" [ -d "$OUT" ] || fail "not a directory: ${OUT}" # Print the resolved form of directory $1. Command substitution would # drop trailing newlines and turn "repo" into "repo", so the x keeps @@ -76,8 +89,71 @@ canon() { printf '%s' "$r" } +# Print path $1 on one line, newlines shown as \n. +oneline() { + printf '%s' "$1" | awk 'BEGIN { ORS = "\\n" } { print }' \ + | sed 's/\\n$//' +} + + + + +# Succeed when file $1 holds a NUL byte, or cannot be read. Git stops +# reading a path at a NUL, while the shell's read drops it and carries +# on, so the two would disagree on what such a file names. +has_nul() { + local all kept + all=$(wc -c 2>/dev/null < "$1" | tr -d ' ') || return 0 + kept=$(tr -d '\000' 2>/dev/null < "$1" | wc -c | tr -d ' ') || return 0 + [ -n "$all" ] && [ "$all" = "$kept" ] && return 1 + return 0 +} + +# Print where absolute path $1 leads, resolved the way the kernel does: +# components from the left, a symlink replaced by its target before any +# later ".." applies, "." dropped; components that do not exist here +# are kept as text (they may exist only where the link was made). In +# worktree mode the scan runs from a snapshot of the stopped container +# itself (retire commits it), so the links on the way are that +# container's own. Every symlink passed is appended to file $2, and +# every path visited (links or not) to file $3, when given. Fails (1) +# after 40 links (a loop), and (2) at a name with a newline, which the +# line-based lists here cannot hold. +real_path() { + local rest="${1#/}" out="" comp n=0 t via="${2:-/dev/null}" + local seen="${3:-/dev/null}" + while [ -n "$rest" ]; do + comp="${rest%%/*}" + if [ "$comp" = "$rest" ]; then rest=""; else rest="${rest#*/}"; fi + case "$comp" in + ''|.) continue ;; + ..) out="${out%/*}"; continue ;; + *$'\n'*) return 2 ;; + esac + # Every path visited, before any later ".." takes it back. + printf '%s\n' "${out}/${comp}" >> "$seen" + if [ -L "${out}/${comp}" ]; then + n=$((n + 1)) + [ "$n" -le 40 ] || return 1 + printf '%s\n' "${out}/${comp}" >> "$via" + t=$(readlink -n "${out}/${comp}" && printf x) || return 1 + t="${t%x}" + case "$t" in + /*) out="" ; t="${t#/}" ;; + esac + if [ -n "$rest" ]; then rest="${t}/${rest}"; else rest="$t"; fi + continue + fi + out="${out}/${comp}" + done + printf '%s' "${out:-/}" +} + # Git records worktree paths resolved; compare against the same form. -ROOT=$(canon "$ROOT") || fail "cannot resolve ${2:-}, or it holds a newline" +if [ "$MODE" != userconfig ]; then + ROOT=$(canon "$ROOT") \ + || fail "cannot resolve ${2:-}, or it holds a newline" +fi _entries=$(ls -A "$OUT") || fail "cannot list ${OUT}" [ -z "$_entries" ] || fail "output directory is not empty: ${OUT}" @@ -122,6 +198,133 @@ fsize() { wc -c < "$1" | tr -d ' ' } +# canon, for paths the repository's own settings name (core.worktree, +# commondir, alternates, gitfiles, worktree back-pointers): also fails +# when resolving the path visits one of the mounts the snapshot lacks +# ($OUT/.mounts, worktree mode), since there it may lead elsewhere in +# the container, and reports the links it passes outside ROOT as +# layer-via (the container may have made them). +canon_nm() { + local v m + : > "$OUT/.cnm" + : > "$OUT/.cnv" + real_path "$1" "$OUT/.cnv" "$OUT/.cnm" >/dev/null || return 1 + # Links outside ROOT on the way are the container's to tell about: + # report them for retire's comparison with docker diff. + while IFS= read -r v; do + case "$v" in + "$ROOT"|"$ROOT"/*) ;; + *) report "layer-via $(oneline "$v")" ;; + esac + done < "$OUT/.cnv" + while IFS= read -r v; do + case "$v" in + "$ROOT"|"$ROOT"/*) ;; + *) report "layer-path $(oneline "$v")" ;; + esac + done < "$OUT/.cnm" + if [ -s "$OUT/.mounts" ]; then + while IFS= read -r v; do + while IFS= read -r m; do + [ -n "$m" ] || continue + case "$v" in + "$m"|"$m"/*) return 1 ;; + esac + done < "$OUT/.mounts" + done < "$OUT/.cnm" + fi + canon "$1" +} + +# ---- userconfig mode ---------------------------------------------------- + +if [ "$MODE" = userconfig ]; then + # The mounts the snapshot lacks: the container's own + # (SWARM_SCAN_MOUNTS) and those Docker makes in every container. + # /workspace too: this check runs without the volume. Each mount + # counts as configured and as resolved here. + printf '%s\n' "${SWARM_SCAN_MOUNTS:-}" /workspace /dev /proc /sys \ + /etc/hosts /etc/hostname /etc/resolv.conf | grep '^/' \ + > "$OUT/.um0" || : > "$OUT/.um0" + cp "$OUT/.um0" "$OUT/.um" + while IFS= read -r m; do + mr=$(real_path "$m" && printf x) || fail "cannot resolve mount ${m}" + printf '%s\n' "${mr%x}" >> "$OUT/.um" + done < "$OUT/.um0" + # The files Git reads for its global and system settings, as the + # git on this PATH names them: a git built with another prefix + # reads its system file elsewhere than /etc/gitconfig. A git that + # cannot name them (older than 2.42, or stopped by a config file it + # cannot parse) is refused, with its first message. + if ! git var GIT_CONFIG_GLOBAL > "$OUT/.uf" 2>"$OUT/.ue" \ + || ! git var GIT_CONFIG_SYSTEM >> "$OUT/.uf" 2>>"$OUT/.ue" \ + || [ -s "$OUT/.ue" ]; then + report "block config-paths-unknown $(oneline "$(head -n 1 "$OUT/.ue")")" + exit 0 + fi + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + /*) ;; + *) report "block config-path ${f}"; continue ;; + esac + : > "$OUT/.us" + rp_rc=0 + r=$(real_path "$f" /dev/null "$OUT/.us" && printf x) || rp_rc=$? + if [ "$rp_rc" -ne 0 ]; then + report "block config-path $(oneline "$f")" + continue + fi + r="${r%x}" + printf '%s\n' "$r" >> "$OUT/.us" + on_mount="" + while IFS= read -r v; do + while IFS= read -r m; do + case "$v" in + "$m"|"$m"/*) on_mount="$m" ;; + esac + done < "$OUT/.um" + done < "$OUT/.us" + if [ -n "$on_mount" ]; then + report "block config-on-mount $(oneline "$f") ${on_mount}" + continue + fi + # A missing file is fine only when its nearest existing + # directory can be searched (else "missing" may be a permission + # error); an existing file must be readable, and git must read + # it without a word on stderr. + if [ ! -e "$r" ] && [ ! -L "$r" ]; then + pdir=$(dirname "$r") + while [ ! -e "$pdir" ] && [ "$pdir" != / ]; do + pdir=$(dirname "$pdir") + done + if [ -d "$pdir" ] && [ -x "$pdir" ]; then + continue + fi + report "block config-unreadable $(oneline "$f")" + continue + fi + if [ ! -f "$r" ] || [ ! -r "$r" ]; then + report "block config-unreadable $(oneline "$f")" + continue + fi + rc=0 + git config -f "$r" --name-only --get-regexp \ + '^(lfs\.storage|core\.worktree|core\.hookspath|include\..*|includeif\..*)$' \ + > "$OUT/.uk" 2>"$OUT/.ue" || rc=$? + if [ -s "$OUT/.ue" ]; then + report "block config-unreadable $(oneline "$f")" + continue + fi + case "$rc" in + 0) report "block config-setting $(oneline "$f") $(tr '\n' ' ' < "$OUT/.uk")" ;; + 1) ;; + *) report "block config-unreadable $(oneline "$f")" ;; + esac + done < "$OUT/.uf" + exit 0 +fi + # ---- Find the git dirs -------------------------------------------------- # A git dir is a directory with HEAD and either an objects/ directory @@ -153,6 +356,74 @@ list_gitdirs() { done < "$OUT/.heads" } +# ---- The container's other mounts (worktree mode) ----------------------- + +# Checked first: an empty volume or one without a repository still +# must not be hidden or reached by another mount. +: > "$OUT/.mounts" +if [ "$MODE" = worktree ]; then + # The container's other mounts (SWARM_SCAN_MOUNTS, one destination + # per line, from retire), as configured and as resolved here: the + # snapshot has neither their contents nor the links inside them, so + # any path a link visits at or below one blocks. + if [ -n "${SWARM_SCAN_MOUNTS:-}" ]; then + printf '%s\n' "$SWARM_SCAN_MOUNTS" | grep '^/' > "$OUT/.mounts-lit" \ + || : > "$OUT/.mounts-lit" + # Each destination as configured and as resolved here, tagged + # with the configured one: "
". + : > "$OUT/.mounts-forms" + while IFS= read -r om; do + omr=$(real_path "$om" && printf x) \ + || fail "cannot resolve mount ${om}" + omr="${omr%x}" + printf '%s\t%s\n%s\t%s\n' "$om" "$om" "$om" "$omr" \ + >> "$OUT/.mounts-forms" + done < "$OUT/.mounts-lit" + while IFS= read -r m; do + case "$m" in + /*) ;; + *) continue ;; + esac + printf '%s\n' "$m" >> "$OUT/.mounts" + # The container made this mount after the others, through + # whatever they hold; the snapshot has none of that, so a + # destination whose path passes through another mount cannot + # be resolved here. + : > "$OUT/.mseen" + mr=$(real_path "$m" /dev/null "$OUT/.mseen" && printf x) \ + || fail "cannot resolve mount ${m}" + # Any path visited at, or below, another mount (as configured + # or as resolved) means the destination goes through it. + while IFS= read -r ms; do + while IFS="$(printf '\t')" read -r om of; do + [ "$om" != "$m" ] || continue + case "$ms" in + "$of"|"$of"/*) + report "block mount-through-mount $(oneline "$m") $(oneline "$om")" ;; + esac + done < "$OUT/.mounts-forms" + done < "$OUT/.mseen" + mr="${mr%x}" + [ "$mr" = "$m" ] || printf '%s\n' "$mr" >> "$OUT/.mounts" + # A mount that lands in the volume (through a link on its + # path) hides part of it in the container; one that holds the + # volume is not this layout either. + for mm in "$m" "$mr"; do + case "$mm" in + "$ROOT"|"$ROOT"/*) + report "block mount-in-workspace $(oneline "$m")" ;; + esac + case "$ROOT" in + "$mm"/*) + report "block mount-around-workspace $(oneline "$m")" ;; + esac + done + done < "$OUT/.inc" 2>/dev/null \ + || rc=$? + case "$rc" in + 0) report "block config-include ${rel}" ;; + 1) ;; + *) fail "cannot parse ${cf}" ;; + esac + # Custom hooks live wherever the setting says, resolved + # against a base that depends on the repository layout; the + # engine never sets it, so any value blocks. + rc=0 + git config -f "$cf" --get core.hooksPath >/dev/null 2>&1 \ + || rc=$? + case "$rc" in + 0) report "block core-hookspath ${rel}" ;; + 1) ;; + *) fail "cannot parse ${cf}" ;; + esac + # core.worktree is relative to the git dir; submodule git + # dirs set it to their checkout inside ROOT. + rc=0 + wtv=$(git config -f "$cf" --get core.worktree 2>/dev/null \ + && printf x) || rc=$? + # git prints the value and a newline; any other newline is + # part of the value. + wtv="${wtv%x}" + wtv="${wtv%$'\n'}" + case "$wtv" in + *$'\n'*) + report "block core-worktree ${rel} (newline in path)" + continue ;; + esac + case "$rc" in + 0) + case "$wtv" in + /*) ;; + *) wtv="$d/$wtv" ;; + esac + wtc=$(canon_nm "$wtv") || wtc="" + case "$wtc" in + "$ROOT"|"$ROOT"/*) ;; + *) report "block core-worktree ${rel} ${wtv}" ;; + esac ;; + 1) ;; + *) fail "cannot parse ${cf}" ;; + esac + done + done < "$GITDIRS" + # Every symlink anywhere in the volume (checkout, git dirs, object + # pools named by alternates) is resolved to where it really leads. + # Links to directories inside ROOT are fine: their contents are in + # ROOT and this walk visits them where they are. A link that leads + # out of ROOT is reported as "layer-link ": whether that path + # holds the container's own data only the container can tell + # (retire compares it with docker diff). A loop blocks. + find "$ROOT" -type l -print0 2>"$ERR" > "$OUT/.links" \ + || fail "cannot list ${ROOT}" + [ ! -s "$ERR" ] || fail "cannot list ${ROOT}: $(cat "$ERR")" + while IFS= read -r -d '' l; do + : > "$OUT/.via" + : > "$OUT/.seen" + # The x keeps a trailing newline of the path. + rp_rc=0 + lt=$(real_path "$l" "$OUT/.via" "$OUT/.seen" && printf x) || rp_rc=$? + case "$rp_rc" in + 0) ;; + 2) + report "block newline-path $(oneline "${l#"$ROOT"/}")" + continue ;; + *) + # Resolution stopped (a loop, or a link it cannot read). + # If it had reached a mount the snapshot lacks, say so. + hit="" + while IFS= read -r v; do + while IFS= read -r m; do + [ -n "$m" ] || continue + case "$v" in + "$m"|"$m"/*) hit="$m" ;; + esac + done < "$OUT/.mounts" + done < "$OUT/.seen" + if [ -n "$hit" ]; then + report "block mount-crossing $(oneline "${l#"$ROOT"/}") $(oneline "$hit")" + else + report "block link-loop $(oneline "${l#"$ROOT"/}")" + fi + continue ;; + esac + lt="${lt%x}" + out_of_root="" + while IFS= read -r v; do + case "$v" in + "$ROOT"|"$ROOT"/*) ;; + *) out_of_root=yes ;; + esac + done < "$OUT/.seen" + case "$lt" in + *$'\n'*) + report "block newline-path $(oneline "${l#"$ROOT"/}")" + continue ;; + esac + case "$lt" in + "$ROOT"|"$ROOT"/*) ;; + *) + # A directory outside ROOT can hold further links that + # lead anywhere; only a file (or a path that does not + # exist here) is compared with the container's changes. + if [ -d "$lt" ]; then + report "block outside-dir-link $(oneline "${l#"$ROOT"/}")" + continue + fi + report "layer-link $(oneline "$lt")" ;; + esac + # The chain's steps outside ROOT, also when it ends inside it: + # links passed (layer-via, compared with the container's + # changes) and every path visited (layer-path, compared with its + # other mounts, whose contents the snapshot lacks). + [ -n "$out_of_root" ] || continue + while IFS= read -r v; do + case "$v" in + "$ROOT"|"$ROOT"/*) ;; + *) report "layer-via $(oneline "$v")" ;; + esac + done < "$OUT/.via" + while IFS= read -r v; do + case "$v" in + "$ROOT"|"$ROOT"/*) ;; + *) + report "layer-path $(oneline "$v")" + while IFS= read -r m; do + [ -n "$m" ] || continue + case "$v" in + "$m"|"$m"/*) + report "block mount-crossing $(oneline "${l#"$ROOT"/}") $(oneline "$m")" ;; + esac + done < "$OUT/.mounts" ;; + esac + done < "$OUT/.seen" + done < "$OUT/.links" + + # Object stores that borrow from elsewhere: every alternates entry, + # followed through the alternates of the stores it names, must stay + # inside ROOT. + : > "$OUT/.alt-seen" + while IFS= read -r d; do + [ -d "$d/objects" ] && printf '%s\n' "$d/objects" + done < "$GITDIRS" > "$OUT/.alt-queue" + while [ -s "$OUT/.alt-queue" ]; do + od=$(head -1 "$OUT/.alt-queue") + tail -n +2 "$OUT/.alt-queue" > "$OUT/.alt-q2" + mv "$OUT/.alt-q2" "$OUT/.alt-queue" + grep -qxF "$od" "$OUT/.alt-seen" && continue + printf '%s\n' "$od" >> "$OUT/.alt-seen" + [ -f "$od/info/alternates" ] || continue + if has_nul "$od/info/alternates"; then + report "block alternates $(oneline "${od#"$ROOT"/}") (unreadable or NUL byte)" + continue + fi + # The last entry counts even without a final newline. + while IFS= read -r alt || [ -n "$alt" ]; do + case "$alt" in + ''|'#'*) continue ;; + '"'*) + # Git decodes a quoted entry (C-style escapes); this + # scan does not, so it cannot tell where it points. + report "block alternates $(oneline "${od#"$ROOT"/}") $(oneline "$alt")" + continue ;; + /*) ;; + *) alt="$od/$alt" ;; + esac + ac=$(canon_nm "$alt") || ac="" + case "$ac" in + "$ROOT"/*) printf '%s\n' "$ac" >> "$OUT/.alt-queue" ;; + *) report "block alternates $(oneline "${od#"$ROOT"/}") $(oneline "$alt")" ;; + esac + done < "$od/info/alternates" + done +fi + # Engine files and the deletions the context mode makes: in the # workspace's own checkout ($2 = "."), where the engine writes those @@ -274,12 +759,16 @@ if [ "$MODE" = worktree ]; then continue ;; esac if [ -f "$e" ] && [ ! -L "$e" ]; then + if has_nul "$e"; then + report "block embedded $(oneline "$rel") (unreadable or NUL byte in .git)" + continue + fi target=$(sed -n 's/^gitdir: //p' "$e") case "$target" in /*) ;; *) target="$wt/$target" ;; esac - target=$(canon "$target") || target="" + target=$(canon_nm "$target") || target="" case "$target" in "$TOP"/worktrees/*) # A linked worktree's own admin dir (one level below @@ -297,6 +786,10 @@ if [ "$MODE" = worktree ]; then # checkout; a copied worktree dir points # at another one and is not checked # there. + if has_nul "$target/gitdir"; then + report "block mismatched-worktree $(oneline "$rel")" + continue + fi bp=$(cat "$target/gitdir") case "$bp" in /*) ;; @@ -349,6 +842,10 @@ if [ "$MODE" = worktree ]; then # outside it (a container's /tmp) their files are not in the volume. while IFS= read -r d; do [ -f "$d/gitdir" ] || continue + if has_nul "$d/gitdir"; then + report "block missing-worktree $(oneline "${d#"$ROOT"/}") (unreadable or NUL byte)" + continue + fi wtgit=$(cat "$d/gitdir") # The back-pointer is relative to this admin dir when the # worktree was added with --relative-paths. @@ -357,7 +854,7 @@ if [ "$MODE" = worktree ]; then *) wtgit="$d/$wtgit" ;; esac wt="${wtgit%/.git}" - if ! wt=$(canon "$wt"); then + if ! wt=$(canon_nm "$wt"); then report "block missing-worktree ${wtgit%/.git}" continue fi @@ -406,6 +903,7 @@ while IFS= read -r d; do if [ -d "$d/objects" ]; then store="$d" else + has_nul "$d/commondir" && fail "cannot read ${d}/commondir" common=$(cat "$d/commondir") case "$common" in /*) store="$common" ;; diff --git a/lib/preserve.sh b/lib/preserve.sh index e5dfb8e..9b7fa27 100644 --- a/lib/preserve.sh +++ b/lib/preserve.sh @@ -429,20 +429,54 @@ swarm_scan_host() { "$out" } +# Run docker ARGS in a process group of its own and wait for it. A +# Ctrl-C at the terminal then reaches the caller (whose cleanup waits +# for this job: wait with no operand) but not the docker client, which, +# stopped mid-request, can leave the daemon to make a container or an +# image after that cleanup ran. Its standard input is the caller's, +# or /dev/null when that is a terminal. +swarm_docker_own_group() { + local rc=0 had_m=false + case "$-" in + *m*) had_m=true ;; + esac + set -m + if [ -t 0 ]; then + docker "$@" < /dev/null & + else + docker "$@" & + fi + if [ "$had_m" = false ]; then + set +m + fi + wait "$!" || rc=$? + return "$rc" +} + +# More docker run arguments for swarm_scan_volume (retire's helper +# label); empty by default. +SWARM_SCAN_DOCKER_ARGS=() + # Scan workspace volume VOL with image IMAGE (context CONTEXT) into the # empty directory OUT: the scanner runs in a throwaway container with # the volume read-only and no network, as root so it can read every -# file, and streams its results back as a tar archive. +# file, and streams its results back as a tar archive. MOUNTS (optional, +# one path per line) names the scanned container's other mount points. swarm_scan_volume() { - local vol="$1" image="$2" context="$3" out="$4" rc=0 - docker run --rm -i --pull never --network none --user 0 \ + local vol="$1" image="$2" context="$3" out="$4" mounts="${5:-}" rc=0 + swarm_docker_own_group run --rm -i --pull never --network none --user 0 \ + "${SWARM_SCAN_DOCKER_ARGS[@]+"${SWARM_SCAN_DOCKER_ARGS[@]}"}" \ + -e "SWARM_SCAN_MOUNTS=${mounts}" \ --entrypoint bash \ --mount "type=volume,src=${vol},dst=/workspace,readonly,volume-nocopy" \ "$image" -c ' - cat > /tmp/scan.sh || exit 3 - mkdir /tmp/out || exit 3 - bash /tmp/scan.sh worktree /workspace "$1" /tmp/out >&2 || exit $? - cd /tmp/out || exit 3 + # The image may be a snapshot of a container with its own + # files anywhere, so the scratch space is a new directory. + d=$(mktemp -d) || exit 3 + cat > "$d/scan.sh" || exit 3 + mkdir "$d/out" || exit 3 + bash "$d/scan.sh" worktree /workspace "$1" "$d/out" >&2 || exit $? + cd "$d/out" || exit 3 set -- report [ ! -f bundle ] || set -- "$@" bundle [ ! -f objects ] || set -- "$@" objects diff --git a/lib/project.sh b/lib/project.sh index 1f7344c..e00ca9d 100644 --- a/lib/project.sh +++ b/lib/project.sh @@ -400,6 +400,22 @@ SWARM_CHECKOUT_LABEL="swarm-core.checkout" # An interactive session's container carries its record's name here: # a running container can be renamed, but not relabeled. SWARM_SESSION_LABEL="swarm-core.session" +# Containers and workspace volumes carry the engagement they belong to. +# A workspace volume also names its container, the kind of work in it +# (agent: numbered agents and post-process; interactive: a human +# session), the context mode of that container, and a random nonce that +# proves this engine made it. The scripts that source this file use +# them. +# shellcheck disable=SC2034 +SWARM_ENGAGEMENT_LABEL="swarm-core.engagement" +# shellcheck disable=SC2034 +SWARM_CONTAINER_LABEL="swarm-core.container" +# shellcheck disable=SC2034 +SWARM_KIND_LABEL="swarm-core.kind" +# shellcheck disable=SC2034 +SWARM_CONTEXT_LABEL="swarm-core.context" +# shellcheck disable=SC2034 +SWARM_NONCE_LABEL="swarm-core.nonce" # docker ps format and jq definitions for the project's containers. A # container is the project's when its project label is the id, diff --git a/lib/retire.sh b/lib/retire.sh new file mode 100644 index 0000000..6e5605e --- /dev/null +++ b/lib/retire.sh @@ -0,0 +1,859 @@ +#!/bin/bash +# launch.sh retire: remove this checkout's stopped engine containers +# without losing anything. A container's /workspace is a Docker volume +# that stays; what dies with the container is its own layer, so retire +# refuses any container whose layer may hold work: one that runs or may +# still start, one whose /workspace is not exactly its volume, one with +# git settings that put repository data outside the volume, one whose +# workspace registers a worktree or links to data outside the volume, +# and one with a Git repository anywhere outside the volume. Every +# selected container is checked before any is removed. +# +# Sourced by launch.sh after lib/project.sh and lib/preserve.sh; uses +# PROJECT, CHECKOUT_PATH, RUNTIME_DIR, STATE_FILE and IMAGE_NAME. + +# What an interrupted retire (Ctrl-C, a timeout's SIGTERM) would leave +# behind: the scratch directory every check writes under, the snapshot +# in use (named before docker commit runs), and containers made from +# it. Every docker command that makes something runs in a process +# group of its own (swarm_docker_own_group): a Ctrl-C does not stop +# the client mid-request, and the cleanup waits for it first. +RETIRE_SCRATCH="" +RETIRE_SNAP_REF="" +# Retire's own helper containers carry this label (with the project), +# so that its sweep removes nothing else made from a snapshot. +RETIRE_HELPER_LABEL="swarm-core.retire" + +# Print a new directory under RETIRE_SCRATCH (or the system's temporary +# directory when retire's own is not set up). The template is explicit: +# macOS mktemp ignores TMPDIR without one. +retire_tmpdir() { + if [ -n "$RETIRE_SCRATCH" ]; then + mktemp -d "$RETIRE_SCRATCH/check.XXXXXX" + else + mktemp -d "${TMPDIR:-/tmp}/swarm-retire.XXXXXX" + fi +} + +# Remove the helper containers retire made from image REF (the checks +# run them with --rm; one a stopped client left behind would keep the +# image). Only those with retire's label go; anything else made from +# the snapshot (a container someone started to look into it) stays, +# and with it the snapshot. +retire_drop_containers() { + local ref="$1" c other + if [ -z "$ref" ]; then + return 0 + fi + for c in $(docker ps -aq --filter "ancestor=${ref}" \ + --filter "label=${RETIRE_HELPER_LABEL}=${PROJECT}" 2>/dev/null); do + docker rm -f "$c" >/dev/null 2>&1 || true + done + other=$(docker ps -aq --filter "ancestor=${ref}" 2>/dev/null) \ + || other="" + if [ -n "$other" ]; then + echo "WARNING: containers made from ${ref} that retire did not" \ + "start stay, and so does ${ref}:" \ + "$(printf '%s' "$other" | tr '\n' ' ')" >&2 + fi +} + +# Remove the snapshot RETIRE_SNAP_REF names, if any. Fails when it is +# still there. The name is forgotten only after the attempt, so that a +# signal in between leaves the cleanup something to remove (doing it +# twice is harmless). +retire_drop_snapshot() { + local ref="$RETIRE_SNAP_REF" rc=0 + if [ -z "$ref" ]; then + return 0 + fi + # Gone when docker rmi works, or when there is no such image (never + # made, or removed already). + if ! docker rmi "$ref" >/dev/null 2>&1 \ + && docker image inspect "$ref" >/dev/null 2>&1; then + rc=1 + fi + RETIRE_SNAP_REF="" + return "$rc" +} + +# Remove the snapshots an earlier retire left (one killed outright, so +# that its cleanup never ran). Under the engagement lock no other +# retire of this project runs, so every image of ${IMAGE_NAME}-retire +# is such a leftover. +retire_sweep_snapshots() { + local refs ref + if ! refs=$(docker images --format '{{.Repository}}:{{.Tag}}' \ + --filter "reference=${IMAGE_NAME}-retire" 2>/dev/null); then + echo "ERROR: cannot list Docker images." >&2 + return 1 + fi + # The list comes on descriptor 3 and nothing in the loop reads + # standard input: a command that does (docker run -i) cannot eat it. + while IFS= read -r -u 3 ref; do + if [ -z "$ref" ]; then + continue + fi + retire_drop_containers "$ref" + if docker rmi "$ref" >/dev/null 2>&1; then + echo "Removed ${ref}, a snapshot an earlier retire left." + else + echo "WARNING: cannot remove ${ref}, a snapshot an earlier" \ + "retire left." >&2 + fi + done 3</dev/null || true + retire_drop_containers "$RETIRE_SNAP_REF" + retire_drop_snapshot || true + if [ -n "$RETIRE_SCRATCH" ]; then + rm -rf "$RETIRE_SCRATCH" + RETIRE_SCRATCH="" + fi +} + +# Print, for container ID named NAME, one line +# +# when it may be removed, or say why not on stderr and fail. +swarm_retire_preflight() { + local id="$1" name="$2" info verdict vol kind context live tmp + local started finished vlabels + info=$(docker container inspect "$id" 2>/dev/null) || { + echo "ERROR: cannot inspect container ${name}." >&2 + return 1 + } + # One verdict from the container's own description: its state, its + # restart and auto-remove settings, and the mounts at and below + # /workspace. + verdict=$(printf '%s' "$info" | jq -r --arg name "$name" ' + .[0] as $c + | ($c.State.Status // "") as $st + | [($c.Mounts // [])[] | select(.Destination == "/workspace" + or (.Destination | startswith("/workspace/")))] as $ws + | (($c.HostConfig.Tmpfs // {}) | keys + | map(select(. == "/workspace" or startswith("/workspace/")))) + as $tmpfs + | if ($st != "exited" and $st != "created") then + "state " + $st + elif (($c.HostConfig.RestartPolicy.Name // "") as $r + | $r != "" and $r != "no") then + "restart policy " + $c.HostConfig.RestartPolicy.Name + elif ($c.HostConfig.AutoRemove // false) then + "auto-remove is on" + elif ($tmpfs | length) > 0 then + "tmpfs at " + ($tmpfs | join(", ")) + elif ($ws | length) != 1 then + "no single workspace mount (" + ($ws | length | tostring) + + " mounts at or below /workspace)" + elif $ws[0].Destination != "/workspace" + or $ws[0].Type != "volume" then + "workspace is not a volume at /workspace" + elif ([($c.HostConfig.Mounts // [])[] + | select(.Target == "/workspace") + | (.VolumeOptions.Subpath // "")] | map(select(. != "")) + | length) > 0 then + "the workspace mounts a subdirectory of its volume" + elif ([($c.Mounts // [])[] | .Destination] + + (($c.HostConfig.Tmpfs // {}) | keys) + | map(select(test("[\u0000-\u001f\u007f]"))) | length) > 0 then + "a mount destination holds a control character" + else + "ok " + $ws[0].Name + " " + ($c.State.StartedAt // "") + + " " + ($c.State.FinishedAt // "") + end') || verdict="unreadable" + case "$verdict" in + "ok "*) ;; + *) + echo "ERROR: container ${name} cannot be retired: ${verdict}." >&2 + case "$verdict" in + "restart policy"*) + echo " docker update --restart no ${id}" >&2 ;; + "no single workspace mount"*|"workspace is not"*) + echo " Its files live in its own layer. Copy what you" \ + "need (docker cp ${id}:/workspace ), then" \ + "docker rm ${id}." >&2 ;; + esac + return 1 ;; + esac + read -r _ vol started finished </dev/null) || vlabels="" + if ! printf '%s' "$vlabels" | jq -e \ + --arg pk "$SWARM_PROJECT_LABEL" --arg p "$PROJECT" \ + --arg ck "$SWARM_CHECKOUT_LABEL" --arg c "$CHECKOUT_PATH" \ + --arg nk "$SWARM_CONTAINER_LABEL" --arg n "$name" ' + .[$pk] == $p and .[$ck] == $c and .[$nk] == $n' \ + >/dev/null 2>&1; then + echo "ERROR: the /workspace volume of ${name} (${vol}) is not" \ + "its workspace volume." >&2 + return 1 + fi + kind=$(printf '%s' "$vlabels" | jq -r --arg k "$SWARM_KIND_LABEL" \ + '.[$k] // ""') + context=$(printf '%s' "$vlabels" | jq -r --arg k "$SWARM_CONTEXT_LABEL" \ + '.[$k] // ""') + # An interactive session that is open (or may still start) keeps its + # container. + if [ "$kind" = interactive ]; then + live=$(swarm_pending_live "$RUNTIME_DIR") || return 1 + # A here-string, not a pipe: under pipefail, grep -q leaving + # early can fail the writer and hide the match. + if grep -qxF -- "$name" <<< "$live"; then + echo "ERROR: the interactive session ${name} is open." >&2 + return 1 + fi + fi + # The checks below run from a snapshot of this very container + # (docker commit: its image plus its own layer), so links and files + # resolve as they would inside it. The snapshot is removed after. + local snap scan_rc=0 + tmp=$(retire_tmpdir) || return 1 + # The container is stopped, so there is nothing to pause. The + # snapshot is named before docker commit runs, so that an + # interrupted retire can still remove it (retire_cleanup). Some + # Docker versions print notices on stdout; only the image id line + # counts. + RETIRE_SNAP_REF="${IMAGE_NAME}-retire:$(printf '%.12s' "$id")-$$" + # The daemon finishes an image even when its client stops, so the + # commit runs in a process group of its own (a Ctrl-C does not reach + # it) and retire_cleanup waits for it. The engine's labels on the + # snapshot are emptied: a check container that outlived retire must + # not pass for one of the engine's. + local strip + strip="LABEL ${SWARM_PROJECT_LABEL}=\"\" ${SWARM_CHECKOUT_LABEL}=\"\"" + strip+=" ${SWARM_SESSION_LABEL}=\"\"" + if ! swarm_docker_own_group commit --change "$strip" \ + "$id" "$RETIRE_SNAP_REF" < /dev/null > "$tmp/commit" \ + 2>"$tmp/err"; then + echo "ERROR: cannot snapshot ${name} for its scan:" \ + "$(cat "$tmp/err")" >&2 + retire_drop_snapshot || true + rm -rf "$tmp" + return 1 + fi + snap=$(grep -E '^sha256:[0-9a-f]{64}$' "$tmp/commit" | tail -1) + if [ -z "$snap" ]; then + echo "ERROR: docker commit of ${name} printed no image id." >&2 + retire_drop_snapshot || true + rm -rf "$tmp" + return 1 + fi + local mounts + mounts=$(printf '%s' "$info" | jq -r '.[0] as $c + | [($c.Mounts // [])[] | .Destination] + + (($c.HostConfig.Tmpfs // {}) | keys) + + ["/etc/hosts", "/etc/hostname", "/etc/resolv.conf", + "/dev", "/proc", "/sys"] + | unique | .[] | select(. != "/workspace")') || { + echo "ERROR: cannot read the mounts of ${name}." >&2 + retire_drop_snapshot || true + rm -rf "$tmp" + return 1 + } + # Git settings in the container's global and system config can keep + # repository data outside the volume. + if ! swarm_retire_config_check "$id" "$name" "$snap" "$mounts"; then + retire_drop_snapshot || true + rm -rf "$tmp" + return 1 + fi + # The scan runs from the snapshot too, so it carries retire's label. + # shellcheck disable=SC2034 # read by swarm_scan_volume + SWARM_SCAN_DOCKER_ARGS=(--label "${RETIRE_HELPER_LABEL}=${PROJECT}") + swarm_scan_volume "$vol" "$snap" "${context:-full}" "$tmp" "$mounts" \ + 2>"$tmp/err" || scan_rc=$? + # shellcheck disable=SC2034 + SWARM_SCAN_DOCKER_ARGS=() + # The snapshot is still needed below; every path from here removes + # it before returning. + local ok=0 + swarm_retire_after_scan "$id" "$name" "$snap" "$info" "$tmp" "$scan_rc" \ + || ok=$? + if ! retire_drop_snapshot; then + echo "WARNING: cannot remove the scan snapshot ${snap} of" \ + "${name}; remove it with docker rmi ${snap}." >&2 + fi + rm -rf "$tmp" + [ "$ok" -eq 0 ] || return 1 + printf '%s %s %s %s %s\n' "$id" "$name" "$vol" "$started" "$finished" +} + +# The checks on scan output in TMP for container ID (snapshot SNAP, +# inspect JSON INFO; SCAN_RC is the scan's status). Fails when the +# container may not be retired. +swarm_retire_after_scan() { + local id="$1" name="$2" snap="$3" info="$4" tmp="$5" scan_rc="$6" + if [ "$scan_rc" -ne 0 ]; then + echo "ERROR: cannot scan the workspace of ${name}:" >&2 + sed 's/^/ /' "$tmp/err" >&2 + return 1 + fi + # Any blocking finding stops retire: some put repository data in the + # container's layer (worktrees elsewhere, a .git link or gitfile, + # LFS storage or config pointing elsewhere, links out of a git dir), + # and the others (nested repositories, files without a repository) + # are not inspected deeply enough to rule that out. Uncommitted + # files in the volume stay with the volume. + local grc=0 + grep -a '^block ' "$tmp/report" > "$tmp/blocks" 2>/dev/null || grc=$? + case "$grc" in + 0) + echo "ERROR: ${name} cannot be retired; its workspace scan" \ + "found:" >&2 + sed 's/^block / /' "$tmp/blocks" >&2 + echo " Data outside /workspace is in the container: copy it" \ + "out (docker cp ${id}: ) or move it into" \ + "/workspace first." >&2 + return 1 ;; + 1) ;; + *) + echo "ERROR: cannot read the workspace scan of ${name}." >&2 + return 1 ;; + esac + # Repositories the container made or changed outside the volume + # (a clone in /tmp, commits in a repository of the image) are in its + # layer only. + swarm_retire_layer_repos "$id" "$name" "$snap" || return 1 + # Symlinks in the volume that lead out of it: fine when they point + # into the image (a virtualenv's python), not when the target, or + # anything below it, is something the container added or changed. + swarm_retire_layer_links "$id" "$name" "$tmp/report" "$info" "$snap" +} + +# Write docker diff of container ID to file OUT, failing when it cannot +# be read or when a name may not be the real one: docker prints paths +# unescaped, so a name with a newline would split into lines that look +# like something else (any line not " /"), and Docker sends +# them as JSON, which turns bytes that are not UTF-8 into U+FFFD. +swarm_retire_diff() { + local id="$1" name="$2" out="$3" + if ! docker diff "$id" > "$out" 2>/dev/null; then + echo "ERROR: cannot list what ${name} changed (docker diff)." >&2 + return 1 + fi + local rc=0 + grep -avE '^[ACD] /' "$out" > /dev/null 2>&1 || rc=$? + case "$rc" in + 0) + echo "ERROR: ${name} changed paths whose names docker diff" \ + "cannot report unambiguously (a newline in a name)." >&2 + return 1 ;; + 1) ;; + *) + echo "ERROR: cannot read what ${name} changed." >&2 + return 1 ;; + esac + rc=0 + grep -aF "$(printf '\357\277\275')" "$out" > /dev/null 2>&1 || rc=$? + case "$rc" in + 0) + echo "ERROR: ${name} changed paths whose names docker diff" \ + "cannot report exactly (not valid UTF-8)." >&2 + return 1 ;; + 1) ;; + *) + echo "ERROR: cannot read what ${name} changed." >&2 + return 1 ;; + esac +} + +# Fail when snapshot SNAP of container ID holds a Git repository +# outside /workspace (the volume, scanned on its own): a .git entry of +# any type, or a directory with a HEAD entry next to objects or +# commondir (a bare repository, or a git dir kept apart from its work +# tree). One the container made (a clone in /tmp) holds work only it +# has. One the image ships may hold commits, tags, objects or edits +# the container added, and docker diff cannot be trusted to show them: +# it goes by metadata, which a write can keep, and an objects/ that is +# a link puts new objects outside the repository. So any repository +# refuses, changed or not. +swarm_retire_layer_repos() { + local id="$1" name="$2" snap="$3" tmp rc=0 + tmp=$(retire_tmpdir) || return 1 + swarm_docker_own_group run --rm --pull never --network none --user 0 \ + --label "${RETIRE_HELPER_LABEL}=${PROJECT}" \ + --entrypoint bash "$snap" -c ' + # swarm-repos: every Git repository under $1 (the root of + # the snapshot) but outside $1/workspace. + set -o pipefail + r="${1%/}" + e=$(mktemp "${TMPDIR:-/tmp}/swarm-repos.XXXXXX") || exit 3 + s=0 + find "${r:-/}" -xdev -path "$r/workspace" -prune \ + -o -name .git -print0 -prune -o -name HEAD -print0 \ + 2>"$e" | + while IFS= read -r -d "" f; do + case "$f" in + */.git) printf "%s\n" "$f" ;; + *) + d="${f%/HEAD}" + if [ -e "$d/objects" ] || [ -L "$d/objects" ] \ + || [ -e "$d/commondir" ] \ + || [ -L "$d/commondir" ]; then + printf "%s\n" "${d:-/}" + fi ;; + esac + done || s=3 + if [ -s "$e" ]; then + cat "$e" >&2 + s=3 + fi + rm -f "$e" + exit "$s" + ' swarm-repos / < /dev/null > "$tmp/repos.raw" 2>"$tmp/err" \ + || rc=$? + if [ "$rc" -ne 0 ]; then + echo "ERROR: cannot check ${name} for Git repositories:" \ + "$(cat "$tmp/err")" >&2 + rm -rf "$tmp" + return 1 + fi + if ! sort -u "$tmp/repos.raw" > "$tmp/repos"; then + echo "ERROR: cannot check ${name} for Git repositories." >&2 + rm -rf "$tmp" + return 1 + fi + if [ -s "$tmp/repos" ]; then + echo "ERROR: ${name} has Git repositories outside its workspace" \ + "volume, in its own layer or in its image; retire cannot" \ + "tell whether it changed them:" >&2 + sed 's/^/ /' "$tmp/repos" >&2 + echo " Push or copy out what they hold (docker cp" \ + "${id}: ); then, if nothing of yours is left in" \ + "them, remove the container yourself (docker rm ${id})." >&2 + rm -rf "$tmp" + return 1 + fi + rm -rf "$tmp" +} + +# Fail when scan REPORT names, as a layer-link, a path that container +# ID added or changed (docker diff), or one with such a path below it; +# or when a layer-link or layer-via path lies at or below one of the +# container's other mounts or tmpfs, including the files Docker mounts +# itself (/etc/hosts, /etc/hostname, /etc/resolv.conf; INFO is its +# inspect JSON): the +# scan's snapshot does not have their contents, so it cannot tell where +# links there lead. +swarm_retire_layer_links() { + local id="$1" name="$2" report="$3" info="$4" snap="$5" tmp grc=0 + tmp=$(retire_tmpdir) || return 1 + grep -aE '^layer-(link|via|path) ' "$report" > "$tmp/links" 2>/dev/null \ + || grc=$? + case "$grc" in + 0) ;; + 1) rm -rf "$tmp"; return 0 ;; + *) + echo "ERROR: cannot read the workspace scan of ${name}." >&2 + rm -rf "$tmp" + return 1 ;; + esac + # Escaped names (a newline shows as \n) cannot be compared with + # docker diff's paths. + if grep -aq '[\]' "$tmp/links"; then + echo "ERROR: ${name} has links out of its volume whose paths" \ + "cannot be compared; check them first:" >&2 + sed 's/^layer-[a-z]* / /' "$tmp/links" >&2 + rm -rf "$tmp" + return 1 + fi + if ! printf '%s' "$info" | jq -r '.[0] as $c + | [($c.Mounts // [])[] | .Destination] + + (($c.HostConfig.Tmpfs // {}) | keys) + + ["/etc/hosts", "/etc/hostname", "/etc/resolv.conf", + "/dev", "/proc", "/sys"] + | unique | .[] | select(. != "/workspace")' > "$tmp/mounts"; then + echo "ERROR: cannot read the mounts of ${name}." >&2 + rm -rf "$tmp" + return 1 + fi + if ! awk 'NR == FNR { m[$0] = 1; next } + { + p = $0; sub(/^layer-(link|via|path) /, "", p) + for (x in m) + if (p == x || index(p, x "/") == 1) print p " (mount " x ")" + }' "$tmp/mounts" "$tmp/links" > "$tmp/mhits"; then + echo "ERROR: cannot compare the links of ${name} with its mounts." >&2 + rm -rf "$tmp" + return 1 + fi + if [ -s "$tmp/mhits" ]; then + echo "ERROR: ${name} has links out of its volume into its other" \ + "mounts, which the scan cannot follow:" >&2 + sed 's/^/ /' "$tmp/mhits" >&2 + rm -rf "$tmp" + return 1 + fi + # docker diff comes through JSON, which replaces bytes that are not + # valid UTF-8; such a path cannot be compared exactly. + if ! command -v iconv >/dev/null 2>&1; then + echo "ERROR: retire needs iconv to compare the links of ${name}" \ + "with what it changed." >&2 + rm -rf "$tmp" + return 1 + fi + if ! iconv -f UTF-8 -t UTF-8 < "$tmp/links" > /dev/null 2>&1; then + echo "ERROR: ${name} has links out of its volume whose paths are" \ + "not valid UTF-8, which docker diff cannot report exactly:" >&2 + sed 's/^layer-[a-z]* / /' "$tmp/links" >&2 + rm -rf "$tmp" + return 1 + fi + if ! swarm_retire_diff "$id" "$name" "$tmp/diff"; then + rm -rf "$tmp" + return 1 + fi + # A layer-link target, or anything below it, that the container + # added or changed; or a link on the way that it added or changed. + # A layer-link target, or anything below it, that the container + # added or changed; a link on the way (layer-via) that it added or + # changed; or a directory on the way (layer-path) that it added. + if ! awk 'NR == FNR { + if (substr($0, 1, 11) == "layer-link ") t[substr($0, 12)] = 1 + if (substr($0, 1, 10) == "layer-via ") v[substr($0, 11)] = 1 + if (substr($0, 1, 11) == "layer-path ") w[substr($0, 12)] = 1 + next + } + ($1 == "A" || $1 == "C") { + p = substr($0, 3) + for (x in t) + if (x == "/" || p == x || index(p, x "/") == 1) + print x " (" $0 ")" + if (p in v) print p " (" $0 ")" + if ($1 == "A" && (p in w)) print p " (" $0 ")" + }' "$tmp/links" "$tmp/diff" > "$tmp/hits.raw" \ + || ! sort -u "$tmp/hits.raw" > "$tmp/hits"; then + echo "ERROR: cannot compare the links of ${name} with what it" \ + "changed." >&2 + rm -rf "$tmp" + return 1 + fi + if [ -s "$tmp/hits" ]; then + echo "ERROR: ${name} has links out of its volume to data in the" \ + "container:" >&2 + sed 's/^/ /' "$tmp/hits" >&2 + echo " Copy it out with docker cp ${id}: first." >&2 + rm -rf "$tmp" + return 1 + fi + # docker diff goes by metadata: a file whose content changed with its + # size, mode and times kept does not show. The link targets and the + # links on the way must read the same in the snapshot as in the + # image the container was made from. + if ! swarm_retire_same_as_image "$name" "$snap" \ + "$(printf '%s' "$info" | jq -r '.[0].Image // empty')" \ + "$tmp/links"; then + rm -rf "$tmp" + return 1 + fi + rm -rf "$tmp" +} + +# Fail unless every layer-link and layer-via path in file LINKS reads the +# same in image SNAP (the container's snapshot) as in image ORIG (its +# original image): a file by its content's SHA-256, a symlink by its +# target, anything else (or nothing) by its kind. +swarm_retire_same_as_image() { + local name="$1" snap="$2" orig="$3" links="$4" tmp img rc + case "$orig" in + sha256:*) ;; + *) + echo "ERROR: cannot tell the image of ${name}." >&2 + return 1 ;; + esac + tmp=$(retire_tmpdir) || return 1 + if ! sed -n -e 's/^layer-link //p' -e 's/^layer-via //p' "$links" \ + > "$tmp/paths"; then + echo "ERROR: cannot list the links of ${name} to compare." >&2 + rm -rf "$tmp" + return 1 + fi + if [ ! -s "$tmp/paths" ]; then + rm -rf "$tmp" + return 0 + fi + for img in "$snap" "$orig"; do + rc=0 + swarm_docker_own_group run --rm -i --pull never --network none \ + --user 0 --label "${RETIRE_HELPER_LABEL}=${PROJECT}" \ + --entrypoint bash "$img" -c ' + # swarm-compare: one line per path read on stdin + while IFS= read -r p; do + if [ -L "$p" ]; then + printf "%s\tl\t%s\n" "$p" "$(readlink -n "$p")" + elif [ -f "$p" ]; then + h=$(sha256sum < "$p") || exit 3 + printf "%s\tf\t%s\n" "$p" "${h%% *}" + elif [ -e "$p" ]; then + printf "%s\to\t-\n" "$p" + else + printf "%s\t-\t-\n" "$p" + fi + done + ' < "$tmp/paths" > "$tmp/out.${img#sha256:}" 2>"$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "ERROR: cannot compare the links of ${name} with its" \ + "image: $(cat "$tmp/err")" >&2 + rm -rf "$tmp" + return 1 + fi + done + if ! cmp -s "$tmp/out.${snap#sha256:}" "$tmp/out.${orig#sha256:}"; then + echo "ERROR: ${name} has links out of its volume to files it" \ + "changed:" >&2 + diff "$tmp/out.${orig#sha256:}" "$tmp/out.${snap#sha256:}" \ + | sed -n 's/^> / /p' >&2 + rm -rf "$tmp" + return 1 + fi + rm -rf "$tmp" +} + +# Fail when the git config that container ID's processes read (global +# and system) sets what puts repository data outside its volume, +# includes other files, or cannot be checked. Git environment variables +# other than the engine's own (GIT_USER_NAME, GIT_USER_EMAIL) can +# redirect any of this, so they refuse; so do HOME and XDG_CONFIG_HOME +# values that are empty, relative, unnormalized or hold a newline. The +# files are then found and read inside snapshot SNAP, as the container's +# user with its HOME and XDG_CONFIG_HOME (Docker derives HOME from that +# user's passwd entry when it is not set), with every path resolved +# through the container's own links; a file on one of the container's +# mounts (MOUNTS) or Docker's own (/dev, /proc, /sys, ...) refuses, since +# the snapshot lacks those contents. +swarm_retire_config_check() { + local id="$1" name="$2" snap="$3" mounts="$4" env bad user tmp rc=0 + local args=() + env=$(docker container inspect -f '{{json .Config.Env}}' "$id" \ + 2>/dev/null) || { + echo "ERROR: cannot read the environment of ${name}." >&2 + return 1 + } + # BASH_ENV and ENV make every bash in the container run a file + # first, which can set any of these. + bad=$(printf '%s' "$env" | jq -r '.[]? | split("=")[0] + | select((startswith("GIT_") and . != "GIT_USER_NAME" + and . != "GIT_USER_EMAIL") or . == "BASH_ENV" or . == "ENV")') \ + || bad="unreadable" + if [ -n "$bad" ]; then + echo "ERROR: ${name} sets Git environment variables that can" \ + "move repository data or config: ${bad//$'\n'/ }" >&2 + return 1 + fi + if ! bad=$(printf '%s' "$env" | jq -r '.[]? + | select(startswith("HOME=") or startswith("XDG_CONFIG_HOME=")) + | select((split("=")[1:] | join("=")) as $v + | (startswith("HOME=") and $v == "") + or ($v != "" and (($v | startswith("/") | not) + or ($v | test("(^|/)[.][.]?(/|$)")) + or ($v | contains("//")) + or ($v | test("[\u0000-\u001f\u007f]"))))) + | split("=")[0]'); then + echo "ERROR: cannot read the environment of ${name}." >&2 + return 1 + fi + if [ -n "$bad" ]; then + echo "ERROR: ${name} sets ${bad//$'\n'/ and } to an empty," \ + "relative or unnormalized path; its git config cannot be" \ + "located." >&2 + return 1 + fi + user=$(docker container inspect -f '{{.Config.User}}' "$id" \ + 2>/dev/null) || { + echo "ERROR: cannot read the user of ${name}." >&2 + return 1 + } + [ -z "$user" ] || args+=(--user "$user") + while IFS= read -r e; do + [ -n "$e" ] && args+=(-e "$e") + done < <(printf '%s' "$env" | jq -r '.[]? + | select(startswith("HOME=") or startswith("XDG_CONFIG_HOME="))') + tmp=$(retire_tmpdir) || return 1 + swarm_docker_own_group run --rm -i --pull never --network none \ + --label "${RETIRE_HELPER_LABEL}=${PROJECT}" \ + "${args[@]+"${args[@]}"}" -e "SWARM_SCAN_MOUNTS=${mounts}" \ + --entrypoint bash "$snap" -c ' + d=$(mktemp -d) || exit 3 + cat > "$d/scan.sh" || exit 3 + mkdir "$d/out" || exit 3 + bash "$d/scan.sh" userconfig - - "$d/out" >&2 || exit $? + cat "$d/out/report" + ' < "$(dirname "${BASH_SOURCE[0]}")/preserve-scan.sh" \ + > "$tmp/report" 2>"$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "ERROR: cannot check the git config of ${name}:" \ + "$(cat "$tmp/err")" >&2 + rm -rf "$tmp" + return 1 + fi + rc=0 + grep -a '^block ' "$tmp/report" > "$tmp/blocks" 2>/dev/null || rc=$? + case "$rc" in + 0) + echo "ERROR: the git config of ${name} keeps repository data" \ + "outside its workspace, or cannot be checked:" >&2 + sed 's/^block / /' "$tmp/blocks" >&2 + rm -rf "$tmp" + return 1 ;; + 1) ;; + *) + echo "ERROR: cannot read the git config check of ${name}." >&2 + rm -rf "$tmp" + return 1 ;; + esac + rm -rf "$tmp" +} + +cmd_retire() { + local rc=0 + RETIRE_SCRATCH="" + RETIRE_SNAP_REF="" + # The traps come first, so that nothing retire makes is ever + # outside them. Once a signal starts the cleanup, further ones wait + # for it. + trap retire_cleanup EXIT + trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 129' HUP + trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 130' INT + trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 143' TERM + if RETIRE_SCRATCH=$(retire_tmpdir); then + retire_run "$@" || rc=$? + else + rc=1 + fi + retire_cleanup + trap - EXIT HUP INT TERM + return "$rc" +} + +# The retire command itself; cmd_retire runs it with cleanup in place. +retire_run() { + local only_post=false + # Reports and docker output are compared byte by byte: in a UTF-8 + # locale GNU grep would drop lines holding invalid UTF-8. + LC_ALL=C + export LC_ALL + while [ $# -gt 0 ]; do + case "$1" in + --post-process) only_post=true; shift ;; + -h|--help) + cat <&2; return 1 ;; + esac + done + if [ ! -f "$STATE_FILE" ]; then + echo "ERROR: no engagement in ${RUNTIME_DIR}; nothing to retire." >&2 + return 1 + fi + swarm_lock "$RUNTIME_DIR" || return 1 + swarm_engagement_join "$STATE_FILE" || return 1 + retire_require_daemon || return 1 + retire_sweep_snapshots || return 1 + + local list sel checked id name + list=$(swarm_own_containers "$PROJECT" "$REPO_ROOT") || { + echo "ERROR: cannot list Docker containers." >&2 + return 1 + } + sel=$(printf '%s\n' "$list" | while read -r name id; do + [ -n "$name" ] || continue + if [ "$only_post" = true ] && [ "$name" != "${IMAGE_NAME}-post" ]; then + continue + fi + printf '%s %s\n' "$id" "$name" + done) + if [ -z "$sel" ]; then + echo "No containers to retire." + return 0 + fi + # Check every container before removing any. The list comes on + # descriptor 3 and the checks get no standard input, so none of them + # can eat the rest of it; and every selected container must have + # been checked. + checked=$(retire_tmpdir) || return 1 + checked="$checked/checked" + : > "$checked" + while read -r -u 3 id name; do + if ! swarm_retire_preflight "$id" "$name" >> "$checked"; then + echo "Nothing was removed." >&2 + return 1 + fi + done 3<&2 + return 1 + fi + # Remove by ID, each only if nothing about it changed since. + local vol started finished now failed=0 + while read -r -u 3 id name vol started finished; do + now=$(docker container inspect -f \ + '{{.State.Status}} {{.State.StartedAt}} {{.State.FinishedAt}}' \ + "$id" 2>/dev/null) || now="" + case "$now" in + "exited $started $finished"|"created $started $finished") ;; + *) + echo "ERROR: ${name} changed since it was checked; left" \ + "in place." >&2 + failed=1 + continue ;; + esac + if docker rm "$id" >/dev/null 2>&1; then + echo " retired ${name} (workspace volume ${vol} kept)" + else + echo "ERROR: docker rm ${name} failed; left in place." >&2 + failed=1 + fi + done 3< "$checked" < /dev/null + echo "Workspace volumes stay, with the files of the removed containers." + return "$failed" +} + +# Stop unless Docker talks to the daemon the engagement recorded. An +# engagement without a recorded daemon is refused: retire acts on +# containers by ID and must know they are the engagement's. +retire_require_daemon() { + local recorded current + recorded=$(swarm_state_read "$STATE_FILE" 2>/dev/null \ + | jq -r '.docker_daemon // empty' 2>/dev/null) || recorded="" + if [ -z "$recorded" ]; then + if [ -z "${SWARM_DOCKER_DAEMON:-}" ]; then + echo "ERROR: ${STATE_FILE} records no Docker daemon. Name the" \ + "one this engagement runs on:" >&2 + echo " SWARM_DOCKER_DAEMON=\$(docker info -f '{{.ID}}')" \ + "./launch.sh retire" >&2 + return 1 + fi + recorded="$SWARM_DOCKER_DAEMON" + fi + if ! swarm_docker_pin || ! current=$(swarm_docker_daemon_id); then + echo "ERROR: cannot read the Docker daemon id." >&2 + return 1 + fi + if [ "$current" != "$recorded" ]; then + echo "ERROR: this engagement runs on Docker daemon ${recorded}," \ + "but Docker now talks to ${current}." >&2 + return 1 + fi +} diff --git a/tests/runtime_retire.sh b/tests/runtime_retire.sh new file mode 100755 index 0000000..8937072 --- /dev/null +++ b/tests/runtime_retire.sh @@ -0,0 +1,520 @@ +#!/bin/bash +set -uo pipefail + +# shellcheck source=_test_env.sh +source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" + +# tests/runtime_retire.sh +# +# launch.sh retire under real Docker. Retire checks each container in +# a snapshot of it (docker commit) and through docker diff, which the +# fake docker in test_retire.sh only imitates. Each case leaves one +# stopped container with a repository in its workspace volume and +# something in its own layer, runs retire, and checks whether the +# container is gone. +# +# Cases (the base image ships no repository): +# 1. A plain file in /tmp -> removed. +# 2. A link in the volume into the unchanged image -> removed. +# 3. A link in the volume to a file it wrote -> refused. +# 4. A link out of the volume, and a name that is not UTF-8 in its +# layer (docker diff reports the name altered) -> refused. +# 5. core.hooksPath in its global git config -> refused. +# 6. core.hooksPath in the system config of a git built with another +# prefix (/usr/local/etc/gitconfig) -> refused. +# 7. A clone with a new commit in /tmp -> refused. +# An image that ships repositories (one whose name is not UTF-8, a bare +# one, and a bare one whose objects/ is a link elsewhere) is refused, +# changed or not, since docker diff cannot show every change: +# 8. Nothing changed -> refused. +# 9. An edit, a same-size edit with its time put back, a new tag, a +# new object (also through the objects/ link) -> refused. +# 10. An edit under a repository rooted at / -> refused. +# Two containers, the second one unsafe: both are checked and neither +# is removed. +# And retire stopped by SIGTERM once its snapshot exists, or by a +# Ctrl-C during its checks or while docker commit runs, removes the +# snapshot and keeps the container; a snapshot an earlier retire left +# is removed first. +# +# Requires Docker. No API key needed. Wall-clock: about a minute +# once the image is cached. + +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$TESTS_DIR/.." && pwd)" +LAUNCH_SH="${REPO_ROOT}/launch.sh" + +if ! command -v docker >/dev/null 2>&1; then + echo "SKIP: docker not found" + exit 0 +fi +if ! docker info >/dev/null 2>&1; then + echo "SKIP: docker daemon not running" + exit 0 +fi + +# shellcheck source=../lib/project.sh +source "$REPO_ROOT/lib/project.sh" + +# The image runtime_signal_trap.sh builds (cached), and three built on +# it. +BASE_IMAGE="swarm-core-runtime-test" +IMAGE="swarm-core-retire-test" +ROOT_IMAGE="swarm-core-retire-root-test" +PREFIX_IMAGE="swarm-core-retire-prefix-test" +P="swarm-core-retire-test" +NAME="${P}-agent-1" +VOL="${NAME}-ws-00000000" +NAME2="${P}-agent-2" +VOL2="${NAME2}-ws-00000000" +WORKDIR=$(mktemp -d /tmp/swarm-core-retire-XXXXXX) + +drop_case() { + docker rm -f "$NAME" "$NAME2" >/dev/null 2>&1 || true + docker volume rm "$VOL" "$VOL2" >/dev/null 2>&1 || true +} +# What a failed run (of this test or of retire) may leave: containers +# with this test's project label, its snapshots. The project id is +# this test's own. +drop_leftovers() { + local c snap + for c in $(docker ps -aq --filter "label=${SWARM_PROJECT_LABEL}=${P}"); do + docker rm -f "$c" >/dev/null 2>&1 || true + done + for snap in $(docker images -q --filter "reference=${P}-agent-retire"); do + docker rmi -f "$snap" >/dev/null 2>&1 || true + done +} +cleanup() { + drop_case + drop_leftovers + docker rmi "$IMAGE" "$ROOT_IMAGE" "$PREFIX_IMAGE" >/dev/null 2>&1 || true + rm -rf "$WORKDIR" +} +trap cleanup EXIT + +PASS=0 +FAIL=0 + +check() { + local label="$1" actual="$2" expected="$3" + if [ "$actual" = "$expected" ]; then + echo " PASS: ${label}" + PASS=$((PASS + 1)) + else + echo " FAIL: ${label} expected=${expected} got=${actual}" + FAIL=$((FAIL + 1)) + fi +} + +drop_leftovers +echo "--- Building ${BASE_IMAGE} (fake driver only) ---" +docker build --quiet --tag "$BASE_IMAGE" \ + --build-arg SWARM_AGENTS=fake "$REPO_ROOT" >/dev/null || exit 1 +# Repositories in the image (one whose name is not UTF-8), a bare +# repository and one whose objects/ is a link to /opt/object-pool; an +# image whose / is a work tree; and an image whose git, like one built +# with another prefix, reads /usr/local/etc/gitconfig. The agent user +# may write to the repositories. +docker build --quiet --tag "$IMAGE" - >/dev/null < /opt/tool/main.c \\ + && git -C /opt/tool add main.c \\ + && git -C /opt/tool -c user.name=t -c user.email=t@t commit -q -m tool \\ + && git init -q --bare /srv/repo.git \\ + && git -C /opt/tool push -q /srv/repo.git HEAD:refs/heads/main \\ + && git clone -q /opt/tool "\$(printf '/opt/repo\\377')" \\ + && git init -q --bare /srv/pool.git && mkdir -p /opt/object-pool \\ + && mv /srv/pool.git/objects /opt/object-pool/objects \\ + && ln -s /opt/object-pool/objects /srv/pool.git/objects \\ + && chown -R agent:agent /opt/tool /srv/repo.git /srv/pool.git \\ + /opt/object-pool "\$(printf '/opt/repo\\377')" +USER agent +EOF +sed "s|@BASE@|${BASE_IMAGE}|" <<'EOF' \ + | docker build --quiet --tag "$PREFIX_IMAGE" - >/dev/null || exit 1 +FROM @BASE@ +USER root +RUN mkdir -p /usr/local/etc \ + && printf '[core]\n\thooksPath = /opt/hooks\n' > /usr/local/etc/gitconfig \ + && printf '#!/bin/sh\nGIT_CONFIG_SYSTEM=/usr/local/etc/gitconfig exec /usr/bin/git "$@"\n' \ + > /usr/local/bin/git \ + && chmod 755 /usr/local/bin/git +USER agent +EOF +docker build --quiet --tag "$ROOT_IMAGE" - >/dev/null < /opt/x && cd / && git init -q \\ + && git add opt/x \\ + && git -c user.name=t -c user.email=t@t commit -q -m root \\ + && chown agent:agent /opt/x +USER agent +EOF + +# The checkout and its runtime directory, on this Docker daemon. +CHECKOUT="$WORKDIR/checkout" +git init -q "$CHECKOUT" +git -C "$CHECKOUT" -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false commit -q --allow-empty -m init +CK=$(swarm_checkout_path "$CHECKOUT") +RT="$WORKDIR/runtime" +mkdir -m 700 "$RT" +swarm_runtime_claim "$RT" "$CHECKOUT" "$P" || exit 1 +DAEMON=$(swarm_docker_daemon_id) || exit 1 +swarm_state_update "$RT/state.json" \ + '.engagement_id = "e1" | .docker_daemon = $d' --arg d "$DAEMON" || exit 1 + +# Leave a stopped container of image IMG with a committed repository in +# its workspace volume, after ACTION ran in it: NAME with VOL (after +# dropping what an earlier case left), or the second one, NAME2 with +# VOL2, when SECOND is given. Fails (with a check) when it cannot. +make_container() { + local label="$1" img="$2" action="$3" name="$NAME" vol="$VOL" + if [ -n "${4:-}" ]; then + name="$NAME2" + vol="$VOL2" + else + drop_case + fi + if ! docker volume create \ + --label "${SWARM_PROJECT_LABEL}=${P}" \ + --label "${SWARM_CHECKOUT_LABEL}=${CK}" \ + --label "${SWARM_CONTAINER_LABEL}=${name}" \ + --label "${SWARM_KIND_LABEL}=agent" \ + --label "${SWARM_CONTEXT_LABEL}=full" \ + --label "${SWARM_ENGAGEMENT_LABEL}=e1" \ + "$vol" >/dev/null; then + check "$label: volume" "failed" "created" + return 1 + fi + if ! docker run --name "$name" \ + --label "${SWARM_PROJECT_LABEL}=${P}" \ + --label "${SWARM_CHECKOUT_LABEL}=${CK}" \ + --mount "type=volume,src=${vol},dst=/workspace" \ + --entrypoint bash "$img" -c ' + set -e + cd /workspace + git init -q + git -c user.name=t -c user.email=t@t commit -q \ + --allow-empty -m init + '"$action" > "$WORKDIR/run.out" 2>&1; then + check "$label: container ran" "failed" "ok" + tail -8 "$WORKDIR/run.out" | sed 's/^/ /' + return 1 + fi +} + +container_state() { + if docker container inspect "${1:-$NAME}" >/dev/null 2>&1; then + echo present + else + echo gone + fi +} + +# Case LABEL: make_container IMG ACTION, then retire. EXPECT is +# "removed", or text retire's refusal must contain. +run_case() { + local label="$1" img="$2" action="$3" expect="$4" rc=0 out + local failed_before="$FAIL" + echo "--- ${label} ---" + make_container "$label" "$img" "$action" || return + out=$(cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + "$BASH" "$LAUNCH_SH" retire 2>&1) || rc=$? + if [ "$expect" = removed ]; then + check "$label: retired" "$rc" "0" + check "$label: container gone" "$(container_state)" "gone" + check "$label: volume kept" \ + "$(docker volume inspect "$VOL" >/dev/null 2>&1 \ + && echo kept || echo gone)" "kept" + else + check "$label: refused" "$rc" "1" + check "$label: for that reason" \ + "$(printf '%s\n' "$out" | LC_ALL=C grep -acF -- "$expect" \ + | awk '{ print ($1 > 0) }')" "1" + check "$label: container kept" "$(container_state)" "present" + fi + if [ "$FAIL" -gt "$failed_before" ]; then + printf '%s\n' "$out" | tail -8 | sed 's/^/ /' + fi +} + +snapshots() { + docker images -q --filter "reference=${P}-agent-retire" | wc -l \ + | tr -d ' ' +} + +# Retire gets SIGTERM (as from a timeout) once its snapshot exists: it +# stops, removes the snapshot, and keeps the container. The workspace +# holds a few thousand files, so the checks take a while. +interrupt_case() { + local label="interrupted by SIGTERM" pid rc=0 seen=0 tries=0 base + local failed_before="$FAIL" + echo "--- ${label} ---" + make_container "$label" "$BASE_IMAGE" ' + mkdir many + for i in $(seq 1 3000); do echo "$i" > "many/f$i"; done + git add many + git -c user.name=t -c user.email=t@t commit -q -m many' || return + # Snapshots left by anything else do not count. + base=$(snapshots) + (cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + exec "$BASH" "$LAUNCH_SH" retire) > "$WORKDIR/int.out" 2>&1 & + pid=$! + # Up to a minute for the snapshot to show up. + while [ "$tries" -lt 600 ]; do + if [ "$(snapshots)" -gt "$base" ]; then + seen=1 + break + fi + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + tries=$((tries + 1)) + done + kill -TERM "$pid" 2>/dev/null + wait "$pid" || rc=$? + check "$label: the snapshot existed first" "$seen" "1" + check "$label: stopped" "$rc" "143" + check "$label: snapshot removed" "$(snapshots)" "$base" + check "$label: container kept" "$(container_state)" "present" + if [ "$FAIL" -gt "$failed_before" ]; then + tail -8 "$WORKDIR/int.out" | sed 's/^/ /' + fi +} + +# A Ctrl-C at the terminal (SIGINT to retire's process group) while +# docker commit works on a large layer: the daemon finishes the image +# even when its client stops, so retire must wait for the commit and +# remove the snapshot after; none may show up later. +group_interrupt_case() { + local label="Ctrl-C during the commit" pid rc=0 tries=0 seen=0 base + local failed_before="$FAIL" + echo "--- ${label} ---" + make_container "$label" "$BASE_IMAGE" \ + 'head -c 400000000 /dev/urandom > /tmp/big' || return + base=$(snapshots) + set -m + (cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + exec "$BASH" "$LAUNCH_SH" retire) > "$WORKDIR/gint.out" 2>&1 & + pid=$! + set +m + # Up to a minute for docker commit to start. + while [ "$tries" -lt 600 ]; do + # A here-string: under pipefail, ps | grep -q can miss a match. + if grep -qE '(^|/)docker commit ' <<< "$(ps -A -o args=)"; then + seen=1 + break + fi + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + tries=$((tries + 1)) + done + sleep 0.5 + kill -INT -- "-$pid" 2>/dev/null + wait "$pid" || rc=$? + check "$label: the commit was running" "$seen" "1" + check "$label: stopped" "$rc" "130" + # Long enough for the daemon to finish an abandoned commit. + sleep 15 + check "$label: no snapshot left" "$(snapshots)" "$base" + check "$label: container kept" "$(container_state)" "present" + if [ "$FAIL" -gt "$failed_before" ]; then + tail -8 "$WORKDIR/gint.out" | sed 's/^/ /' + fi +} + +# A Ctrl-C at the terminal once the snapshot exists (while the checks +# run containers from it): retire stops, removes those containers and +# the snapshot, and keeps the container. The snapshot carries the +# engine's labels emptied. +group_interrupt_scan_case() { + local label="Ctrl-C during the checks" pid rc=0 seen=0 tries=0 base + local snap="" labels="" failed_before="$FAIL" + echo "--- ${label} ---" + make_container "$label" "$BASE_IMAGE" ' + mkdir many + for i in $(seq 1 3000); do echo "$i" > "many/f$i"; done + git add many + git -c user.name=t -c user.email=t@t commit -q -m many' || return + base=$(snapshots) + set -m + (cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + exec "$BASH" "$LAUNCH_SH" retire) > "$WORKDIR/gsint.out" 2>&1 & + pid=$! + set +m + # Up to a minute for the snapshot to show up. + while [ "$tries" -lt 600 ]; do + if [ "$(snapshots)" -gt "$base" ]; then + seen=1 + break + fi + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + tries=$((tries + 1)) + done + if [ "$seen" = 1 ]; then + snap=$(docker images -q --no-trunc \ + --filter "reference=${P}-agent-retire" | head -n 1) + labels=$(docker image inspect -f \ + '{{index .Config.Labels "swarm-core.project"}}|{{index .Config.Labels "swarm-core.checkout"}}' \ + "$snap" 2>/dev/null) + fi + kill -INT -- "-$pid" 2>/dev/null + wait "$pid" || rc=$? + check "$label: the snapshot existed first" "$seen" "1" + check "$label: its engine labels are empty" "$labels" "|" + check "$label: stopped" "$rc" "130" + check "$label: snapshot removed" "$(snapshots)" "$base" + check "$label: no container made from it is left" \ + "$(docker ps -a --no-trunc --format '{{.Image}}' \ + | grep -cF -- "${snap:-none}" || true)" "0" + check "$label: container kept" "$(container_state)" "present" + if [ "$FAIL" -gt "$failed_before" ]; then + tail -8 "$WORKDIR/gsint.out" | sed 's/^/ /' + fi +} + +# Whether a job started here under job control can be stopped by +# SIGINT: not when this run started with SIGINT ignored (a background +# job of a script), since the ignore is inherited and cannot be undone. +# Bash 3.2 does not report such an ignore through trap -p, so try it, +# in a subshell: job control switched on in the test's own shell would +# stay with it on Bash 3.2. +sigint_works() { + local pid tries=0 + set -m + sleep 30 & + pid=$! + set +m + # Signal it once it runs sleep: before its exec, the shell's own + # handling of SIGINT still applies and the signal can be lost. + while [ "$tries" -lt 40 ]; do + case "$(ps -o comm= -p "$pid" 2>/dev/null)" in + *sleep) break ;; + esac + sleep 0.05 + tries=$((tries + 1)) + done + kill -INT "$pid" 2>/dev/null + tries=0 + while kill -0 "$pid" 2>/dev/null && [ "$tries" -lt 20 ]; do + sleep 0.05 + tries=$((tries + 1)) + done + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 1 + fi + wait "$pid" 2>/dev/null + return 0 +} + +# Two containers, the second one unsafe (a clone in /tmp): retire +# checks both, so it refuses and removes neither. +two_container_case() { + local label="a safe container and an unsafe one" rc=0 out + local failed_before="$FAIL" + echo "--- ${label} ---" + make_container "$label" "$BASE_IMAGE" 'echo notes > /tmp/notes' \ + || return + make_container "$label" "$BASE_IMAGE" ' + git clone -q --no-hardlinks /workspace /tmp/experiment + git -C /tmp/experiment -c user.name=t -c user.email=t@t \ + commit -q --allow-empty -m unpushed' second || return + out=$(cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + "$BASH" "$LAUNCH_SH" retire 2>&1) || rc=$? + check "$label: refused" "$rc" "1" + check "$label: for the unsafe one" \ + "$(printf '%s\n' "$out" | grep -c "${NAME2} has Git repositories")" "1" + check "$label: the safe one kept" "$(container_state "$NAME")" "present" + check "$label: the unsafe one kept" "$(container_state "$NAME2")" "present" + if [ "$FAIL" -gt "$failed_before" ]; then + printf '%s\n' "$out" | tail -8 | sed 's/^/ /' + fi +} + +# A snapshot an earlier retire left (one killed outright) is removed +# first. +sweep_case() { + local label="a snapshot an earlier retire left" rc=0 out + local failed_before="$FAIL" + echo "--- ${label} ---" + make_container "$label" "$BASE_IMAGE" 'true' || return + docker tag "$BASE_IMAGE" "${P}-agent-retire:left-behind" || { + check "$label: tagged" "failed" "ok" + return + } + out=$(cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + "$BASH" "$LAUNCH_SH" retire 2>&1) || rc=$? + check "$label: retired" "$rc" "0" + check "$label: removed" \ + "$(docker image inspect "${P}-agent-retire:left-behind" \ + >/dev/null 2>&1 && echo present || echo gone)" "gone" + check "$label: said so" \ + "$(printf '%s\n' "$out" | grep -c 'a snapshot an earlier retire left')" \ + "1" + if [ "$FAIL" -gt "$failed_before" ]; then + printf '%s\n' "$out" | tail -8 | sed 's/^/ /' + fi +} + +run_case "a plain file in /tmp" "$BASE_IMAGE" \ + 'echo notes > /tmp/notes' removed +run_case "a link in the volume into the unchanged image" "$BASE_IMAGE" \ + 'ln -s /usr/bin/git /workspace/git-link' removed +run_case "a link in the volume to a file it wrote" "$BASE_IMAGE" ' + echo result > /tmp/result + ln -s /tmp/result /workspace/result' \ + "links out of its volume to data in the container" +run_case "a link out, and a name that is not UTF-8 in its layer" \ + "$BASE_IMAGE" ' + ln -s /usr/bin/git /workspace/git-link + echo result > "$(printf "/tmp/result\377")"' "cannot report exactly" +run_case "core.hooksPath in its global git config" "$BASE_IMAGE" \ + 'git config --global core.hooksPath /home/agent/hooks' \ + "core.hookspath" +run_case "a system git config outside /etc" "$PREFIX_IMAGE" 'true' \ + "config-setting /usr/local/etc/gitconfig core.hookspath" +run_case "a clone with a new commit in /tmp" "$BASE_IMAGE" ' + git clone -q --no-hardlinks /workspace /tmp/experiment + git -C /tmp/experiment -c user.name=t -c user.email=t@t \ + commit -q --allow-empty -m unpushed' " /tmp/experiment/.git" +run_case "repositories the image ships, unchanged" "$IMAGE" 'true' \ + " /opt/tool/.git" +run_case "an edit in a repository of the image" "$IMAGE" \ + 'echo "/* edit */" >> /opt/tool/main.c' " /opt/tool/.git" +run_case "a same-size edit with its time put back" "$IMAGE" ' + f=/opt/tool/main.c + t=$(stat -c %y "$f") + sed -i "s/main/mian/" "$f" + touch -d "$t" "$f"' " /opt/tool/.git" +run_case "a new tag in a bare repository of the image" "$IMAGE" \ + 'git -C /srv/repo.git tag v1 main' " /srv/repo.git" +run_case "a new object in a bare repository of the image" "$IMAGE" \ + 'echo data | git -C /srv/repo.git hash-object -w --stdin >/dev/null' \ + " /srv/repo.git" +run_case "a new object through an objects/ link" "$IMAGE" \ + 'echo data | git -C /srv/pool.git hash-object -w --stdin >/dev/null' \ + " /srv/pool.git" +run_case "an edit in an image repository whose name is not UTF-8" "$IMAGE" \ + 'echo "/* edit */" >> "$(printf "/opt/repo\377")/main.c"' \ + "$(printf ' /opt/repo\377/.git')" +run_case "an edit under a repository rooted at /" "$ROOT_IMAGE" \ + 'echo edited >> /opt/x' " /.git" +two_container_case +interrupt_case +if ! (sigint_works); then + echo "--- SKIP: the Ctrl-C cases (SIGINT is ignored here) ---" +else + group_interrupt_scan_case + group_interrupt_case +fi +sweep_case + +echo "" +echo " ${PASS} passed, ${FAIL} failed" +[ "$FAIL" -eq 0 ] diff --git a/tests/test.sh b/tests/test.sh index 45d2198..85f67ec 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -18,7 +18,8 @@ fi # # Options: # --unit Run unit tests only (no Docker or API key needed). -# --runtime Runtime emergency-push test (Docker only, no API key). +# --runtime Runtime tests: emergency push and retire (Docker +# only, no API key). # --all Unit + runtime, then full integration matrix. # --oauth Integration test using OAuth token (needs Docker + # CLAUDE_CODE_OAUTH_TOKEN). @@ -72,7 +73,9 @@ run_all_tests() { # test_session_end_push.sh ยง6 / ยง7 cannot prove the signal trap # actually fires; this phase does. Skips cleanly when Docker # is unavailable so --all stays useful on hosts without it. - # Also runs the real Pi CLI against a local test API, without keys. + # Also runs the real Pi CLI against a local test API, without keys, + # and launch.sh retire on real containers, whose snapshots and + # docker diff test_retire.sh only imitates. echo "=== Phase 1.5: Docker runtime tests ===" echo "" if ! command -v docker >/dev/null 2>&1; then @@ -83,7 +86,8 @@ run_all_tests() { runtime_skip=1 else local rt_test rt_start rt_elapsed rt_rc - for rt_test in runtime_signal_trap.sh runtime_pi.sh; do + for rt_test in runtime_signal_trap.sh runtime_pi.sh \ + runtime_retire.sh; do rt_rc=0 rt_start=$(date +%s) "$TESTS_DIR/$rt_test" || rt_rc=$? @@ -395,9 +399,10 @@ Run swarm-core tests. Options: (no args) Single integration smoke test (needs Docker + API key). --unit Unit tests only (no Docker or API key needed). - --runtime Runtime emergency-push test (needs Docker, no API key). - Drives SIGTERM / SIGINT / fatal-exit paths under a - real container with a synthetic test-committer driver. + --runtime Runtime tests (need Docker, no API key): drives + SIGTERM / SIGINT / fatal-exit paths under a real + container with a synthetic test-committer driver, + then launch.sh retire on real stopped containers. --all Unit + runtime, then full integration matrix. --oauth Integration test using CLAUDE_CODE_OAUTH_TOKEN (needs Docker + OAuth token). @@ -453,8 +458,10 @@ if $RUN_OAUTH; then fi if $RUN_RUNTIME; then - "$TESTS_DIR/runtime_signal_trap.sh" - exit $? + rt_rc=0 + "$TESTS_DIR/runtime_signal_trap.sh" || rt_rc=1 + "$TESTS_DIR/runtime_retire.sh" || rt_rc=1 + exit "$rt_rc" fi if $RUN_ALL; then diff --git a/tests/test_preserve.sh b/tests/test_preserve.sh index c8cc5af..4087f02 100755 --- a/tests/test_preserve.sh +++ b/tests/test_preserve.sh @@ -425,6 +425,89 @@ assert_eq "a bare repo under agent_logs blocks" "1" \ "$(has_line "$out" 'block embedded agent_logs/x.git')" rm -rf "$W/cache.git" "$W/agent_logs" +git -C "$W" config include.path /elsewhere +assert_eq "a config include blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block config-include .git')" +git -C "$W" config --unset include.path +ln -s /tmp "$W/.git/out-link" +assert_eq "a git-dir symlink to a directory out of the volume blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block outside-dir-link .git/out-link')" +ln -s ../../home/agent/result "$W/result" +ln -s "$W/nowhere" "$W/dangling" +ln -s loop-b "$W/loop-a" +ln -s loop-a "$W/loop-b" +out=$(scan worktree "$W" full) +assert_eq "a relative link out of the volume reports its target" "1" \ + "$(grep -c "^layer-link $(dirname "$(dirname "$(cd -P "$W" && pwd -P)")")/home/agent/result$" \ + "$out/report" || true)" +assert_eq "a dangling link inside the volume is not reported" "0" \ + "$(grep -cE '^(layer-link|block).*dangling' "$out/report" || true)" +assert_eq "a link loop blocks" "1" \ + "$(grep -cE '^block link-loop loop-[ab]$' "$out/report" | awk '{ print ($1 > 0) }')" +rm "$W/result" "$W/dangling" "$W/loop-a" "$W/loop-b" +ln -s "$(cd -P "$W" && pwd -P)"$'\n' "$W/nl-target" +assert_eq "a link whose target ends in a newline blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block newline-path nl-target')" +rm "$W/nl-target" +git -C "$W" config core.worktree "$(cd -P "$W" && pwd -P)"$'\n' +assert_eq "a core.worktree ending in a newline blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block core-worktree .git (newline in path)')" +git -C "$W" config --unset core.worktree +git init -q --bare "$TMPDIR/common-out.git" +CW="$TMPDIR/commondir-ws" +git init -q "$CW" +g -C "$CW" commit -q --allow-empty -m c +echo "$TMPDIR/common-out.git" > "$CW/.git/commondir" +assert_eq "a commondir outside the volume blocks" "1" \ + "$(has_line "$(scan worktree "$CW" full)" 'block commondir .git')" +rm "$W/.git/out-link" +git -C "$W" config core.hooksPath ../hooks +assert_eq "any core.hooksPath blocks" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block core-hookspath .git')" +git -C "$W" config --unset core.hooksPath +mkdir -p "$W/lfs-in-ws" +ln -s "$W/lfs-in-ws" "$W/.git/lfs" +assert_eq "a git-dir symlink to a directory inside the volume is fine" "0" \ + "$(grep -c 'link.*\.git/lfs' "$(scan worktree "$W" full)/report" || true)" +rm "$W/.git/lfs" +rmdir "$W/lfs-in-ws" +git init -q --bare "$TMPDIR/alt-outside.git" +printf '%s' "$TMPDIR/alt-outside.git/objects" > "$W/.git/objects/info/alternates" +assert_eq "alternates outside the volume block (no final newline)" "1" \ + "$(has_line "$(scan worktree "$W" full)" 'block alternates .git/objects')" +rm "$W/.git/objects/info/alternates" +# A NUL byte in a file the scan parses with the shell blocks: Git would +# stop at it while the shell drops it, so they would read different +# paths. +git init -q --bare "$TMPDIR/nul-store.git" +printf '%s/objects\000/../..%s/.git/objects\n' \ + "$(cd -P "$TMPDIR/nul-store.git" && pwd -P)" "$(cd -P "$W" && pwd -P)" \ + > "$W/.git/objects/info/alternates" +out=$(scan worktree "$W" full) +assert_eq "alternates with a NUL byte: the scan completes" "" \ + "$(cat "$out/.rc" 2>/dev/null)" +assert_eq "alternates with a NUL byte block" "1" \ + "$(has_line "$out" 'block alternates .git/objects (unreadable or NUL byte)')" +rm -f "$W/.git/objects/info/alternates" +NW="$TMPDIR/nul-wt-ws" +git init -q "$NW" +g -C "$NW" commit -q --allow-empty -m n +git -C "$NW" worktree add -q --detach "$NW/inside" 2>/dev/null +_nadm=$(find "$NW/.git/worktrees" -mindepth 1 -maxdepth 1 -type d | head -1) +printf '%s\000x\n' "$(cd -P "$NW" && pwd -P)/inside/.git" > "$_nadm/gitdir" +out=$(scan worktree "$NW" full) +assert_eq "a worktree back-pointer with a NUL byte blocks" "1" \ + "$(grep -c 'block missing-worktree .*(unreadable or NUL byte)' "$out/report" || true)" +assert_eq "so does the checkout's gitfile pointing at that admin dir" "1" \ + "$(grep -cE '^block (embedded|mismatched-worktree) inside' "$out/report" || true)" +printf '..\000/..\n' > "$_nadm/commondir" +assert_eq "a commondir with a NUL byte blocks" "1" \ + "$(grep -c 'block commondir .*(unreadable or NUL byte)' "$(scan worktree "$NW" full)/report" || true)" +GLK="$TMPDIR/gitlink-ws" +mkdir -p "$GLK" +ln -s /tmp "$GLK/.git" +assert_eq "a .git symlink is a gitlink, not nogit" "1" \ + "$(has_line "$(scan worktree "$GLK" full)" 'block gitlink')" git init -q "$W/inner" assert_eq "an embedded repository blocks" "1" \ "$(has_line "$(scan worktree "$W" full)" 'block embedded inner')" @@ -460,8 +543,11 @@ assert_eq "its detached commit is kept" "commit" \ mkdir -p "$TMPDIR/clean-elsewhere" echo dirty-again >> "$W/.gitmodules" git -C "$W" config core.worktree "$TMPDIR/clean-elsewhere" +out=$(scan worktree "$W" full) assert_eq "core.worktree does not hide dirty files" "1" \ - "$(has_line "$(scan worktree "$W" full)" 'uncommitted dirty .: M .gitmodules')" + "$(has_line "$out" 'uncommitted dirty .: M .gitmodules')" +assert_eq "a core.worktree outside the volume blocks" "1" \ + "$(has_line "$out" 'block core-worktree .git ')" git -C "$W" config --unset core.worktree git -C "$W" checkout -q -- .gitmodules diff --git a/tests/test_retire.sh b/tests/test_retire.sh new file mode 100755 index 0000000..b7ecdae --- /dev/null +++ b/tests/test_retire.sh @@ -0,0 +1,1378 @@ +#!/bin/bash +set -euo pipefail + +# shellcheck source=_test_env.sh +source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" + +# Unit tests for launch.sh retire, with a fake docker CLI whose answers +# come from files: no Docker or API key required. + +PASS=0 +FAIL=0 +TMPDIR=$(mktemp -d) +# Exported, so that the fake docker and the programs it runs make their +# scratch files here, under the cleanup below. +export TMPDIR +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +LAUNCH="$REPO_ROOT/launch.sh" +trap 'rm -rf "$TMPDIR"' EXIT + +# The git config check asks git which files it reads (git var, Git 2.42 +# or later); here the scanner runs on this host's git. +if ! git var GIT_CONFIG_SYSTEM >/dev/null 2>&1; then + echo "SKIP: these tests need Git 2.42 or later (git var)" + exit 0 +fi + +# shellcheck source=../lib/project.sh +source "$REPO_ROOT/lib/project.sh" + +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + echo " PASS: ${label}" + PASS=$((PASS + 1)) + else + echo " FAIL: ${label}" + echo " expected: ${expected}" + echo " actual: ${actual}" + FAIL=$((FAIL + 1)) + fi +} + +# A checkout (no swarmfile: retire must not need one) and its runtime. +REPO="$TMPDIR/proj" +git init -q "$REPO" +git -C "$REPO" -c user.name=t -c user.email=t@t -c commit.gpgsign=false \ + commit -q --allow-empty -m init +CK=$(cd -P "$REPO" && pwd -P) +P=proj +RT="$TMPDIR/runtime" +mkdir -m 700 "$RT" +swarm_runtime_claim "$RT" "$REPO" "$P" +swarm_state_update "$RT/state.json" \ + '.engagement_id = "e1" | .docker_daemon = "d1"' + +# Fake docker. $F holds the answers: +# containers " " lines (all labeled for this checkout) +# inspect/ docker container inspect JSON +# volumes/ volume labels JSON +# cp/ files docker cp finds in the container +# scan/ the scanner's report for that volume ("fail": exit 3) +# rmfail/ docker rm fails +# snaproot/ the / of every snapshot, for the repository search +# daemon docker info id (default d1) +# log every call +F="$TMPDIR/fake" +mkdir -p "$F/inspect" "$F/volumes" "$F/cp" "$F/scan" "$F/rmfail" "$TMPDIR/bin" +cat > "$TMPDIR/bin/docker" <<'EOF' +#!/bin/bash +F="$FAKE_DOCKER" +printf '%s\n' "$*" >> "$F/log" +case "$1" in + info) cat "$F/daemon" 2>/dev/null || echo d1; exit 0 ;; + context) echo default; exit 0 ;; + ps) + case "$*" in + # Containers made from a snapshot: $F/left-from-snapshot + # ("", or " unlabeled" for one retire did not make, + # from any snapshot) and $F/made (what docker run made while + # $F/leave-helpers exists, " labeled|unlabeled", + # matched by the snapshot's name or image id). + *"ancestor="*) + ref=$(printf '%s\n' "$*" \ + | sed -n 's/.*ancestor=\([^ ]*\).*/\1/p') + sha=$(awk -v r="$ref" '$1 == r { print $2 }' \ + "$F/images" 2>/dev/null) + only="" + case "$*" in + *"label=swarm-core.retire=proj"*) only=labeled ;; + esac + if [ -e "$F/left-from-snapshot" ]; then + if [ -n "$only" ]; then + grep -v ' unlabeled$' "$F/left-from-snapshot" || true + else + sed 's/ unlabeled$//' "$F/left-from-snapshot" + fi + fi + [ ! -e "$F/made" ] || awk -v r="$ref" -v s="$sha" \ + -v o="$only" '($2 == r || (s != "" && $2 == s)) \ + && (o == "" || $3 == o) { print $1 }' "$F/made" + exit 0 ;; + *"label=swarm-core.session="*) exit 0 ;; + *"label=swarm-core.project="*) + while read -r id name; do + [ -n "$id" ] || continue + jq -nc --arg id "$id" --arg n "$name" --arg o "$FAKE_CK" \ + '{id: $id, name: $n, project: "proj", owner: $o}' + done < "$F/containers" ;; + esac + exit 0 ;; + container) + shift + [ "$1" = inspect ] || exit 1 + shift + if [ "$1" = -f ] || [ "$1" = --format ]; then + fmt="$2"; id="$3" + [ -f "$F/inspect/$id" ] || exit 1 + case "$fmt" in + *State.Status*StartedAt*) + jq -r '.[0].State | "\(.Status) \(.StartedAt) \(.FinishedAt)"' \ + "$F/inspect/$id.now" 2>/dev/null \ + || jq -r '.[0].State | "\(.Status) \(.StartedAt) \(.FinishedAt)"' \ + "$F/inspect/$id" ;; + *Labels*) jq -c '.[0].Config.Labels' "$F/inspect/$id" ;; + *Config.Env*) jq -c '.[0].Config.Env // []' "$F/inspect/$id" ;; + *Config.User*) jq -r '.[0].Config.User // ""' "$F/inspect/$id" ;; + *Destination*) + jq -r '.[0].Mounts[]?.Destination, + ((.[0].HostConfig.Tmpfs // {}) | keys[])' \ + "$F/inspect/$id" ;; + *) jq -r '.[0].Id' "$F/inspect/$id" ;; + esac + exit 0 + fi + [ -f "$F/inspect/$1" ] || exit 1 + cat "$F/inspect/$1" + exit 0 ;; + volume) + [ "$2" = inspect ] || exit 1 + [ -f "$F/volumes/$5" ] || exit 1 + cat "$F/volumes/$5" + exit 0 ;; + cp) + follow="" + if [ "$2" = -L ]; then follow=h; shift; fi + src="${2#*:}"; id="${2%%:*}" + if [ ! -e "$F/cp/$id$src" ] && [ ! -L "$F/cp/$id$src" ]; then + echo "Error response from daemon: Could not find the file $src in container $id" >&2 + exit 1 + fi + tar -C "$(dirname "$F/cp/$id$src")" "-c${follow}f" - \ + "$(basename "$src")" + exit 0 ;; + run) + # $F/leave-helpers: each container docker run makes stays behind, + # as a client stopped between create and start leaves it, in + # $F/made as " labeled|unlabeled"; the label comes + # from the run's own arguments. + if [ -e "$F/leave-helpers" ]; then + img=""; prev=""; lab=unlabeled + for a in "$@"; do + [ "$a" != -c ] || img="$prev" + if [ "$prev" = --label ] \ + && [ "$a" = swarm-core.retire=proj ]; then + lab=labeled + fi + prev="$a" + done + n=$(wc -l < "$F/made" 2>/dev/null) || n=0 + echo "made-$((n + 1)) ${img:-none} $lab" >> "$F/made" + fi + d=$(mktemp -d "$F/run.XXXXXX") + # The repository search in the snapshot: retire's own program, + # run here with $F/snaproot as the snapshot's root. Found paths + # are mapped back to the snapshot's names. + case "$*" in + *swarm-repos*) + # docker run -i copies standard input into the container, + # whether or not the program reads it. + for a in "$@"; do + if [ "$a" = -i ]; then + cat > /dev/null + break + fi + done + prog=""; prev="" + for a in "$@"; do + [ "$prev" != -c ] || prog="$a" + prev="$a" + done + root="$F/snaproot" + bash -c "$prog" swarm-repos "$root" > "$d/found" || exit $? + while IFS= read -r r; do + case "$r" in + "$root") printf '/\n' ;; + "$root"/*) printf '%s\n' "${r#"$root"}" ;; + *) printf 'outside the root: %s\n' "$r" ;; + esac + done < "$d/found" + exit 0 ;; + esac + # The image comparison: read each path on this file system, as + # the snapshot would; for the original image (sha256:aaa...), + # the paths listed in $F/orig-changed read as another content. + case "$*" in + *swarm-compare*) + img="" + for a in "$@"; do + case "$a" in sha256:*) img="$a" ;; esac + done + while IFS= read -r p; do + if [ -L "$p" ]; then + printf '%s\tl\t%s\n' "$p" "$(readlink "$p")" + elif [ -f "$p" ]; then + h=$(shasum -a 256 < "$p" 2>/dev/null \ + || sha256sum < "$p") + if [ "$img" = "sha256:$(printf 'a%.0s' $(seq 64))" ] \ + && grep -qxF "$p" "$F/orig-changed" 2>/dev/null + then + h="0000 -" + fi + printf '%s\tf\t%s\n' "$p" "${h%% *}" + elif [ -e "$p" ]; then + printf '%s\to\t-\n' "$p" + else + printf '%s\t-\t-\n' "$p" + fi + done + exit 0 ;; + esac + # The git config check: the real scanner in userconfig mode, with + # the container's HOME and XDG_CONFIG_HOME from -e (a test home + # when none is given), never the caller's own. + case "$*" in + *userconfig*) + cat > "$d/scan.sh" + mkdir "$d/out" + prev=""; uhome="$F/home-default"; uxdg=""; umounts="" + for a in "$@"; do + if [ "$prev" = -e ]; then + case "$a" in + HOME=*) uhome="${a#HOME=}" ;; + XDG_CONFIG_HOME=*) uxdg="${a#XDG_CONFIG_HOME=}" ;; + SWARM_SCAN_MOUNTS=*) umounts="${a#SWARM_SCAN_MOUNTS=}" ;; + esac + fi + prev="$a" + done + mkdir -p "$F/home-default" + if [ -n "$uxdg" ]; then + env -i PATH="$PATH" HOME="$uhome" XDG_CONFIG_HOME="$uxdg" \ + SWARM_SCAN_MOUNTS="$umounts" \ + bash "$d/scan.sh" userconfig - - "$d/out" >&2 || exit $? + else + env -i PATH="$PATH" HOME="$uhome" \ + SWARM_SCAN_MOUNTS="$umounts" \ + bash "$d/scan.sh" userconfig - - "$d/out" >&2 || exit $? + fi + cat "$d/out/report" || exit 3 + exit 0 ;; + esac + vol=$(printf '%s\n' "$*" | sed -n 's/.*type=volume,src=\([^,]*\),.*/\1/p') + # $F/term-on-scan: retire gets SIGTERM while it scans (after + # its snapshot exists), as from a timeout. + if [ -n "$vol" ] && [ -e "$F/term-on-scan" ]; then + kill -TERM "$PPID" + fi + # A volume with a directory under voldir/ is scanned for real: + # the scanner script comes on stdin, the context is the last + # argument. + if [ -d "$F/voldir/$vol" ]; then + cat > "$d/scan.sh" + mkdir "$d/out" + prev="" + for a in "$@"; do + ctx="$a" + if [ "$prev" = -e ]; then + case "$a" in + SWARM_SCAN_MOUNTS=*) + export SWARM_SCAN_MOUNTS="${a#SWARM_SCAN_MOUNTS=}" ;; + esac + fi + prev="$a" + done + bash "$d/scan.sh" worktree "$F/voldir/$vol" "$ctx" "$d/out" \ + >&2 || exit $? + tar -C "$d/out" -cf - report + exit 0 + fi + cat > /dev/null + if [ "$(cat "$F/scan/$vol" 2>/dev/null)" = fail ]; then exit 3; fi + cp "$F/scan/$vol" "$d/report" 2>/dev/null || : > "$d/report" + tar -C "$d" -cf - report + exit 0 ;; + diff) + cat "$F/diff/$2" 2>/dev/null + exit 0 ;; + commit) + [ ! -e "$F/commitfail" ] || exit 1 + echo "Flag --pause has been deprecated." + # commit [--change ...] : one image id per + # container, its digits padded; $F/images holds " " + # per image made. + cid="${*: -2:1}" + cname="${*: -1}" + snapid=$(printf 'sha256:%064d' "$(printf '%s' "$cid" | tr -cd '0-9')") + if [ -e "$F/int-on-commit" ]; then + # Ctrl-C at the terminal while the daemon commits: SIGINT + # to retire's process group. The daemon makes the image a + # second later all the same; this client waits for it, + # unless the SIGINT stops it first. + ( trap '' INT; sleep 1 + printf '%s %s\n' "$cname" "$snapid" >> "$F/images" ) & + trap 'exit 130' INT + kill -INT -- "-$(ps -o pgid= -p "$PPID" | tr -d ' ')" + sleep 2 + else + printf '%s %s\n' "$cname" "$snapid" >> "$F/images" + fi + echo "$snapid" + exit 0 ;; + rmi) + # $F/term-on-rmi: retire gets SIGTERM during its first docker + # rmi, which then removes nothing. + if [ -e "$F/term-on-rmi" ]; then + rm -f "$F/term-on-rmi" + kill -TERM "$PPID" + exit 1 + fi + if [ -e "$F/images" ] && awk -v r="$2" '$1 == r || $2 == r { f = 1 } + END { exit !f }' "$F/images"; then + awk -v r="$2" '$1 != r && $2 != r' "$F/images" > "$F/images.new" + mv "$F/images.new" "$F/images" + exit 0 + fi + echo "Error response from daemon: No such image: $2" >&2 + exit 1 ;; + image) + # image inspect + [ -e "$F/images" ] && awk -v r="$3" '$1 == r { f = 1 } + END { exit !f }' "$F/images" + exit $? ;; + images) + [ ! -e "$F/imagesfail" ] || exit 1 + ref="" + for a in "$@"; do + case "$a" in reference=*) ref="${a#reference=}" ;; esac + done + [ ! -e "$F/images" ] || awk -v p="$ref:" \ + 'index($1, p) == 1 { print $1 }' "$F/images" + exit 0 ;; + rm) + [ ! -e "$F/rmfail/$2" ] || exit 1 + echo "$2" + exit 0 ;; +esac +exit 0 +EOF +chmod +x "$TMPDIR/bin/docker" + +# Add container NAME (id id-NAME) with state STATE and a proper +# workspace volume NAME-ws-00000000 of kind KIND. +add() { + local name="$1" state="${2:-exited}" kind="${3:-agent}" + local id="id-$name" vol="$name-ws-00000000" + printf '%s %s\n' "$id" "$name" >> "$F/containers" + # The image's passwd, as docker cp finds it. + mkdir -p "$F/cp/$id/etc" + printf 'root:x:0:0:root:/root:/bin/bash\nagent:x:1000:1000::/home/agent:/bin/bash\n' \ + > "$F/cp/$id/etc/passwd" + jq -n --arg id "$id" --arg st "$state" --arg vol "$vol" --arg n "$name" \ + --arg ck "$CK" '[{ + Id: $id, Name: ("/" + $n), + Image: ("sha256:" + ("a" * 64)), + State: {Status: $st, StartedAt: "S1", FinishedAt: "F1"}, + HostConfig: {RestartPolicy: {Name: "no"}, AutoRemove: false, + Tmpfs: null}, + Mounts: [ + {Type: "volume", Name: $vol, Destination: "/workspace"}, + {Type: "bind", Source: "/x", Destination: "/upstream"}], + Config: {Labels: {"swarm-core.project": "proj", + "swarm-core.checkout": $ck}} + }]' > "$F/inspect/$id" + jq -n --arg n "$name" --arg ck "$CK" --arg k "$kind" '{ + "swarm-core.project": "proj", "swarm-core.checkout": $ck, + "swarm-core.container": $n, "swarm-core.kind": $k, + "swarm-core.context": "full", "swarm-core.engagement": "e1" + }' > "$F/volumes/$vol" +} +# Change container NAME's inspect JSON with jq filter FILTER. +edit() { + local n="$1" + shift + jq "$@" "$F/inspect/id-$n" > "$F/inspect/id-$n.tmp" + mv "$F/inspect/id-$n.tmp" "$F/inspect/id-$n" +} +reset_fake() { + chmod -R u+w "$F" 2>/dev/null || true + rm -rf "$F" + mkdir -p "$F/inspect" "$F/volumes" "$F/cp" "$F/scan" "$F/rmfail" \ + "$F/voldir" "$F/diff" "$F/snaproot" + : > "$F/images" + : > "$F/containers" + : > "$F/log" +} +# Run launch.sh retire in a process group of its own, as from a +# terminal, so that a SIGINT to that group stays there; print +# "". +retire_own_group() { + local rc=0 pid + set -m + (cd "$REPO" && PATH="$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" FAKE_CK="$CK" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + exec "$BASH" "$LAUNCH" retire > "$TMPDIR/out" 2>&1) & + pid=$! + set +m + wait "$pid" || rc=$? + printf '%s' "$rc" +} +# Run launch.sh retire ARGS, with SHIM_DIR (when set) first on PATH; +# print "". +SHIM_DIR="" +retire() { + local rc=0 + (cd "$REPO" && PATH="${SHIM_DIR:+$SHIM_DIR:}$TMPDIR/bin:$PATH" \ + FAKE_DOCKER="$F" FAKE_CK="$CK" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + "$BASH" "$LAUNCH" retire "$@" > "$TMPDIR/out" 2>&1) || rc=$? + printf '%s' "$rc" +} +# The number of entries in directory DIR. +entries() { + find "$1" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ' +} +# Engine containers removed (docker rm without -f; a forced removal is +# a check container made from a snapshot). +removed() { + grep -cE '^rm [^-]' "$F/log" || true +} + +echo "=== 1. Stopped containers whose files are in their volumes ===" +reset_fake +assert_eq "nothing to retire succeeds" "0" "$(retire)" +add proj-agent-1 +add proj-agent-2 +assert_eq "retire removes stopped agents" "0" "$(retire)" +assert_eq "both are removed" "2" "$(removed)" +assert_eq "by id, without -f" "2" "$(grep -cE '^rm id-proj-agent-[12]$' "$F/log")" +assert_eq "it needs no swarmfile" "false" \ + "$([ -e "$REPO/swarm.json" ] && echo true || echo false)" + +reset_fake +add proj-agent-1 +add proj-agent-post +assert_eq "--post-process retires only the post-process container" "0" \ + "$(retire --post-process)" +assert_eq "only it is removed" "rm id-proj-agent-post" \ + "$(grep '^rm ' "$F/log")" + +echo "" +echo "=== 2. Containers whose layer may hold work are refused ===" +# Each case: one container breaks a rule; nothing at all is removed, +# not even a sound container checked first. +# Run retire and expect a refusal with nothing removed. EXPECT, when +# given, must appear in retire's output: the specific finding, so that a +# scanner failure cannot pass for it. With SHOW_REASON set, print each +# refusal's first error line. +refused() { + local label="$1" expect="${2:-}" + assert_eq "$label: refused" "1" "$(retire)" + if [ -n "$expect" ]; then + assert_eq "$label: for that reason" "1" \ + "$(grep -cF -- "$expect" "$TMPDIR/out" | awk '{ print ($1 > 0) }')" + fi + [ -z "${SHOW_REASON:-}" ] \ + || sed -n 's/^ERROR: / reason: /p; s/^ \(.*mount-crossing.*\)/ detail: \1/p; s/^ \(config-.*\)/ detail: \1/p' \ + "$TMPDIR/out" | head -2 + assert_eq "$label: nothing removed" "0" "$(removed)" +} +case_setup() { + reset_fake + add proj-agent-1 + add proj-agent-2 "${1:-exited}" "${2:-agent}" +} +case_setup running; refused "a running container" "state running" +# Every selected container is checked, whatever the checks of the ones +# before it read: here the third one is unsafe. +reset_fake +add proj-agent-1 +add proj-agent-2 +add proj-agent-3 running +refused "a third container that is unsafe" "state running" +assert_eq "all three were checked" "3" \ + "$(grep -cE '^container inspect id-proj-agent-[123]$' "$F/log" || true)" +case_setup paused; refused "a paused container" "state paused" +case_setup restarting; refused "a restarting container" "state restarting" +case_setup +edit proj-agent-2 '.[0].HostConfig.RestartPolicy.Name = "always"' +refused "a restart policy" "restart policy always" +case_setup +edit proj-agent-2 '.[0].HostConfig.AutoRemove = true' +refused "auto-remove" "auto-remove is on" +case_setup +edit proj-agent-2 '.[0].HostConfig.Tmpfs = {"/workspace/tmp": ""}' +refused "a tmpfs below /workspace" "tmpfs at /workspace/tmp" +case_setup +edit proj-agent-2 '.[0].Mounts += [{Type: "volume", Name: "x", + Destination: "/workspace/sub"}]' +refused "a second mount below /workspace" "no single workspace mount" +case_setup +edit proj-agent-2 '.[0].Mounts[0].Type = "bind"' +refused "a bind-mounted /workspace" "workspace is not a volume" +case_setup +edit proj-agent-2 '.[0].Mounts = [.[0].Mounts[1]]' +refused "no /workspace mount (a pre-volume container)" "no single workspace mount" +case_setup +jq '.["swarm-core.container"] = "other"' \ + "$F/volumes/proj-agent-2-ws-00000000" > "$F/v.tmp" +mv "$F/v.tmp" "$F/volumes/proj-agent-2-ws-00000000" +refused "another container's volume" "is not its workspace volume" + +# Git config the container's processes read, checked inside its +# snapshot (here: this file system, with HOME and XDG_CONFIG_HOME from +# the container's environment). +H="$(cd -P "$TMPDIR" && pwd -P)/homes" +case_setup +mkdir -p "$H/a" +printf '[lfs]\n\tstorage = /home/agent/lfs\n' > "$H/a/.gitconfig" +edit proj-agent-2 --arg h "$H/a" '.[0].Config.Env = ["HOME=" + $h]' +refused "lfs.storage in the container's global git config" "config-setting" +case_setup +mkdir -p "$H/b" +printf '[include]\n\tpath = /x\n' > "$H/b/.gitconfig" +edit proj-agent-2 --arg h "$H/b" '.[0].Config.Env = ["HOME=" + $h]' +refused "an include in the global git config" "include.path" +case_setup +mkdir -p "$H/c" +printf '[user]\n\tname = a\n' > "$H/c/.gitconfig" +edit proj-agent-2 --arg h "$H/c" '.[0].Config.Env = ["HOME=" + $h]' +assert_eq "a plain git config is fine" "0" "$(retire)" +case_setup +mkdir -p "$H/d" "$H/elsewhere" +printf '[core]\n\thooksPath = /home/agent/hooks\n' > "$H/elsewhere/real" +ln -sfn "$H/elsewhere/real" "$H/d/.gitconfig" +edit proj-agent-2 --arg h "$H/d" '.[0].Config.Env = ["HOME=" + $h]' +refused "a symlinked .gitconfig is read through the link" "core.hookspath" +case_setup +mkdir -p "$H/x/git" +printf '[core]\n\tworktree = /x\n' > "$H/x/git/config" +edit proj-agent-2 --arg x "$H/x" '.[0].Config.Env = ["XDG_CONFIG_HOME=" + $x]' +refused "config under the container's XDG_CONFIG_HOME" "core.worktree" +case_setup +mkdir -p "$H/e" +printf '[lfs]\n\tstorage = /x\n[broken\n' > "$H/e/.gitconfig" +edit proj-agent-2 --arg h "$H/e" '.[0].Config.Env = ["HOME=" + $h]' +refused "a config file that does not parse" "config-paths-unknown fatal: bad config line 3 in file $H/e/.gitconfig" +# Config on a mount, or reached through a link onto one, cannot be +# read from the snapshot; Docker's own mounts count too. +case_setup +mkdir -p "$H/m" +edit proj-agent-2 --arg h "$H/m" '.[0].Config.Env = ["HOME=" + $h] + | .[0].Mounts += [{Type: "bind", Source: "/gone", Destination: $h}]' +refused "git config on a mount of the container" "config-on-mount" +case_setup +mkdir -p "$H/t" +ln -sfn "$H/t" "$H/alias" +edit proj-agent-2 --arg h "$H/alias" --arg t "$H/t" \ + '.[0].Config.Env = ["HOME=" + $h] | .[0].HostConfig.Tmpfs = {($t): ""}' +refused "git config through a link onto a tmpfs" "config-on-mount" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["HOME=/dev/shm"]' +refused "git config in Docker's /dev/shm" "config-on-mount" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["HOME=/workspace"]' +refused "git config in the workspace (not in the snapshot)" "config-on-mount" +mkdir -p "$H/run/git-home" +ln -sfn "$H/run" "$H/varrun" +case_setup +edit proj-agent-2 --arg h "$H/run/git-home" --arg m "$H/varrun/git-home" \ + '.[0].Config.Env = ["HOME=" + $h] + | .[0].Mounts += [{Type: "bind", Source: "/g", Destination: $m}]' +refused "git config on a mount configured through a link" "config-on-mount" +if [ "$(id -u)" != 0 ]; then + case_setup + mkdir -p "$H/locked/inner" + chmod 000 "$H/locked" + edit proj-agent-2 --arg h "$H/locked/inner" '.[0].Config.Env = ["HOME=" + $h]' + refused "a home git cannot search" "config-unreadable" + chmod 700 "$H/locked" + case_setup + mkdir -p "$H/noread" + printf '[user]\n\tname = a\n' > "$H/noread/.gitconfig" + chmod 000 "$H/noread/.gitconfig" + edit proj-agent-2 --arg h "$H/noread" '.[0].Config.Env = ["HOME=" + $h]' + refused "an unreadable .gitconfig" "config-unreadable" + chmod 600 "$H/noread/.gitconfig" +fi +# The files checked are those the container's git names: a git built +# with another prefix reads its system file elsewhere than /etc. +REAL_GIT=$(command -v git) +mkdir -p "$TMPDIR/bin-git-prefix" "$TMPDIR/bin-git-old" "$H/prefix/etc" +printf '[core]\n\thooksPath = /opt/hooks\n' > "$H/prefix/etc/gitconfig" +cat > "$TMPDIR/bin-git-prefix/git" < "$TMPDIR/bin-git-old/git" <)" >&2 + exit 129 +fi +exec '$REAL_GIT' "\$@" +EOF +chmod +x "$TMPDIR/bin-git-prefix/git" "$TMPDIR/bin-git-old/git" +case_setup +SHIM_DIR="$TMPDIR/bin-git-prefix" +refused "a system git config where this git reads it, not in /etc" \ + "config-setting $H/prefix/etc/gitconfig core.hookspath" +SHIM_DIR="" +case_setup +SHIM_DIR="$TMPDIR/bin-git-old" +refused "a git that cannot name its config files" "config-paths-unknown" +SHIM_DIR="" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["HOME=/custom\n"]' +refused "a HOME ending in a newline" "HOME to an empty, relative or unnormalized" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["XDG_CONFIG_HOME=/a\nb"]' +refused "an XDG_CONFIG_HOME with a newline inside" "XDG_CONFIG_HOME to an empty" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["HOME="]' +refused "an empty HOME" "HOME to an empty" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["HOME=/opt/alias/.."]' +refused "a HOME with .." "HOME to an empty" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["XDG_CONFIG_HOME=/a//b"]' +refused "an XDG_CONFIG_HOME with //" "XDG_CONFIG_HOME to an empty" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["HOME=."]' +refused "a relative HOME" "HOME to an empty" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["XDG_CONFIG_HOME=cfg"]' +refused "a relative XDG_CONFIG_HOME" "XDG_CONFIG_HOME to an empty" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["GIT_CONFIG_GLOBAL=/x", "GIT_USER_NAME=a"]' +refused "a Git environment variable in the container" "GIT_CONFIG_GLOBAL" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["BASH_ENV=/home/agent/env.sh"]' +refused "a BASH_ENV in the container" "BASH_ENV" +case_setup +edit proj-agent-2 '.[0].Config.Env = ["GIT_USER_NAME=a", "GIT_USER_EMAIL=b"]' +assert_eq "the engine's own GIT_USER_* variables are fine" "0" "$(retire)" +case_setup +edit proj-agent-2 '.[0].Config.User = "agent"' +assert_eq "the container's user is passed to the check" "1" \ + "$(retire >/dev/null; grep -c -- '--user agent' "$F/log" || true)" + +# Worktrees outside the volume, and a scan that cannot tell. +case_setup +echo "block external /tmp/scratch" > "$F/scan/proj-agent-2-ws-00000000" +refused "a worktree outside the volume" "external /tmp/scratch" +case_setup +echo "block mismatched-worktree agent_logs/copy" \ + > "$F/scan/proj-agent-2-ws-00000000" +refused "a mismatched worktree" "mismatched-worktree" +case_setup +echo fail > "$F/scan/proj-agent-2-ws-00000000" +refused "a failed scan" "cannot scan the workspace" +for _b in "gitfile /workspace/.git is not a directory" \ + "gitlink /workspace/.git is a symlink" \ + "lfs-storage .git /home/agent/lfs" "newline-path a\nb"; do + case_setup + echo "block $_b" > "$F/scan/proj-agent-2-ws-00000000" + refused "scan finding ${_b%% *}" +done +case_setup +echo "block embedded cache.git" > "$F/scan/proj-agent-2-ws-00000000" +refused "a nested repository (not inspected deeply)" "embedded cache.git" + +# The real scanner on a directory standing in for the volume. +real_ws() { + local d="$F/voldir/proj-agent-2-ws-00000000" + git init -q "$d" + git -C "$d" -c user.name=t -c user.email=t@t -c commit.gpgsign=false \ + commit -q --allow-empty -m ws + printf '%s' "$d" +} +# The container's own paths, here under the canonical temp dir (a path +# like /home goes through a host link on macOS). +_ct=$(cd -P "$TMPDIR" && pwd -P) +L="$_ct/layer" +case_setup +real_ws >/dev/null +assert_eq "a clean real workspace is retired" "0" "$(retire)" +case_setup +_d=$(real_ws) +mkdir -p "$TMPDIR/outside-wt" +git -C "$_d" config core.worktree "$TMPDIR/outside-wt" +refused "a repository-local core.worktree outside the volume" "core-worktree" +case_setup +_d=$(real_ws) +ln -s "$L/lfs" "$_d/.git/lfs" +printf 'A %s/lfs\nA %s/lfs/objects\n' "$L" "$L" > "$F/diff/id-proj-agent-2" +refused "a git-dir symlink to storage the container wrote" "links out of its volume to data" +case_setup +_d=$(real_ws) +git init -q --bare "$_d/cache.git" +refused "an embedded bare repository" "embedded cache.git" +case_setup +_d=$(real_ws) +git -C "$_d" config include.path /home/agent/extra +refused "a repository-local include" "config-include" +case_setup +_d=$(real_ws) +git -C "$_d" config core.hooksPath /home/agent/hooks +refused "a repository-local core.hooksPath outside the volume" "core-hookspath" +case_setup +_d=$(real_ws) +mkdir -p "$_d/lfs" +ln -s "$L/lfs-objects" "$_d/lfs/objects" +ln -s "$_d/lfs" "$_d/.git/lfs" +printf 'A %s/lfs-objects\n' "$L" > "$F/diff/id-proj-agent-2" +refused "a git-dir link to a directory that links into the layer" "links out of its volume to data" +# Links out of the volume: fine into the image, refused into what the +# container added or changed (docker diff). +mkdir -p "$F/diff" +case_setup +_d=$(real_ws) +ln -s "$L/result" "$_d/result" +mkdir -p "$F/diff" +printf 'C %s\nA %s/result\nA %s/result/x\n' "$L" "$L" "$L" \ + > "$F/diff/id-proj-agent-2" +refused "a checkout symlink to data the container wrote" "links out of its volume to data" +case_setup +_d=$(real_ws) +mkdir -p "$_d/.venv/bin" +ln -s /usr/bin/python3 "$_d/.venv/bin/python" +mkdir -p "$F/diff" +printf 'C /tmp\nA /tmp/x\n' > "$F/diff/id-proj-agent-2" +assert_eq "a link into the unchanged image (a virtualenv) is fine" "0" \ + "$(retire)" +# Links are resolved where they were made: the scan runs from a +# snapshot of the container (here: this file system), the kernel's way. +mkdir -p "$_ct/real-run" "$_ct/store" "$_ct/srv/run" "$_ct/opt" +ln -sfn "$_ct/real-run" "$_ct/img-run" +case_setup +_d=$(real_ws) +ln -s "$_ct/img-run/results" "$_d/results" +printf 'A %s/real-run/results\n' "$_ct" > "$F/diff/id-proj-agent-2" +refused "a link through another link to data the container wrote" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -s "$_ct/img-run/newdir/result" "$_d/result" +printf 'A %s/real-run/newdir\nA %s/real-run/newdir/result\n' "$_ct" "$_ct" \ + > "$F/diff/id-proj-agent-2" +refused "a link through a directory only the container has" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -sfn "$_ct/store" "$_ct/alias" +ln -s "$_ct/alias/result" "$_d/result" +printf 'A %s/alias\nA %s/store/result\n' "$_ct" "$_ct" \ + > "$F/diff/id-proj-agent-2" +refused "a link through an alias the container created" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -sfn "$_ct/srv/run" "$_ct/opt/alias" +ln -s "$_ct/opt/alias/../result" "$_d/result" +printf 'A %s/srv/result\n' "$_ct" > "$F/diff/id-proj-agent-2" +refused "a link whose .. comes after another link" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -sfn "$_d" "$_ct/opt/current" +ln -s "$_ct/opt/current/inside" "$_d/back" +printf 'A /anything/else\n' > "$F/diff/id-proj-agent-2" +assert_eq "a link that leads back into the volume is fine" "0" "$(retire)" +assert_eq "the scan ran from a snapshot of the container" "1" \ + "$(grep -c '^commit .* id-proj-agent-2 proj-agent-retire:' "$F/log" || true)" +assert_eq "with the engine's labels emptied on the snapshot" "2" \ + "$(grep -cF -- '--change LABEL swarm-core.project="" swarm-core.checkout="" swarm-core.session=""' \ + "$F/log" || true)" +assert_eq "each snapshot is removed after, by its name" "2" \ + "$(grep -c '^rmi proj-agent-retire:id-proj-agen-[0-9][0-9]*$' "$F/log" \ + || true)" +# The config check, the scan, the search for repositories and the +# comparison with the image (this case has a link on the way) all run +# from the snapshot. +assert_eq "config check, scan, repository search and comparison use the snapshot" "4" \ + "$(grep -c "sha256:$(printf '%064d' 2) -c " "$F/log" || true)" +case_setup +_d=$(real_ws) +ln -s /upstream/alias/result "$_d/result" +refused "a link into another mount of the container" "mount-crossing result /upstream" +case_setup +_d=$(real_ws) +ln -s "$_ct/real-run" "$_d/rundir" +refused "a link to a directory outside the volume" "outside-dir-link" +case_setup +_d=$(real_ws) +ln -s /upstream/alias/../../result "$_d/result" +refused "a link whose .. crosses another mount" "mount-crossing result /upstream" +mkdir -p "$_ct/usrbin" +: > "$_ct/usrbin/tool" +case_setup +_d=$(real_ws) +ln -sfn "$_ct/usrbin" "$_ct/opt/tools" +ln -s "$_ct/opt/tools/tool" "$_d/tool" +printf 'A %s/opt/tools\n' "$_ct" > "$F/diff/id-proj-agent-2" +refused "a link through a link the container created" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -sfn "$_d" "$_ct/opt/current" +ln -s "$_ct/opt/current/inside" "$_d/back" +printf 'A %s/opt/current\n' "$_ct" > "$F/diff/id-proj-agent-2" +refused "a link back into the volume through a link the container made" "links out of its volume to data" +# A mount whose configured destination goes through a link is also +# found under the link's target. +mkdir -p "$_ct/run/data" +ln -sfn "$_ct/run" "$_ct/varrun" +case_setup +_d=$(real_ws) +edit proj-agent-2 --arg m "$_ct/varrun/data" \ + '.[0].Mounts += [{Type: "bind", Source: "/h", Destination: $m}]' +ln -s "$_ct/run/data/result" "$_d/result" +refused "a link into a mount reached through another link" "mount-crossing" +case_setup +_d=$(real_ws) +ln -s /usr/bin "$_d"$'\n'"cache" +ln -s "$_d"$'\n'"cache/true" "$_d/tool" +refused "a link through a name with a newline" "newline-path" +case_setup +_d=$(real_ws) +ln -sfn "$_d" "$_ct/zalias" +edit proj-agent-2 --arg m "$_ct/zalias/.git" \ + '.[0].Mounts += [{Type: "bind", Source: "/h", Destination: $m}]' +refused "a mount that lands in the volume through a link" "mount-in-workspace" +case_setup +edit proj-agent-2 '.[0].HostConfig.Mounts = [{Target: "/workspace", + VolumeOptions: {Subpath: "sub"}}]' +refused "a workspace mount of a volume subdirectory" "subdirectory of its volume" +case_setup +edit proj-agent-2 '.[0].Mounts += [{Type: "bind", Source: "/h", + Destination: "/varrun\n/data"}]' +refused "a mount destination with a newline" "control character" +case_setup +edit proj-agent-2 '.[0].Mounts += [{Type: "bind", Source: "/h", + Destination: "/mnt/a\tb"}]' +refused "a mount destination with a tab" "control character" +# A mount whose destination passes through another mount cannot be +# resolved from the snapshot. +mkdir -p "$_ct/mnt" +case_setup +edit proj-agent-2 --arg a "$_ct/mnt" --arg b "$_ct/mnt/ws" \ + '.[0].Mounts += [{Type: "bind", Source: "/h1", Destination: $a}, + {Type: "bind", Source: "/h2", Destination: $b}]' +real_ws >/dev/null +refused "a mount whose destination is inside another mount" "mount-through-mount" +mkdir -p "$_ct/runp" +ln -sfn "$_ct/runp" "$_ct/varrunp" +case_setup +edit proj-agent-2 --arg a "$_ct/varrunp" --arg b "$_ct/varrunp/ws" \ + '.[0].Mounts += [{Type: "bind", Source: "/h1", Destination: $a}, + {Type: "bind", Source: "/h2", Destination: $b}]' +real_ws >/dev/null +refused "a mount below another mount whose path is a symlink" "mount-through-mount" +# Mount checks run for an empty volume too. +case_setup +mkdir -p "$F/voldir/proj-agent-2-ws-00000000" +ln -sfn "$F/voldir/proj-agent-2-ws-00000000" "$_ct/zempty" +edit proj-agent-2 --arg m "$_ct/zempty" \ + '.[0].Mounts += [{Type: "bind", Source: "/h", Destination: $m}]' +refused "an empty volume with a mount landing on it" "mount-in-workspace" +case_setup +_d=$(real_ws) +ln -s /etc/hosts "$_d/hosts" +refused "a link to a file Docker mounts itself" "mount-crossing hosts" +# Paths the repository's settings name may not resolve through a mount +# the snapshot lacks (there, a link may lead elsewhere). +mkdir -p "$_ct/optm" +case_setup +_d=$(real_ws) +ln -sfn "$_d" "$_ct/optm/wt" +git -C "$_d" config core.worktree "$_ct/optm/wt" +edit proj-agent-2 --arg m "$_ct/optm" \ + '.[0].Mounts += [{Type: "bind", Source: "/o", Destination: $m}]' +refused "a core.worktree through a mount" "core-worktree" +case_setup +_d=$(real_ws) +git init -q --bare "$_d/pool.git" +ln -sfn "$_d/pool.git" "$_ct/optm/pool" +echo "$_ct/optm/pool/objects" > "$_d/.git/objects/info/alternates" +edit proj-agent-2 --arg m "$_ct/optm" \ + '.[0].Mounts += [{Type: "bind", Source: "/o", Destination: $m}]' +refused "alternates through a mount back into the volume" "alternates" +# docker diff goes by metadata; a link to an image file whose content +# changed with its metadata kept is caught by comparing with the image. +mkdir -p "$_ct/imgfiles" +echo original > "$_ct/imgfiles/tool" +case_setup +_d=$(real_ws) +ln -s "$_ct/imgfiles/tool" "$_d/tool" +: > "$F/diff/id-proj-agent-2" +assert_eq "a link to an unchanged image file is fine" "0" "$(retire)" +case_setup +_d=$(real_ws) +ln -s "$_ct/imgfiles/tool" "$_d/tool" +: > "$F/diff/id-proj-agent-2" +printf '%s\n' "$_ct/imgfiles/tool" > "$F/orig-changed" +refused "a link to an image file changed without a trace in docker diff" \ + "to files it changed" +rm -f "$F/orig-changed" +# Any Git repository in the snapshot outside /workspace refuses, made by +# the container or shipped by the image, changed or not: docker diff +# cannot show every change. The search runs retire's program over +# $F/snaproot, the snapshot's / here; docker diff plays no part. A +# HEAD file alone, HEAD with refs/ but no objects, and a repository +# under /workspace (the volume, scanned on its own) do not count. +case_setup +mkdir -p "$F/snaproot/tmp" "$F/snaproot/srv/x/refs" "$F/snaproot/workspace" +: > "$F/snaproot/tmp/HEAD" +: > "$F/snaproot/srv/x/HEAD" +git init -q "$F/snaproot/workspace/repo" +assert_eq "a snapshot without repositories is fine" "0" "$(retire)" +assert_eq "the search runs in the snapshot, offline, as root" "1" \ + "$(grep -cF -- "--network none --user 0 --label swarm-core.retire=proj --entrypoint bash sha256:$(printf '%064d' 2) -c" \ + "$F/log" | awk '{ print ($1 > 0) }')" +case_setup +git init -q "$F/snaproot/tmp/experiment" +refused "a clone in the container's /tmp" " /tmp/experiment/.git" +case_setup +git init -q --bare "$F/snaproot/tmp/x.git" +refused "a bare clone in the container's layer" " /tmp/x.git" +case_setup +git init -q "$F/snaproot/opt/tool" +refused "a repository the image ships, even unchanged" " /opt/tool/.git" +case_setup +mkdir -p "$F/snaproot/opt/wt" +printf 'gitdir: /opt/tool/.git/worktrees/wt\n' > "$F/snaproot/opt/wt/.git" +refused "a linked worktree (a .git file)" " /opt/wt/.git" +case_setup +mkdir -p "$F/snaproot/opt/ln" +ln -s /nonexistent/gitdir "$F/snaproot/opt/ln/.git" +refused "a dangling .git link" " /opt/ln/.git" +case_setup +git init -q "$F/snaproot" +refused "a repository rooted at /" " /.git" +case_setup +git init -q --bare "$F/snaproot/srv/repo.git" +refused "a bare repository of the image" " /srv/repo.git" +# New objects of a bare repository whose objects/ is a link land outside +# it; the repository is found all the same, dangling link or not. So is +# a git dir kept apart from its work tree (commondir). +case_setup +git init -q --bare "$F/snaproot/srv/p.git" +mkdir -p "$F/snaproot/opt/pool" +mv "$F/snaproot/srv/p.git/objects" "$F/snaproot/opt/pool/objects" +ln -s "$F/snaproot/opt/pool/objects" "$F/snaproot/srv/p.git/objects" +refused "a bare repository whose objects/ is a link" " /srv/p.git" +case_setup +mkdir -p "$F/snaproot/srv/d.git/refs" +: > "$F/snaproot/srv/d.git/HEAD" +ln -s /nonexistent/objects "$F/snaproot/srv/d.git/objects" +refused "a git dir whose objects/ link dangles" " /srv/d.git" +case_setup +mkdir -p "$F/snaproot/var/gd" +: > "$F/snaproot/var/gd/HEAD" +echo ../.. > "$F/snaproot/var/gd/commondir" +refused "a git dir kept apart from its work tree" " /var/gd" +case_setup +git init -q --bare "$F/snaproot" +refused "a bare repository rooted at /" "Git repositories outside its workspace" +assert_eq "named as /" "1" "$(grep -cx ' /' "$TMPDIR/out" || true)" +case_setup +git init -q "$F/snaproot/tmp/a +b" +refused "a repository whose path holds a newline" \ + "Git repositories outside its workspace" +if [ "$(id -u)" != 0 ]; then + case_setup + mkdir -p "$F/snaproot/root/locked" + chmod 000 "$F/snaproot/root/locked" + # (Both containers share the fake's snapshot: the first one stops.) + refused "a directory the search cannot read" \ + "cannot check proj-agent-1 for Git repositories" + chmod 700 "$F/snaproot/root/locked" +fi +# docker diff is read for links out of the volume. It prints names +# unescaped: a newline makes it ambiguous. +case_setup +_d=$(real_ws) +ln -s "$_ct/imgfiles/tool" "$_d/tool" +printf 'A /tmp/experiment\nname\n' > "$F/diff/id-proj-agent-2" +refused "a docker diff with a newline in a name" "cannot report unambiguously" +# Docker sends names as JSON, which turns bytes that are not UTF-8 into +# U+FFFD: such a name does not exist in the container. +case_setup +_d=$(real_ws) +ln -s "$_ct/imgfiles/tool" "$_d/tool" +printf 'C /opt/data\nA /opt/data/x\357\277\275\n' > "$F/diff/id-proj-agent-2" +refused "a name docker diff reports altered (not UTF-8)" "cannot report exactly" +# The list of links to compare cannot be built: refuse. +mkdir -p "$TMPDIR/bin-sed" +cat > "$TMPDIR/bin-sed/sed" <<'EOF' +#!/bin/bash +case "$*" in *'s/^layer-link //p'*) exit 4 ;; esac +exec /usr/bin/sed "$@" +EOF +chmod +x "$TMPDIR/bin-sed/sed" +case_setup +_d=$(real_ws) +ln -s "$_ct/imgfiles/tool" "$_d/tool" +: > "$F/diff/id-proj-agent-2" +rc=0 +(cd "$REPO" && PATH="$TMPDIR/bin-sed:$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" \ + FAKE_CK="$CK" SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + SWARM_SKIP_DEP_CHECK=1 "$BASH" "$LAUNCH" retire > "$TMPDIR/out" 2>&1) \ + || rc=$? +assert_eq "a failed link list for the image comparison refuses" "1" "$rc" +assert_eq "for that reason" "1" \ + "$(grep -c 'cannot list the links of' "$TMPDIR/out" || true)" +assert_eq "nothing removed then" "0" "$(removed)" +# A quoted alternates entry is decoded by Git; the scan refuses it even +# when a directory spelled with the quotes exists in the volume. +git init -q --bare "$_ct/qpool.git" +case_setup +_d=$(real_ws) +mkdir -p "$_d/.git/objects/\"$_ct/qpool.git/objects\"" +printf '"%s/qpool.git/objects"\n' "$_ct" > "$_d/.git/objects/info/alternates" +refused "a quoted alternates entry" "alternates .git/objects \"$_ct/qpool.git/objects\"" +# A git dir split into a common dir that is not a listed git dir. +case_setup +_d=$(real_ws) +mkdir -p "$_d/.git/common" +mv "$_d/.git/objects" "$_d/.git/refs" "$_d/.git/common/" +echo common > "$_d/.git/commondir" +refused "a common dir the scan does not inspect" "commondir .git " +# An alternates file with a NUL byte: Git and the shell would read +# different paths from it. +case_setup +_d=$(real_ws) +# Git reads up to the NUL: an existing store outside, so git status +# works and the refusal comes from the NUL rule. +printf '%s/qpool.git/objects\000/../..%s/.git/objects\n' "$_ct" \ + "$(cd -P "$_d" && pwd -P)" > "$_d/.git/objects/info/alternates" +refused "an alternates file with a NUL byte" "(unreadable or NUL byte)" +# A link target that is not valid UTF-8: docker diff (JSON) reports it +# with U+FFFD, so it cannot be compared and refuses. +case_setup +_d=$(real_ws) +ln -s "$_ct/layer/result$(printf '\377')" "$_d/bad-name" +printf 'A %s/layer/result\357\277\275\n' "$_ct" > "$F/diff/id-proj-agent-2" +# Run under a UTF-8 locale, where GNU grep would drop the line: retire +# must compare bytes whatever locale it is called with. +LC_ALL=C.UTF-8 refused "a link to a path that is not valid UTF-8" \ + "not valid UTF-8" +case_setup +_d=$(real_ws) +ln -s "$_ct/layer/caf$(printf '\303\251')" "$_d/good-name" +: > "$F/diff/id-proj-agent-2" +assert_eq "a valid non-ASCII link target is compared as usual" "0" "$(retire)" +# A link outside the volume that a repository setting passes, made by +# the container: compared with docker diff like links in the volume. +mkdir -p "$_ct/optv" +case_setup +_d=$(real_ws) +ln -sfn "$_d" "$_ct/optv/wt" +git -C "$_d" config core.worktree "$_ct/optv/wt" +printf 'A %s/optv/wt\n' "$_ct" > "$F/diff/id-proj-agent-2" +refused "a core.worktree through a link the container made" "links out of its volume to data" +# A directory the container added, passed on the way back in. +mkdir -p "$_ct/home2/transient" +case_setup +_d=$(real_ws) +_dc=$(cd -P "$_d" && pwd -P) +# Up out of transient and home2, then down into the volume again. +ln -s "$_ct/home2/transient/../..${_dc#"$_ct"}/result" "$_d/back" +echo r > "$_d/result" +printf 'A %s/home2/transient\n' "$_ct" > "$F/diff/id-proj-agent-2" +refused "a link through a directory the container added" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -s /proc/1/cwd/result "$_d/saved" +refused "a link through /proc" "mount-crossing saved /proc" +case_setup +: > "$F/commitfail" +refused "a container that cannot be snapshotted" "cannot snapshot" +# Interrupted while it scans (SIGTERM after the snapshot exists), retire +# removes the snapshot and every scratch file, and nothing else. +reset_fake +add proj-agent-1 +: > "$F/term-on-scan" +_before=$(entries "$TMPDIR") +assert_eq "an interrupted retire stops (SIGTERM)" "143" "$(retire)" +assert_eq "it removes the snapshot it made" "1" \ + "$(grep -c '^rmi proj-agent-retire:id-proj-agen-[0-9][0-9]*$' "$F/log" \ + || true)" +assert_eq "it leaves no scratch files" "$_before" "$(entries "$TMPDIR")" +assert_eq "it removes no container" "0" "$(removed)" +rm -f "$F/term-on-scan" +# A container made from a snapshot that retire did not start (no label +# of retire's) stays: when an interrupted retire cleans up, and when a +# later one sweeps the snapshot a killed one left (with a warning). +reset_fake +add proj-agent-1 +: > "$F/term-on-scan" +echo "look-1 unlabeled" > "$F/left-from-snapshot" +assert_eq "an interrupted retire with someone's container on the snapshot" \ + "143" "$(retire)" +assert_eq "does not remove that container" "0" \ + "$(grep -c '^rm -f look-1$' "$F/log" || true)" +rm -f "$F/term-on-scan" +reset_fake +add proj-agent-1 +echo "proj-agent-retire:left-1 sha256:$(printf '%064d' 9)" >> "$F/images" +echo "look-1 unlabeled" > "$F/left-from-snapshot" +assert_eq "a retire that finds a snapshot a killed one left" "0" "$(retire)" +assert_eq "does not remove someone's container made from it" "0" \ + "$(grep -c '^rm -f look-1$' "$F/log" || true)" +assert_eq "and says so" "1" \ + "$(grep -c 'that retire did not start stay' "$TMPDIR/out" || true)" +rm -f "$F/left-from-snapshot" +# Check containers a stopped client left (made from the snapshot) go +# before the snapshot does. Each run leaves its container here, with +# the labels its own arguments gave it. +reset_fake +add proj-agent-1 +: > "$F/term-on-scan" +: > "$F/leave-helpers" +assert_eq "an interrupted retire with its check containers left" "143" \ + "$(retire)" +_snap="sha256:$(printf '%064d' 1)" +_from=$(awk -v s="$_snap" '$2 == s' "$F/made" | grep -c . || true) +assert_eq "its checks made containers from the snapshot" "1" \ + "$([ "$_from" -gt 1 ] && echo 1 || echo 0)" +assert_eq "every one of them with retire's label" "0" \ + "$(awk -v s="$_snap" '$2 == s && $3 != "labeled"' "$F/made" \ + | grep -c . || true)" +assert_eq "it removes each of them" "$_from" \ + "$(grep -c '^rm -f made-' "$F/log" || true)" +assert_eq "before the snapshot" "rm -f made|rmi proj-agent-retire" \ + "$(grep -E '^(rm -f |rmi proj-agent-retire)' "$F/log" \ + | sed 's/:.*//; s/-[0-9]*$//' | uniq | tr '\n' '|' | sed 's/|$//')" +assert_eq "and no engine container" "0" "$(removed)" +rm -f "$F/term-on-scan" "$F/leave-helpers" "$F/made" +# Whether a job started here under job control can be stopped by +# SIGINT: not when this run started with SIGINT ignored (a background +# job of a script), since the ignore is inherited and cannot be undone. +# Bash 3.2 does not report such an ignore through trap -p, so try it, +# in a subshell: job control switched on in the test's own shell would +# stay with it on Bash 3.2. +sigint_works() { + local pid tries=0 + set -m + sleep 30 & + pid=$! + set +m + # Signal it once it runs sleep: before its exec, the shell's own + # handling of SIGINT still applies and the signal can be lost. + while [ "$tries" -lt 40 ]; do + case "$(ps -o comm= -p "$pid" 2>/dev/null)" in + *sleep) break ;; + esac + sleep 0.05 + tries=$((tries + 1)) + done + kill -INT "$pid" 2>/dev/null + tries=0 + while kill -0 "$pid" 2>/dev/null && [ "$tries" -lt 20 ]; do + sleep 0.05 + tries=$((tries + 1)) + done + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 1 + fi + wait "$pid" 2>/dev/null + return 0 +} +# A Ctrl-C while the daemon commits: retire waits for the commit, then +# removes the snapshot, so none is left once the daemon is done. +if ! (sigint_works); then + echo " SKIP: a Ctrl-C during the commit (SIGINT is ignored here)" +else + reset_fake + add proj-agent-1 + : > "$F/int-on-commit" + assert_eq "a Ctrl-C during the commit stops retire" "130" \ + "$(retire_own_group)" + sleep 2 + assert_eq "no snapshot is left once the daemon is done" "0" \ + "$(grep -c . "$F/images" || true)" + assert_eq "it removes no container then" "0" "$(removed)" + rm -f "$F/int-on-commit" +fi +# A signal while retire removes its snapshot (TERM during the first +# docker rmi, which removes nothing): the cleanup still has the name +# and removes it. +reset_fake +add proj-agent-1 +: > "$F/term-on-rmi" +assert_eq "a retire stopped while it removes its snapshot" "143" "$(retire)" +assert_eq "the snapshot is removed all the same" "0" \ + "$(grep -c . "$F/images" || true)" +assert_eq "it removes no container there" "0" "$(removed)" +# A signal right after the commit job starts, before its pid is known: +# the cleanup still waits for the commit, then removes the snapshot. +reset_fake +_rc=0 +PATH="$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" RETIRE_LIB="$REPO_ROOT/lib/retire.sh" \ + "$BASH" -c ' + source "$RETIRE_LIB" + RETIRE_SNAP_REF="proj-agent-retire:boundary-1" + # The commit job, whose daemon makes the image a second later. + ( sleep 1 + printf "%s sha256:%064d\n" "$RETIRE_SNAP_REF" 7 \ + >> "$FAKE_DOCKER/images" ) & + retire_cleanup + ' || _rc=$? +# Long enough for the daemon of a commit the cleanup did not wait for. +sleep 2 +assert_eq "a cleanup before the commit's pid is known succeeds" "0" "$_rc" +assert_eq "it waits for the commit and removes the snapshot" "0" \ + "$(grep -c . "$F/images" || true)" +assert_eq "by its name" "1" \ + "$(grep -c '^rmi proj-agent-retire:boundary-1$' "$F/log" || true)" +# Snapshots an earlier retire left (one killed outright) go first. +reset_fake +add proj-agent-1 +printf 'proj-agent-retire:old-1 sha256:%064d\n' 9 > "$F/images" +assert_eq "retire with a snapshot an earlier one left" "0" "$(retire)" +assert_eq "it removes that snapshot" "1" \ + "$(grep -c '^rmi proj-agent-retire:old-1$' "$F/log" || true)" +assert_eq "and says so" "1" \ + "$(grep -c 'a snapshot an earlier retire left' "$TMPDIR/out" || true)" +assert_eq "no snapshot is left after" "0" "$(grep -c . "$F/images" || true)" +reset_fake +add proj-agent-1 +: > "$F/imagesfail" +refused "images that cannot be listed" "cannot list Docker images" +case_setup +_d=$(real_ws) +ln -s "$_ct/img-run/x" "$_d/x" +printf 'C /usr\nA /usr/local/x\n' > "$F/diff/id-proj-agent-2" +assert_eq "unrelated changes in the container do not stop retire" "0" \ + "$(retire)" +# A report that cannot be read refuses rather than passing as clean. +mkdir -p "$TMPDIR/bin-grep" +cat > "$TMPDIR/bin-grep/grep" <<'EOF' +#!/bin/bash +for a in "$@"; do + case "$a" in */report) [ "$prev" = "^block " ] && exit 2 ;; esac + prev="$a" +done +exec /usr/bin/grep "$@" +EOF +chmod +x "$TMPDIR/bin-grep/grep" +case_setup +rc=0 +(cd "$REPO" && PATH="$TMPDIR/bin-grep:$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" \ + FAKE_CK="$CK" SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + SWARM_SKIP_DEP_CHECK=1 "$BASH" "$LAUNCH" retire > "$TMPDIR/out" 2>&1) \ + || rc=$? +assert_eq "an unreadable config check report refuses" "1" "$rc" +assert_eq "for that reason" "1" \ + "$(grep -c 'cannot read the git config check' "$TMPDIR/out" || true)" +assert_eq "nothing removed then" "0" "$(removed)" + +# A failing comparison refuses rather than passing. +mkdir -p "$TMPDIR/bin-awk" +cat > "$TMPDIR/bin-awk/awk" <<'EOF' +#!/bin/bash +case "$*" in *'index(p, x'*) exit 2 ;; esac +exec /usr/bin/awk "$@" +EOF +chmod +x "$TMPDIR/bin-awk/awk" +case_setup +_d=$(real_ws) +ln -s "$_ct/img-run/x" "$_d/x" +: > "$F/diff/id-proj-agent-2" +rc=0 +(cd "$REPO" && PATH="$TMPDIR/bin-awk:$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" \ + FAKE_CK="$CK" SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + SWARM_SKIP_DEP_CHECK=1 "$BASH" "$LAUNCH" retire > /dev/null 2>&1) \ + || rc=$? +assert_eq "a failed link comparison refuses" "1" "$rc" +assert_eq "nothing removed then" "0" "$(removed)" + +case_setup +_d=$(real_ws) +mkdir -p "$_d/pool/info" +ln -s "$L/alternates" "$_d/pool/info/alternates" +echo "$_d/pool" > "$_d/.git/objects/info/alternates" +mkdir -p "$F/diff" +printf 'A %s/alternates\n' "$L" > "$F/diff/id-proj-agent-2" +refused "an in-volume object pool whose alternates link into the layer" "links out of its volume to data" +case_setup +_d=$(real_ws) +ln -s / "$_d/rootlink" +mkdir -p "$F/diff" +printf 'A /anything\n' > "$F/diff/id-proj-agent-2" +refused "a link to / with anything in the layer" "outside-dir-link" +case_setup +_d=$(real_ws) +git init -q --bare "$TMPDIR/alt-store.git" +printf '%s' "$TMPDIR/alt-store.git/objects" > "$_d/.git/objects/info/alternates" +refused "alternates to an existing store outside, without a final newline" "alternates .git/objects" +[ -z "${SHOW_REASON:-}" ] || grep -c 'alternates' "$TMPDIR/out" + +case_setup +echo "uncommitted dirty .: M a" > "$F/scan/proj-agent-2-ws-00000000" +assert_eq "uncommitted files in the volume do not stop retire" "0" \ + "$(retire)" + +# An open interactive session keeps its container. +case_setup exited interactive +sleep 60 & +_live=$! +swarm_pending_write "$RT" proj-agent-2 "$_live" d1 +refused "an open interactive session" "session proj-agent-2 is open" +kill "$_live" 2>/dev/null || true +wait "$_live" 2>/dev/null || true + +echo "" +echo "=== 3. Removal acts only on what was checked ===" +reset_fake +add proj-agent-1 +jq '.[0].State.StartedAt = "S2"' "$F/inspect/id-proj-agent-1" \ + > "$F/inspect/id-proj-agent-1.now" +assert_eq "a container restarted since the check is left" "1" "$(retire)" +assert_eq "and not removed" "0" "$(removed)" +reset_fake +add proj-agent-1 +add proj-agent-2 +: > "$F/rmfail/id-proj-agent-1" +assert_eq "a failed docker rm fails retire" "1" "$(retire)" +assert_eq "the other container is still removed" "rm id-proj-agent-2" \ + "$(grep '^rm ' "$F/log" | grep -v proj-agent-1 || true)" + +echo "" +echo "=== 4. Engagement, daemon and lock ===" +reset_fake +add proj-agent-1 +echo d2 > "$F/daemon" +assert_eq "another Docker daemon is refused" "1" "$(retire)" +rm -f "$F/daemon" +swarm_state_update "$RT/state.json" 'del(.docker_daemon)' +assert_eq "no recorded daemon is refused" "1" "$(retire)" +assert_eq "the message says how to name it" "1" \ + "$(grep -c 'SWARM_DOCKER_DAEMON=' "$TMPDIR/out" || true)" +rc=0 +(cd "$REPO" && PATH="$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" FAKE_CK="$CK" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + SWARM_DOCKER_DAEMON=d1 "$BASH" "$LAUNCH" retire > /dev/null 2>&1) \ + || rc=$? +assert_eq "SWARM_DOCKER_DAEMON names it" "0" "$rc" +swarm_state_update "$RT/state.json" '.docker_daemon = "d1"' +reset_fake +add proj-agent-1 +rc=0 +(cd "$REPO" && PATH="$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" FAKE_CK="$CK" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + SWARM_ENGAGEMENT_ID=e2 "$BASH" "$LAUNCH" retire > /dev/null 2>&1) \ + || rc=$? +assert_eq "another engagement id is refused" "1" "$rc" +exec 8>>"$RT/engagement.lock" +flock -n 8 +assert_eq "a held lock is refused" "1" "$(retire 8>&-)" +assert_eq "nothing removed then" "0" "$(removed)" +exec 8>&- + +echo "" +echo "===============================" +echo " ${PASS} passed, ${FAIL} failed" +echo "===============================" + +[ "$FAIL" -eq 0 ] From 5fae7225c8a1935bc61cd43fdcceb1922578ab62 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Wed, 4 Nov 2026 23:04:49 +0100 Subject: [PATCH 07/27] Remove launch.sh's temporary files however it exits --- CHANGELOG.md | 6 +++ launch.sh | 36 ++++++++++++- tests/_test_env.sh | 36 +++++++++++++ tests/test_launch.sh | 118 +++++++++++++++++++++++++++++++++++++++++++ tests/test_retire.sh | 35 ------------- 5 files changed, 194 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef85d68..289b605 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- **Fix: launch.sh leaves no temporary files behind.** The profile file + of `interactive` and the agents' list of `start` were removed only + when the command got that far; any earlier exit, a failure or Ctrl-C + left them, on macOS in the user's temporary directory whatever + `TMPDIR` said. They are now made in `TMPDIR` (or `/tmp`) and removed + when `launch.sh` exits, however it does. - **Feature: `launch.sh retire [--post-process]`.** Removes this checkout's stopped engine containers (or only the post-process one) by ID, without `-f`, after checking every selected container first: diff --git a/launch.sh b/launch.sh index 9a0acb7..b22ce45 100755 --- a/launch.sh +++ b/launch.sh @@ -438,6 +438,38 @@ case "${1:-}" in ;; esac +# Temporary files this run makes, removed when it exits, however it +# does: Bash runs the EXIT trap on exit, under set -e, and when HUP, +# INT or TERM end the script (retire, above, makes none and sets its +# own traps). +SWARM_TMP_FILES=() +swarm_tmp_cleanup() { + if [ "${#SWARM_TMP_FILES[@]}" -gt 0 ]; then + rm -f "${SWARM_TMP_FILES[@]}" + fi +} +trap swarm_tmp_cleanup EXIT + +# Make a temporary file swarm-NAME.XXXXXX in TMPDIR, removed when this +# run exits, and set the variable VAR to its absolute path (the run +# may change directory before it exits). +swarm_tmp_file() { + local _dir="${TMPDIR:-/tmp}" _tmp + case "$_dir" in + /*) ;; + *) + # As for the swarmfile's directory: CDPATH cleared, links + # resolved, trailing newlines kept. + _dir=$(CDPATH='' cd -P -- "$_dir" && pwd -P && echo x) \ + || return 1 + _dir="${_dir%x}" + _dir="${_dir%$'\n'}" ;; + esac + _tmp=$(mktemp "${_dir%/}/swarm-$2.XXXXXX") || return 1 + SWARM_TMP_FILES+=("$_tmp") + printf -v "$1" '%s' "$_tmp" +} + CONFIG_FILE="${SWARM_CONFIG:-}" if [ -z "$CONFIG_FILE" ] && [ -f "$REPO_ROOT/swarm.json" ]; then CONFIG_FILE="$REPO_ROOT/swarm.json" @@ -630,7 +662,7 @@ HELP done local profile_file - profile_file=$(mktemp) + swarm_tmp_file profile_file profile || exit 1 select_interactive_profile "$selector" "$selector_index" "$profile_file" local profile_name profile_label safe_profile short_id branch name @@ -844,7 +876,7 @@ cmd_start() { # Build per-agent config (model|base_url|api_key|effort|auth|context|prompt|auth_token|tag|driver per line). # Uses pipe delimiter because bash IFS=$'\t' collapses consecutive tabs. - AGENTS_CFG=$(mktemp) + swarm_tmp_file AGENTS_CFG agents || exit 1 jq -r '.tag as $dt | .driver as $dd | .agents[] | range(.count // 0) as $i | [.model, (.base_url // ""), (.api_key // ""), (.effort // ""), (.auth // ""), (.context // ""), (.prompt // ""), (.auth_token // ""), (.tag // $dt // ""), (.driver // $dd // "")] | join("|")' \ diff --git a/tests/_test_env.sh b/tests/_test_env.sh index cfebde5..a48360d 100644 --- a/tests/_test_env.sh +++ b/tests/_test_env.sh @@ -8,3 +8,39 @@ unset GIT_CONFIG GIT_CONFIG_PARAMETERS # Keep it independent of the caller's swarm-core settings too. unset SWARM_PROJECT SWARM_RUNTIME_DIR SWARM_ENGAGEMENT_ID SWARM_LOCK_FD + +# Succeed when a job started under job control can be stopped by +# SIGINT: not when the test started with SIGINT ignored (a background +# job of a script), since the ignore is inherited and cannot be undone. +# Bash 3.2 does not report such an ignore through trap -p, so this +# tries it. Call it in a subshell, (sigint_works): job control +# switched on in the test's own shell would stay with it on Bash 3.2. +sigint_works() { + local pid tries=0 + set -m + sleep 30 & + pid=$! + set +m + # Signal it once it runs sleep: before its exec, the shell's own + # handling of SIGINT still applies and the signal can be lost. + while [ "$tries" -lt 40 ]; do + case "$(ps -o comm= -p "$pid" 2>/dev/null)" in + *sleep) break ;; + esac + sleep 0.05 + tries=$((tries + 1)) + done + kill -INT "$pid" 2>/dev/null + tries=0 + while kill -0 "$pid" 2>/dev/null && [ "$tries" -lt 20 ]; do + sleep 0.05 + tries=$((tries + 1)) + done + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null + wait "$pid" 2>/dev/null + return 1 + fi + wait "$pid" 2>/dev/null + return 0 +} diff --git a/tests/test_launch.sh b/tests/test_launch.sh index 988c7b9..3658465 100755 --- a/tests/test_launch.sh +++ b/tests/test_launch.sh @@ -3298,6 +3298,124 @@ assert_eq "the old record is gone" "false" \ "$([ -e "$_pd_rt/interactive-$_pd_id_a.pending" ] \ && echo true || echo false)" +# ============================================================ +echo "" +echo "=== 54. No temporary file outlives launch.sh ===" + +# interactive stopped by an agent index that does not exist, start +# stopped by a driver that does not exist once it wrote the agents' +# list, and interactive stopped by a Ctrl-C while the image builds: +# every file mktemp made for them is gone when they exit. A mktemp +# first on PATH logs what the real one makes, NUL-terminated. +_tf_bin="$TMPDIR/tf-bin" +_tf_sig="$TMPDIR/tf-sig" +_tf_log="$TMPDIR/tf-mktemp.log" +_tf_rt="$TMPDIR/tf-runtime" +mkdir -p "$_tf_bin" "$_tf_sig" +cat > "$_tf_bin/mktemp" <<'EOF' +#!/bin/bash +out=$(PATH="$TF_PATH" mktemp "$@") || exit $? +printf '%s\0' "$out" >> "$TF_LOG" +printf '%s\n' "$out" +EOF +# docker build runs until a signal ends it, as a long build would; +# every other call goes to the start tests' docker. +cat > "$_tf_sig/docker" <<'EOF' +#!/bin/bash +[ "$1" != build ] || exec sleep 30 +exec "$TF_DOCKER" "$@" +EOF +chmod +x "$_tf_bin/mktemp" "$_tf_sig/docker" +run_tf() { + local orig="$PATH" + : > "$_tf_log" + (cd "$_st_repo" && TF_PATH="$orig" TF_LOG="$_tf_log" \ + PATH="$_tf_bin:$_st_bin:$orig" SWARM_RUNTIME_DIR="$_tf_rt" \ + SWARM_SKIP_DEP_CHECK=1 "$@" 2>&1) +} +# Print how many of the files mktemp made (the log) are still there. +tf_left() { + local f n=0 + while IFS= read -r -d '' f; do + [ ! -e "$f" ] || n=$((n + 1)) + done < "$_tf_log" + printf '%s' "$n" +} +# Remove what is left of them. +tf_drop() { + local f + while IFS= read -r -d '' f; do + rm -f -- "$f" + done < "$_tf_log" +} +_tf_rc=0 +run_tf env SWARM_CONFIG=sub/configs/swarm.json \ + bash "$_LAUNCH_SH" interactive --agent-index 9 >/dev/null || _tf_rc=$? +assert_eq "interactive with an unknown agent index fails" "1" "$_tf_rc" +assert_eq "it made a temporary file" "true" \ + "$([ -s "$_tf_log" ] && echo true || echo false)" +assert_eq "which is gone" "0" "$(tf_left)" +tf_drop +# A relative TMPDIR is made absolute first (a run may change directory +# before it exits), whatever CDPATH says. +mkdir -p "$_st_repo/sub/tf-rel" +_tf_rc=0 +run_tf env SWARM_CONFIG=sub/configs/swarm.json TMPDIR=sub/tf-rel CDPATH=. \ + bash "$_LAUNCH_SH" interactive --agent-index 9 >/dev/null || _tf_rc=$? +assert_eq "so does a run with a relative TMPDIR" "1" "$_tf_rc" +_tf_first="" +IFS= read -r -d '' _tf_first < "$_tf_log" || true +assert_eq "whose file had an absolute path in it" \ + "$(cd -P "$_st_repo/sub/tf-rel" && pwd -P)" "${_tf_first%/swarm-profile.*}" +assert_eq "and is gone" "0" "$(tf_left)" +tf_drop +rmdir "$_st_repo/sub/tf-rel" +jq '.agents[0].driver = "no-such-driver"' \ + "$_st_repo/sub/configs/swarm.json" > "$_st_repo/sub/configs/tf.json" +_tf_rc=0 +_tf_out=$(run_tf env SWARM_CONFIG=sub/configs/tf.json \ + bash "$_LAUNCH_SH" start) || _tf_rc=$? +assert_eq "start with an unknown driver fails" "1" "$_tf_rc" +assert_eq "once it wrote the agents' list" "true" \ + "$(grep -q 'unknown driver' <<< "$_tf_out" && echo true || echo false)" +assert_eq "every temporary file it made is gone" "0" "$(tf_left)" +tf_drop +rm -f "$_st_repo/sub/configs/tf.json" +# A Ctrl-C at the terminal while the image builds: SIGINT to the +# process group of launch.sh, run in one of its own with /bin/bash +# (Bash 3.2 on macOS), once docker build runs. +if ! (sigint_works); then + echo " SKIP: a Ctrl-C while the image builds (SIGINT is ignored here)" +else + _tf_rc=0 + _tf_orig="$PATH" + : > "$_tf_log" + set -m + (cd "$_st_repo" && TF_PATH="$_tf_orig" TF_LOG="$_tf_log" \ + TF_DOCKER="$_st_bin/docker" \ + PATH="$_tf_sig:$_tf_bin:$_st_bin:$_tf_orig" \ + SWARM_RUNTIME_DIR="$_tf_rt" SWARM_SKIP_DEP_CHECK=1 \ + SWARM_CONFIG=sub/configs/swarm.json \ + exec /bin/bash "$_LAUNCH_SH" interactive --agent-index 1 \ + > "$TMPDIR/tf-int.out" 2>&1) & + _tf_pid=$! + set +m + _tf_tries=0 + while [ "$_tf_tries" -lt 200 ] \ + && ! pgrep -g "$_tf_pid" -x sleep >/dev/null 2>&1; do + sleep 0.05 + _tf_tries=$((_tf_tries + 1)) + done + kill -INT -- "-$_tf_pid" 2>/dev/null + wait "$_tf_pid" || _tf_rc=$? + assert_eq "a Ctrl-C while the image builds stops interactive" "130" \ + "$_tf_rc" + assert_eq "after it made its temporary file" "true" \ + "$([ -s "$_tf_log" ] && echo true || echo false)" + assert_eq "which is gone" "0" "$(tf_left)" + tf_drop +fi + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" diff --git a/tests/test_retire.sh b/tests/test_retire.sh index b7ecdae..8f8df3b 100755 --- a/tests/test_retire.sh +++ b/tests/test_retire.sh @@ -1143,41 +1143,6 @@ assert_eq "before the snapshot" "rm -f made|rmi proj-agent-retire" \ | sed 's/:.*//; s/-[0-9]*$//' | uniq | tr '\n' '|' | sed 's/|$//')" assert_eq "and no engine container" "0" "$(removed)" rm -f "$F/term-on-scan" "$F/leave-helpers" "$F/made" -# Whether a job started here under job control can be stopped by -# SIGINT: not when this run started with SIGINT ignored (a background -# job of a script), since the ignore is inherited and cannot be undone. -# Bash 3.2 does not report such an ignore through trap -p, so try it, -# in a subshell: job control switched on in the test's own shell would -# stay with it on Bash 3.2. -sigint_works() { - local pid tries=0 - set -m - sleep 30 & - pid=$! - set +m - # Signal it once it runs sleep: before its exec, the shell's own - # handling of SIGINT still applies and the signal can be lost. - while [ "$tries" -lt 40 ]; do - case "$(ps -o comm= -p "$pid" 2>/dev/null)" in - *sleep) break ;; - esac - sleep 0.05 - tries=$((tries + 1)) - done - kill -INT "$pid" 2>/dev/null - tries=0 - while kill -0 "$pid" 2>/dev/null && [ "$tries" -lt 20 ]; do - sleep 0.05 - tries=$((tries + 1)) - done - if kill -0 "$pid" 2>/dev/null; then - kill -TERM "$pid" 2>/dev/null - wait "$pid" 2>/dev/null - return 1 - fi - wait "$pid" 2>/dev/null - return 0 -} # A Ctrl-C while the daemon commits: retire waits for the commit, then # removes the snapshot, so none is left once the daemon is done. if ! (sigint_works); then From eff0bff56c69fcd88bf728870874f6399d161d8a Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:20:08 +0100 Subject: [PATCH 08/27] Add launch.sh cleanup that copies every repository location into the checkout before deleting it --- CHANGELOG.md | 20 + USAGE.md | 66 ++- harvest.sh | 1 + launch.sh | 36 +- lib/cleanup.sh | 906 +++++++++++++++++++++++++++++++++++++ lib/preserve-scan.sh | 198 ++++++--- lib/preserve.sh | 90 +++- lib/project.sh | 36 +- lib/retire.sh | 11 +- tests/runtime_cleanup.sh | 218 +++++++++ tests/test.sh | 15 +- tests/test_cleanup.sh | 934 +++++++++++++++++++++++++++++++++++++++ tests/test_preserve.sh | 161 +++++++ tests/test_retire.sh | 37 ++ 14 files changed, 2618 insertions(+), 111 deletions(-) create mode 100644 lib/cleanup.sh create mode 100755 tests/runtime_cleanup.sh create mode 100755 tests/test_cleanup.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 289b605..4c9a145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +- **Feature: `launch.sh cleanup [--discard-uncommitted]`.** Deletes this + checkout's engagement (containers, through retire's checks; workspace + volumes; the bare repo; the mirrors; leftovers of an earlier cleanup) + after copying everything Git keeps in each location into the checkout + as recovery refs (`refs/swarm-recovery/e//s/`, + `` being the engagement id in hex, with a `sources` blob naming + each) and verifying them with `git fsck --full`. It refuses while the + bare repo holds agent work the checkout lacks, while a session is open + or starting, when a location holds work outside Git's history (unless + `--discard-uncommitted`, which drops exactly what it lists), when + the checkout cannot keep the copies by itself, or when something is + mounted where it would delete. A checkpoint in the state file lets a + run that stopped half way be finished by the next; every run checks + everything again and checks the recovery refs earlier runs wrote, and + start, post-process, interactive, stop and harvest.sh refuse + meanwhile. Files the containers wrote are deleted through a throwaway + engine-image container as root, only once a token shows it sees the + same directory. The scanner gains a `files` mode for what a deletion + left of a repository. retire and cleanup ignore the caller's `GIT_*` + variables (except the choice of config files). - **Fix: launch.sh leaves no temporary files behind.** The profile file of `interactive` and the agents' list of `start` were removed only when the command got that far; any earlier exit, a failure or Ctrl-C diff --git a/USAGE.md b/USAGE.md index 6c183b2..ba7c9a7 100644 --- a/USAGE.md +++ b/USAGE.md @@ -1218,26 +1218,60 @@ that recorded none needs `SWARM_DOCKER_DAEMON=`). ## Cleanup -After a swarm run, the agent containers (`-agent-N`, -`-agent-post`, `-agent-interactive-*`) and the -[runtime directory](#runtime-directory) remain. Harvest first, then -remove both for a fresh start. The agents write to the bare repo as -the container's user, and on Linux a plain `rm -rf` cannot delete -those files, so the bare repo goes through Docker first, the way -`launch.sh` removes an old one. Use the directory that -`launch.sh start` printed on its `Runtime:` line: +`./launch.sh cleanup` deletes this checkout's engagement: its +containers, its workspace volumes, the bare repo and the mirrors. It +loses nothing Git kept there: before it deletes a repository location +it copies everything Git keeps in it (every ref and reflog entry, +every object, stashes, unfinished operations, and every file of the +git dir) into the checkout as recovery refs, and checks them there. ```bash -docker rm -f $(docker ps -aq \ - --filter "label=swarm-core.project=") 2>/dev/null -docker run --rm -v ":/runtime" alpine \ - rm -rf /runtime/upstream.git /runtime/mirrors && -rm -rf "" +./launch.sh cleanup [--discard-uncommitted] ``` -The bare repo and mirrors are removed before the runtime directory -and its state file. This needs a running Docker daemon and the -`alpine` image, which Docker pulls if it is missing. +It stops, and deletes nothing, when + +- the bare repo holds agent work the checkout does not have yet + (`agent-work`, `swarm/*`, `agent-parked/*`): harvest first; +- a container cannot be retired (see [Retiring + containers](#retiring-containers)), or an interactive session is open + or starting; +- a workspace, the bare repo or a mirror holds work outside Git's + history (uncommitted or ignored files, files too large to copy): + commit or copy it out, or pass `--discard-uncommitted`, which drops + exactly what it lists; +- `mirrors/` holds anything that is not a mirror; +- the checkout cannot keep the copies by itself (a shallow or partial + clone, alternates, an object store inside what cleanup deletes), or + `git fsck --full` finds problems in it; +- the Docker daemon is not the engagement's, or sees other files than + this host at the runtime directory's path (a remote daemon); +- the runtime directory is `/`, your home directory or another user's, + holds or lies inside the checkout or its git dir, or has a comma, a + quote, a tab or a newline in its path; +- something is mounted inside the runtime directory, or at a workspace + volume's directory on this host (cleanup never deletes through a + mount). + +The recovery refs are `refs/swarm-recovery/e//s/...`, +`` being the engagement id in hex; +`git cat-file blob refs/swarm-recovery/e//sources` +says what each `s` was. The engine never deletes them. To drop them +once you are sure you do not need them: + +```bash +git for-each-ref --format='%(refname)' refs/swarm-recovery/ \ + | xargs -n 1 git update-ref -d +``` + +Files the agents wrote belong to the container's user; what this user +cannot delete goes through a throwaway container of the engine image, +as root. A cleanup that stops half way (an error, Ctrl-C) leaves a +checkpoint in the state file: start, post-process, interactive, stop +and harvest.sh refuse until `./launch.sh cleanup` runs again and +finishes. Each run checks everything again and copies whatever is still +there; it also checks that the recovery refs earlier runs wrote are +intact. The engagement lock, the runtime directory and its claim stay. ## Verify image diff --git a/harvest.sh b/harvest.sh index 19df272..bd0fc42 100755 --- a/harvest.sh +++ b/harvest.sh @@ -89,6 +89,7 @@ fi swarm_runtime_check "$RUNTIME_DIR" || exit 1 swarm_runtime_owner_check "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || exit 1 swarm_lock "$RUNTIME_DIR" || exit 1 +swarm_cleanup_pending_check "$RUNTIME_DIR/state.json" || exit 1 swarm_engagement_env_check || exit 1 swarm_engagement_join "$RUNTIME_DIR/state.json" || exit 1 CHECKOUT_PATH="$(swarm_checkout_path "$REPO_ROOT")" diff --git a/launch.sh b/launch.sh index b22ce45..9cfa0ac 100755 --- a/launch.sh +++ b/launch.sh @@ -31,6 +31,10 @@ Commands: retire [--post-process] Remove stopped containers whose files all live in their workspace volumes. + cleanup [--discard-uncommitted] + Delete the engagement (containers, workspace + volumes, bare repo, mirrors) after copying what + Git keeps there into the checkout. Start options: --dashboard Open the TUI dashboard after launch. @@ -64,6 +68,20 @@ check_deps git jq docker # shellcheck source=lib/project.sh source "$SWARM_DIR/lib/project.sh" +# retire and cleanup read and write Git data: none of the caller's GIT_* +# variables may point them at another repository or inject settings +# (only the choice of config files stays). Before the repository is +# looked up below. +case "${1:-}" in + retire|cleanup) + for _v in $(env | sed -n 's/^\(GIT_[A-Za-z0-9_]*\)=.*/\1/p'); do + case "$_v" in + GIT_CONFIG_GLOBAL|GIT_CONFIG_SYSTEM|GIT_CONFIG_NOSYSTEM) ;; + *) unset "$_v" ;; + esac + done ;; +esac + REPO_ROOT="$(git rev-parse --show-toplevel)" PROJECT_RAW="$(basename "$REPO_ROOT")" PROJECT="$(swarm_project_resolve "$REPO_ROOT")" || exit 1 @@ -240,6 +258,7 @@ prepare_runtime() { || exit 1 swarm_runtime_init "$RUNTIME_DIR" || exit 1 swarm_lock "$RUNTIME_DIR" || exit 1 + swarm_cleanup_pending_check "$STATE_FILE" || exit 1 swarm_runtime_claim "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || exit 1 } @@ -436,12 +455,24 @@ case "${1:-}" in cmd_retire "$@" || rc=$? exit "$rc" ;; + cleanup) + shift + # shellcheck source=lib/preserve.sh + source "$SWARM_DIR/lib/preserve.sh" + # shellcheck source=lib/retire.sh + source "$SWARM_DIR/lib/retire.sh" + # shellcheck source=lib/cleanup.sh + source "$SWARM_DIR/lib/cleanup.sh" + rc=0 + cmd_cleanup "$@" || rc=$? + exit "$rc" + ;; esac # Temporary files this run makes, removed when it exits, however it # does: Bash runs the EXIT trap on exit, under set -e, and when HUP, -# INT or TERM end the script (retire, above, makes none and sets its -# own traps). +# INT or TERM end the script (retire and cleanup, above, make none and +# set their own traps). SWARM_TMP_FILES=() swarm_tmp_cleanup() { if [ "${#SWARM_TMP_FILES[@]}" -gt 0 ]; then @@ -1015,6 +1046,7 @@ cmd_stop() { # mid-rebase on a busy bare repo. Override via env: # SWARM_STOP_TIMEOUT=120 ./launch.sh stop local stop_timeout="${SWARM_STOP_TIMEOUT:-60}" + swarm_cleanup_pending_check "$STATE_FILE" || exit 1 swarm_engagement_join "$STATE_FILE" || exit 1 echo "--- Stopping agents (grace ${stop_timeout}s) ---" for i in $(seq 1 "$NUM_AGENTS"); do diff --git a/lib/cleanup.sh b/lib/cleanup.sh new file mode 100644 index 0000000..d87f690 --- /dev/null +++ b/lib/cleanup.sh @@ -0,0 +1,906 @@ +#!/bin/bash +# launch.sh cleanup: delete this checkout's engagement (its workspace +# volumes, the bare repo, the mirrors, and the engagement's records) +# without losing anything Git kept there. Every repository location +# about to go is first copied into the checkout as recovery refs +# refs/swarm-recovery/e//s/... +# (with a "sources" blob that names each s), and verified there. +# Every run, first or resumed, checks everything again and preserves +# what is still there under a new snapshot; a checkpoint in the state +# file records what each run preserved, so a run that stops half way +# is finished by the next one, and other commands wait until it is. +# +# Sourced by launch.sh after lib/project.sh, lib/preserve.sh and +# lib/retire.sh; uses PROJECT, REPO_ROOT, CHECKOUT_PATH, RUNTIME_DIR, +# STATE_FILE, BARE_REPO, MIRROR_DIR and IMAGE_NAME. + +CLEANUP_DISCARD=false +CLEANUP_HELPER_LABEL="swarm-core.cleanup" + +# Print STRING as lowercase hex: any engagement id becomes a valid ref +# component. +cleanup_hex() { + printf '%s' "$1" | od -An -tx1 | tr -d ' \n' +} + +# Print the cleanup checkpoint of the state file as JSON (null when +# there is none). +cleanup_checkpoint() { + swarm_state_read "$STATE_FILE" | jq -c '.cleanup // null' +} + +# Print the sha256 of the " " lines under PREFIX/ in the +# checkout, sorted: the evidence that what a run preserved is still +# there, ref for ref. +cleanup_inventory() { + local prefix="$1" list + list=$(swarm_recovery_git -C "$REPO_ROOT" for-each-ref \ + --format='%(refname) %(objectname)' "$prefix/" 2>/dev/null) \ + || return 1 + printf '%s\n' "$list" | LC_ALL=C sort | swarm_sha256 +} + +# sha256 of standard input, hex only. +swarm_sha256() { + local h + if command -v sha256sum >/dev/null 2>&1; then + h=$(sha256sum) || return 1 + else + h=$(shasum -a 256) || return 1 + fi + printf '%s\n' "${h%% *}" +} + +# Print the number of lines of FILE; fails when it cannot be read. A +# read that fails ends a while-read loop like the end of its list, so +# every loop over a list cleanup acts on compares its count with this. +cleanup_lines() { + local count + count=$(wc -l < "$1") || return 1 + printf '%s' "$((count + 0))" +} + +# Succeed when the runtime directory is safe to delete things in: a +# directory of this user's, and not /, $HOME, the checkout, its git +# dir or anything around them. +cleanup_runtime_sane() { + local rt home root common gitdir x owner + if [ -L "$RUNTIME_DIR" ] || [ ! -d "$RUNTIME_DIR" ]; then + echo "ERROR: ${RUNTIME_DIR} is not a directory." >&2 + return 1 + fi + rt=$(cd -P "$RUNTIME_DIR" && pwd -P) || return 1 + owner=$(swarm_owner_uid "$rt") || return 1 + if [ "$owner" != "$(id -u)" ]; then + echo "ERROR: ${RUNTIME_DIR} does not belong to this user." >&2 + return 1 + fi + home=$(cd -P "${HOME:-/}" 2>/dev/null && pwd -P) || home=/ + # The checkout, its common git dir and its own git dir (a linked + # worktree's lives apart from the common one). + if ! root=$(cd -P "$REPO_ROOT" 2>/dev/null && pwd -P) \ + || ! common=$(swarm_recovery_git -C "$REPO_ROOT" rev-parse \ + --git-common-dir 2>/dev/null) \ + || ! gitdir=$(swarm_recovery_git -C "$REPO_ROOT" rev-parse \ + --absolute-git-dir 2>/dev/null); then + echo "ERROR: cannot resolve the git dir of ${REPO_ROOT}." >&2 + return 1 + fi + case "$common" in + /*) ;; + *) common="$REPO_ROOT/$common" ;; + esac + if ! common=$(cd -P "$common" 2>/dev/null && pwd -P) \ + || ! gitdir=$(cd -P "$gitdir" 2>/dev/null && pwd -P); then + echo "ERROR: cannot resolve the git dir of ${REPO_ROOT}." >&2 + return 1 + fi + if [ "$rt" = / ] || [ "$rt" = "$home" ]; then + echo "ERROR: refusing to clean up in ${RUNTIME_DIR}." >&2 + return 1 + fi + # The Docker helper mounts it with --mount, whose value is + # comma-separated, and cleanup's lists are tab-separated lines. + case "$RUNTIME_DIR" in + *[,\"]*|*$'\n'*|*$'\t'*) + echo "ERROR: ${RUNTIME_DIR} holds a comma, a quote, a tab or a" \ + "newline; cleanup cannot work with it." >&2 + return 1 ;; + esac + for x in "$root" "$common" "$gitdir"; do + case "$x/" in + "$rt"/*) + echo "ERROR: ${RUNTIME_DIR} holds the checkout or its git" \ + "dir (${x})." >&2 + return 1 ;; + esac + case "$rt/" in + "$x"/*) + echo "ERROR: ${RUNTIME_DIR} is inside the checkout or its" \ + "git dir (${x})." >&2 + return 1 ;; + esac + done +} + +# Print the numeric owner of PATH. +swarm_owner_uid() { + if stat -c %u / >/dev/null 2>&1; then + stat -c %u "$1" + else + stat -f %u "$1" + fi +} + +# Wait for, then remove, the helper containers a killed cleanup of this +# project left (they carry the cleanup label), for at most +# SWARM_CLEANUP_HELPER_WAIT seconds (default 900) each. Nothing is +# inspected before they are gone, and only a listing that works and +# shows none counts as gone. +cleanup_settle_helpers() { + local ids c st waited limit="${SWARM_CLEANUP_HELPER_WAIT:-900}" + case "$limit" in + ''|*[!0-9]*) limit=900 ;; + esac + if ! ids=$(docker ps -aq --filter \ + "label=${CLEANUP_HELPER_LABEL}=${PROJECT}" 2>/dev/null); then + echo "ERROR: cannot list Docker containers." >&2 + return 1 + fi + for c in $ids; do + echo "Waiting for ${c}, a helper an earlier cleanup left..." + waited=0 + while :; do + if ! st=$(docker container inspect -f '{{.State.Status}}' "$c" \ + 2>/dev/null); then + # Gone already, or Docker cannot say: the listing below + # decides. + break + fi + case "$st" in + exited|created|dead) break ;; + esac + if [ "$waited" -ge "$limit" ]; then + echo "ERROR: the helper container ${c} is still ${st}" \ + "after ${limit}s; stop it (docker stop ${c}) and run" \ + "cleanup again." >&2 + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + docker rm "$c" >/dev/null 2>&1 || true + done + if ! ids=$(docker ps -aq --filter \ + "label=${CLEANUP_HELPER_LABEL}=${PROJECT}" 2>/dev/null); then + echo "ERROR: cannot list Docker containers." >&2 + return 1 + fi + if [ -n "$ids" ]; then + echo "ERROR: helper containers an earlier cleanup left remain:" \ + "$(printf '%s' "$ids" | tr '\n' ' ')" >&2 + return 1 + fi +} + +# Fail when an interactive session is pending or open: its launch can +# run after the lock is released, and would use what cleanup deletes. +cleanup_no_pending() { + local live + live=$(swarm_pending_live "$RUNTIME_DIR") || return 1 + if [ -n "$live" ]; then + echo "ERROR: interactive sessions are open or starting:" >&2 + printf '%s\n' "$live" | sed 's/^/ /' >&2 + echo " End them first." >&2 + return 1 + fi +} + +# Fail unless every snapshot the checkpoint records is still intact in +# the checkout: each source's " " list and each run's sources +# blob, from the first run on. A remnant deleted part of the way can +# no longer give back what is gone, so its earlier snapshot must hold. +cleanup_check_evidence() { + local cp tmp kind prefix want got seen=0 total + cp=$(cleanup_checkpoint) || return 1 + [ "$cp" != null ] || return 0 + tmp=$(retire_tmpdir) || return 1 + printf '%s' "$cp" | jq -r '.runs[]? as $r + | ($r.sources[]? | select(.inventory != null) + | "src\t\(.prefix)\t\(.inventory)"), + "map\t\($r.sources_ref)\t\($r.sources_oid)"' > "$tmp/evidence" \ + || return 1 + while IFS="$(printf '\t')" read -r -u 3 kind prefix want; do + seen=$((seen + 1)) + case "$kind" in + src) + got=$(cleanup_inventory "$prefix") || got="" ;; + map) + got=$(swarm_recovery_git -C "$REPO_ROOT" rev-parse \ + --verify --quiet "$prefix" 2>/dev/null) || got="" ;; + *) got="" ;; + esac + if [ -z "$got" ] || [ "$got" != "$want" ]; then + echo "ERROR: recovery refs an earlier cleanup wrote changed or" \ + "went: ${prefix}." >&2 + echo " Cleanup will not delete what they no longer keep." \ + "Restore them, or see USAGE.md (Cleanup) for the manual" \ + "steps." >&2 + return 1 + fi + done 3< "$tmp/evidence" < /dev/null || return 1 + total=$(cleanup_lines "$tmp/evidence") || return 1 + if [ "$seen" -ne "$total" ]; then + echo "ERROR: cannot read the recovery records of an earlier" \ + "cleanup." >&2 + return 1 + fi +} + +# Write the sources cleanup will delete to FILE, one per line: +# \t +# kind: removing, mirror, new, bare, volume. Remnants (*.removing) +# come first and are recognised before mirrors/ is checked: a mirror +# deleted part of the way is no longer a repository. Anything else in +# mirrors/ refuses, since nothing would preserve it. +cleanup_sources() { + local out="$1" e name base vols v rest odd="" + # Every write is checked: a location left out of this list would be + # neither preserved nor deleted. + : > "$out" || return 1 + # Hidden names too: a submodule's mirror can start with a dot. + for e in "$RUNTIME_DIR"/*.removing "$RUNTIME_DIR"/.[!.]*.removing \ + "$RUNTIME_DIR"/..?*.removing "$MIRROR_DIR"/*.removing \ + "$MIRROR_DIR"/.[!.]*.removing "$MIRROR_DIR"/..?*.removing; do + [ -e "$e" ] || [ -L "$e" ] || continue + case "$e" in + *$'\t'*|*$'\n'*) odd="${odd} ${e}"$'\n'; continue ;; + esac + if [ -L "$e" ]; then + echo "ERROR: ${e} is a link; cleanup will not follow it." >&2 + return 1 + fi + printf 'removing\t%s\n' "$e" >> "$out" || return 1 + done + if [ -e "$MIRROR_DIR" ] || [ -L "$MIRROR_DIR" ]; then + if [ -L "$MIRROR_DIR" ] || [ ! -d "$MIRROR_DIR" ]; then + echo "ERROR: ${MIRROR_DIR} is not a directory." >&2 + return 1 + fi + rest="" + for e in "$MIRROR_DIR"/* "$MIRROR_DIR"/.[!.]* "$MIRROR_DIR"/..?*; do + [ -e "$e" ] || [ -L "$e" ] || continue + name="${e##*/}" + case "$name" in + *.removing) continue ;; + *$'\t'*|*$'\n'*) odd="${odd} ${e}"$'\n'; continue ;; + esac + if [ ! -L "$e" ] && [ -d "$e" ] && [ -f "$e/HEAD" ] \ + && { [ -d "$e/objects" ] || [ -f "$e/commondir" ]; }; then + printf 'mirror\t%s\n' "$e" >> "$out" || return 1 + else + rest="${rest} ${e}"$'\n' + fi + done + if [ -n "$rest" ]; then + echo "ERROR: ${MIRROR_DIR} holds what is not a mirror, and" \ + "cleanup would delete it unpreserved:" >&2 + printf '%s' "$rest" >&2 + echo " Move it out first." >&2 + return 1 + fi + fi + if [ -n "$odd" ]; then + echo "ERROR: names with a tab or a newline, which cleanup's lists" \ + "cannot carry:" >&2 + printf '%s' "$odd" >&2 + echo " Rename them first." >&2 + return 1 + fi + for base in "${BARE_REPO}.new" "$BARE_REPO"; do + [ -e "$base" ] || [ -L "$base" ] || continue + if [ "$base" = "$BARE_REPO" ]; then + printf 'bare\t%s\n' "$base" >> "$out" || return 1 + else + printf 'new\t%s\n' "$base" >> "$out" || return 1 + fi + done + if ! vols=$(docker volume ls -q \ + --filter "label=${SWARM_PROJECT_LABEL}=${PROJECT}" \ + --filter "label=${SWARM_CHECKOUT_LABEL}=${CHECKOUT_PATH}" \ + 2>/dev/null); then + echo "ERROR: cannot list Docker volumes." >&2 + return 1 + fi + for v in $vols; do + printf 'volume\t%s\n' "$v" >> "$out" || return 1 + done +} + +# Print the mount points of this host under directory DIR (canonical), +# and with AT set, at DIR as well; one per line. Fails when the mount +# table cannot be read. +cleanup_mounts_under() { + local dir="$1" at="${2:-}" table + if [ -r /proc/self/mountinfo ]; then + # Field 5 is the mount point, with a space, a tab, a newline and + # a backslash written as \040, \011, \012 and \134. + D="$dir" AT="$at" awk '{ + m = $5 + gsub(/\\040/, " ", m); gsub(/\\011/, "\t", m) + gsub(/\\012/, "\n", m); gsub(/\\134/, "\\", m) + d = ENVIRON["D"] + if (index(m, d "/") == 1 || (ENVIRON["AT"] != "" && m == d)) + print m + }' /proc/self/mountinfo + else + # mount prints " on ()". + table=$(mount) || return 1 + printf '%s\n' "$table" | D="$dir" AT="$at" awk '{ + d = ENVIRON["D"] + if (index($0, " on " d "/") || + (ENVIRON["AT"] != "" && index($0, " on " d " ("))) + print + }' + fi +} + +# Fail when something is mounted under PATH, or with AT set, at PATH: +# deleting there would delete what the mount shows (a bind mount of the +# checkout's git dir, say). PATH need not exist on this host. +cleanup_no_mounts() { + local path="$1" at="${2:-}" canon found + canon=$(swarm_path_resolve "$path") || return 1 + if ! found=$(cleanup_mounts_under "$canon" "$at"); then + echo "ERROR: cannot read this host's mount table." >&2 + return 1 + fi + if [ -n "$found" ]; then + echo "ERROR: something is mounted in ${path}; cleanup will not" \ + "delete through a mount:" >&2 + printf '%s\n' "$found" | sed 's/^/ /' >&2 + return 1 + fi +} + +# Print the Mountpoint of volume NAME. +cleanup_volume_mountpoint() { + docker volume inspect -f '{{.Mountpoint}}' "$1" 2>/dev/null +} + +# Succeed when volume NAME is this checkout's (labels) and no container +# uses it. +cleanup_volume_free() { + local name="$1" labels users + labels=$(docker volume inspect -f '{{json .Labels}}' "$name" \ + 2>/dev/null) || { + echo "ERROR: cannot inspect volume ${name}." >&2 + return 1 + } + if ! printf '%s' "$labels" | jq -e \ + --arg pk "$SWARM_PROJECT_LABEL" --arg p "$PROJECT" \ + --arg ck "$SWARM_CHECKOUT_LABEL" --arg c "$CHECKOUT_PATH" \ + '.[$pk] == $p and .[$ck] == $c' >/dev/null 2>&1; then + echo "ERROR: volume ${name} is not this checkout's." >&2 + return 1 + fi + if ! users=$(docker ps -aq --filter "volume=${name}" 2>/dev/null); then + echo "ERROR: cannot list Docker containers." >&2 + return 1 + fi + if [ -n "$users" ]; then + echo "ERROR: volume ${name} is in use by containers:" \ + "$(printf '%s' "$users" | tr '\n' ' ')" >&2 + return 1 + fi +} + +# Scan source KIND at PATH (a volume name for kind volume) into the +# empty directory OUT; for a remnant, gitdir mode first and files mode +# when it is no longer a git dir or that scan fails. Prints the mode +# used. +cleanup_scan() { + local kind="$1" path="$2" out="$3" format context tag + case "$kind" in + volume) + context=$(docker volume inspect -f \ + "{{index .Labels \"${SWARM_CONTEXT_LABEL}\"}}" "$path" \ + 2>/dev/null) || context="" + tag=$(cleanup_hex "$path" | cut -c1-24) || return 1 + # shellcheck disable=SC2034 # read by swarm_scan_volume + SWARM_SCAN_DOCKER_ARGS=(--name "${PROJECT}-cleanup-scan-${tag}-$$" + --label "${CLEANUP_HELPER_LABEL}=${PROJECT}") + if swarm_scan_volume "$path" "$IMAGE_NAME" "${context:-full}" \ + "$out" 2>"$out.err"; then + # shellcheck disable=SC2034 + SWARM_SCAN_DOCKER_ARGS=() + echo volume + return 0 + fi + # shellcheck disable=SC2034 + SWARM_SCAN_DOCKER_ARGS=() + sed 's/^/ /' "$out.err" >&2 + return 1 ;; + removing) + if [ -f "$path/HEAD" ] \ + && { [ -d "$path/objects" ] \ + || [ -f "$path/commondir" ]; } \ + && swarm_scan_host "$path" "$out" 2>"$out.err"; then + echo gitdir + return 0 + fi + # What is left of a repository a deletion got part of the way + # through: keep every file. + rm -rf "$out" + mkdir "$out" || return 1 + format=$(swarm_recovery_git -C "$REPO_ROOT" rev-parse \ + --show-object-format 2>/dev/null) || return 1 + if bash "$(dirname "${BASH_SOURCE[0]}")/preserve-scan.sh" \ + files "$path" "$format" "$out" 2>"$out.err"; then + echo files + return 0 + fi + sed 's/^/ /' "$out.err" >&2 + return 1 ;; + *) + if swarm_scan_host "$path" "$out" 2>"$out.err"; then + echo gitdir + return 0 + fi + sed 's/^/ /' "$out.err" >&2 + return 1 ;; + esac +} + +# Apply the policy to scan report REPORT of SOURCE: block lines refuse; +# uncommitted lines refuse unless --discard-uncommitted, which lists +# them. Fails when the source may not be deleted. +cleanup_policy() { + local report="$1" source="$2" rc=0 + grep -a '^block ' "$report" > "$report.block" 2>/dev/null || rc=$? + case "$rc" in + 0) + echo "ERROR: ${source} holds what cleanup cannot keep:" >&2 + sed 's/^block / /' "$report.block" >&2 + return 1 ;; + 1) ;; + *) + echo "ERROR: cannot read the scan of ${source}." >&2 + return 1 ;; + esac + rc=0 + grep -a '^uncommitted ' "$report" > "$report.unc" 2>/dev/null || rc=$? + case "$rc" in + 0) + if [ "$CLEANUP_DISCARD" = true ]; then + echo "Discarding, from ${source}:" + sed 's/^uncommitted / /' "$report.unc" + return 0 + fi + echo "ERROR: ${source} holds work outside Git's history:" >&2 + sed 's/^uncommitted / /' "$report.unc" >&2 + echo " Commit or copy it out, or run cleanup with" \ + "--discard-uncommitted to drop exactly these." >&2 + return 1 ;; + 1) return 0 ;; + *) + echo "ERROR: cannot read the scan of ${source}." >&2 + return 1 ;; + esac +} + +# Record a change to source ID of the current run in the checkpoint: +# jq FILTER applied to that source object. Fails unless the run holds +# exactly one record of ID. +cleanup_record() { + local id="$1" filter="$2" + shift 2 + swarm_state_update "$STATE_FILE" " + if [.cleanup.runs[-1].sources[] | select(.id == \$sid)] | length != 1 + then error(\"no record of \" + \$sid) + else (.cleanup.runs[-1].sources[] | select(.id == \$sid)) + |= (${filter}) + end" --arg sid "$id" "$@" || { + echo "ERROR: cannot record source ${id} in ${STATE_FILE}." >&2 + return 1 + } +} + +# Delete host source PATH (the source with ID): renamed to PATH.removing +# first (unless it already is a remnant), then removed, by this user +# where it can be and through a Docker helper as root for what the +# containers wrote; it must be gone at the end. +cleanup_delete_host() { + local id="$1" path="$2" target token rel name rc=0 parent + # Only what lies inside the runtime directory (or its mirrors/) is + # ever renamed or deleted. + parent=$(cd -P "$(dirname "$path")" 2>/dev/null && pwd -P) || parent="" + if [ "$parent" != "$(cd -P "$RUNTIME_DIR" && pwd -P)" ] \ + && [ "$parent" != "$(cd -P "$MIRROR_DIR" 2>/dev/null && pwd -P)" ] + then + echo "ERROR: ${path} is not in ${RUNTIME_DIR}; not deleting it." >&2 + return 1 + fi + cleanup_no_mounts "$RUNTIME_DIR" || return 1 + case "$path" in + *.removing) target="$path" ;; + *) + target="${path}.removing" + if [ -e "$target" ] || [ -L "$target" ]; then + echo "ERROR: ${target} exists; cleanup will not rename into" \ + "it." >&2 + return 1 + fi + mv "$path" "$target" || { + echo "ERROR: cannot rename ${path}." >&2 + return 1 + } + cleanup_record "$id" '.renamed = $t | .state = "renamed"' \ + --arg t "$target" || return 1 ;; + esac + cleanup_record "$id" '.state = "deleting"' || return 1 + chmod -R u+rwX "$target" 2>/dev/null || true + rm -rf "$target" 2>/dev/null || true + if [ -e "$target" ] || [ -L "$target" ]; then + # What this user cannot delete (files the containers wrote as + # their own user) goes through Docker, as root. A token written + # here must be what the helper finds at the mounted path, or the + # daemon sees another file system (a remote daemon) and nothing + # is deleted. + # The marker gets a name nobody can know beforehand and is made + # only when nothing has that name (noclobber), so it cannot be + # sent through a link someone left in the remnant. + token=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') || return 1 + if [ -L "$target" ] || [ ! -d "$target" ] \ + || ! ( set -C; printf '%s\n' "$token" \ + > "$target/.swarm-cleanup-$token" ); then + echo "ERROR: cannot write into ${target}." >&2 + return 1 + fi + rel="${target#"$RUNTIME_DIR"/}" + name="${PROJECT}-cleanup-rm-${id}-$$" + cleanup_record "$id" '.helper = $n' --arg n "$name" || return 1 + swarm_docker_own_group run --rm --pull never --network none \ + --user 0 --entrypoint sh --name "$name" \ + --label "${CLEANUP_HELPER_LABEL}=${PROJECT}" \ + --mount "type=bind,src=${RUNTIME_DIR},dst=/runtime" \ + "$IMAGE_NAME" -c ' + t="/runtime/$1" + [ -f "$t/.swarm-cleanup-$2" ] \ + && [ "$(cat "$t/.swarm-cleanup-$2")" = "$2" ] \ + || { echo "not the same directory" >&2; exit 4; } + # Nothing may be mounted at or under it here: Docker + # binds a directory with the mounts below it, which the + # daemon may see and this host not. The table is read + # whole, by a command whose failure counts (a read in a + # loop would end at an error as at the end), and writes + # a space as \040 and a backslash as \134. + m=$(cat /proc/self/mountinfo) && [ -n "$m" ] \ + || { echo "cannot read the mount table" >&2; exit 5; } + e=$(printf "%s" "$t" | sed "s/\\\\/\\\\134/g; s/ /\\\\040/g") + m=$(printf "%s\n" "$m" | while read -r _ _ _ _ p _; do + case "$p" in + ("$e"|"$e"/*) printf "%s\n" "$p" ;; + esac + done) + [ -z "$m" ] \ + || { echo "something is mounted in it: $m" >&2; exit 5; } + rm -rf -- "$t" + ' sh "$rel" "$token" < /dev/null > /dev/null \ + 2>"$RETIRE_SCRATCH/rm.err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "ERROR: cannot delete ${target} through Docker:" \ + "$(cat "$RETIRE_SCRATCH/rm.err")" >&2 + return 1 + fi + fi + if [ -e "$target" ] || [ -L "$target" ]; then + echo "ERROR: ${target} is still there." >&2 + return 1 + fi + cleanup_record "$id" '.state = "deleted"' +} + +# The cleanup command; cmd_cleanup runs it with retire's cleanup and +# traps in place. +cleanup_run() { + while [ $# -gt 0 ]; do + case "$1" in + --discard-uncommitted) CLEANUP_DISCARD=true; shift ;; + -h|--help) + cat <&2; return 1 ;; + esac + done + if [ ! -f "$STATE_FILE" ]; then + echo "ERROR: no state file in ${RUNTIME_DIR}; cannot tell which" \ + "engagement to clean up." >&2 + return 1 + fi + swarm_lock "$RUNTIME_DIR" || return 1 + # Checked before the lock too, but another checkout of this project + # id could have claimed the runtime and started in between. + swarm_runtime_verify "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || return 1 + swarm_engagement_join "$STATE_FILE" || return 1 + cleanup_runtime_sane || return 1 + local engagement e + engagement=$(swarm_engagement_recorded "$STATE_FILE") || return 1 + if [ -z "$engagement" ]; then + # Nothing recorded: a cleaned-up runtime directory, unless engine + # state is left without an engagement to file it under. + for e in "$BARE_REPO" "${BARE_REPO}.new" "$MIRROR_DIR" \ + "$RUNTIME_DIR"/*.removing; do + if [ -e "$e" ] || [ -L "$e" ]; then + echo "ERROR: ${STATE_FILE} records no engagement, but" \ + "${e} is there." >&2 + echo " See USAGE.md (Cleanup) for the manual steps." >&2 + return 1 + fi + done + echo "No engagement recorded in ${STATE_FILE}; nothing to clean up." + return 0 + fi + retire_require_daemon || return 1 + cleanup_settle_helpers || return 1 + cleanup_no_pending || return 1 + cleanup_check_evidence || return 1 + + # The bare repo's agent work must be in the checkout already + # (harvest); recovery refs are the safety net, not the workflow. + local rc=0 + if [ -d "$BARE_REPO" ]; then + swarm_preservation_check "$BARE_REPO" "$REPO_ROOT" \ + > "$RETIRE_SCRATCH/e3" || rc=$? + case "$rc" in + 0) ;; + 1) + swarm_preservation_explain "$BARE_REPO" \ + < "$RETIRE_SCRATCH/e3" + return 1 ;; + *) return 1 ;; + esac + fi + + # Every container goes first, through retire's checks. + retire_run || { + echo "Cleanup stops: every container must be retired first." >&2 + return 1 + } + local left + left=$(swarm_own_containers "$PROJECT" "$REPO_ROOT") || return 1 + if [ -n "$left" ]; then + echo "ERROR: containers of this checkout remain:" >&2 + printf '%s\n' "$left" | sed 's/^/ /' >&2 + return 1 + fi + + local sources="$RETIRE_SCRATCH/sources" + cleanup_sources "$sources" || return 1 + + # Storage: the checkout must keep what it gets by itself. + local targets=() kind path mp seen=0 total + cleanup_no_mounts "$RUNTIME_DIR" || return 1 + while IFS="$(printf '\t')" read -r -u 3 kind path; do + seen=$((seen + 1)) + if [ "$kind" = volume ]; then + cleanup_volume_free "$path" || return 1 + mp=$(cleanup_volume_mountpoint "$path") || { + echo "ERROR: cannot inspect volume ${path}." >&2 + return 1 + } + if [ -n "$mp" ]; then + cleanup_no_mounts "$mp" at || return 1 + targets+=("$mp") + fi + else + targets+=("$path") + fi + done 3< "$sources" < /dev/null || return 1 + total=$(cleanup_lines "$sources") || return 1 + if [ "$seen" -ne "$total" ]; then + echo "ERROR: checked the storage of ${seen} of the ${total}" \ + "locations to clean up." >&2 + return 1 + fi + swarm_checkout_storage_ok "$REPO_ROOT" "$RUNTIME_DIR" \ + "${targets[@]+"${targets[@]}"}" || return 1 + + # Scan every source and apply the policy before anything is copied. + local n=0 dir mode + : > "$RETIRE_SCRATCH/scanned" || return 1 + while IFS="$(printf '\t')" read -r -u 3 kind path; do + n=$((n + 1)) + dir="$RETIRE_SCRATCH/scan.$n" + mkdir "$dir" || return 1 + mode=$(cleanup_scan "$kind" "$path" "$dir") || { + echo "ERROR: cannot scan ${path}." >&2 + return 1 + } + cleanup_policy "$dir/report" "$path" || return 1 + printf '%s\t%s\t%s\t%s\n' "$n" "$kind" "$path" "$mode" \ + >> "$RETIRE_SCRATCH/scanned" || return 1 + done 3< "$sources" < /dev/null || return 1 + if [ "$n" -ne "$total" ]; then + echo "ERROR: scanned ${n} of the ${total} locations to clean up." >&2 + return 1 + fi + + # Preserve every source under a new snapshot, then verify the + # checkout's whole object store once. + local snap hex base prefix inv empty sources_blob sources_ref + snap=$(swarm_recovery_snapshot_id) || return 1 + hex=$(cleanup_hex "$engagement") || return 1 + base="${SWARM_RECOVERY_NS}/e${hex}/${snap}" + printf '[]' > "$RETIRE_SCRATCH/run.json" || return 1 + while IFS="$(printf '\t')" read -r -u 3 n kind path mode; do + dir="$RETIRE_SCRATCH/scan.$n" + prefix="${base}/s${n}" + empty=false + inv=null + if grep -qx empty "$dir/report" 2>/dev/null \ + && [ ! -f "$dir/bundle" ]; then + empty=true + else + swarm_recovery_import "$REPO_ROOT" "$dir" "$prefix" || return 1 + inv=$(cleanup_inventory "$prefix") || return 1 + inv="\"$inv\"" + fi + jq --arg id "s$n" --arg k "$kind" --arg p "$path" \ + --arg pre "$prefix" --arg m "$mode" --argjson inv "$inv" \ + --argjson e "$empty" '. + [{id: $id, kind: $k, path: $p, + prefix: (if $e then null else $pre end), mode: $m, + inventory: $inv, empty: $e, state: "preserved"}]' \ + "$RETIRE_SCRATCH/run.json" > "$RETIRE_SCRATCH/run.json.new" \ + && mv "$RETIRE_SCRATCH/run.json.new" "$RETIRE_SCRATCH/run.json" \ + || return 1 + done 3< "$RETIRE_SCRATCH/scanned" < /dev/null || return 1 + local kept + kept=$(jq length "$RETIRE_SCRATCH/run.json") || return 1 + if [ "$kept" != "$total" ]; then + echo "ERROR: preserved ${kept} of the ${total} locations to clean" \ + "up." >&2 + return 1 + fi + # "s " per source, for people. + jq -r '.[] | "\(.id) \(.kind) \(.path | tojson)"' \ + "$RETIRE_SCRATCH/run.json" > "$RETIRE_SCRATCH/map" || return 1 + sources_blob=$(swarm_recovery_git -C "$REPO_ROOT" hash-object -w \ + --stdin < "$RETIRE_SCRATCH/map") || return 1 + sources_ref="${base}/sources" + swarm_recovery_git -C "$REPO_ROOT" update-ref "$sources_ref" \ + "$sources_blob" "" || return 1 + swarm_recovery_verify_checkout "$REPO_ROOT" || return 1 + + # The checkpoint, before the first deletion. + swarm_state_update "$STATE_FILE" ' + .cleanup = ((.cleanup // {engagement: $e, started_at: $at, + runs: []}) + | .runs += [{snapshot: $s, sources_ref: $ref, + sources_oid: $oid, sources: $src}])' \ + --arg e "$engagement" \ + --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg s "$snap" --arg ref "$sources_ref" --arg oid "$sources_blob" \ + --argjson src "$(cat "$RETIRE_SCRATCH/run.json")" || return 1 + + # Delete, volumes first and the bare repo last, each step checked + # again right before it acts. + local order id rest + for order in volume mirror new removing bare; do + while IFS="$(printf '\t')" read -r -u 3 n kind path mode; do + [ "$kind" = "$order" ] || continue + id="s$n" + prefix="${base}/${id}" + # Its one record in the checkpoint must show it preserved: + # an inventory that still holds, or a scan that found it + # empty. + rest=$(swarm_state_read "$STATE_FILE" | jq -r --arg id "$id" \ + --arg k "$kind" --arg p "$path" ' + [.cleanup.runs[-1].sources[] + | select(.id == $id and .kind == $k and .path == $p)] + | if length != 1 then "none" + elif .[0].empty == true then "empty" + elif (.[0].inventory | type) == "string" + then "inventory " + .[0].inventory + else "none" end') || rest=none + case "$rest" in + empty) ;; + "inventory "*) + inv=$(cleanup_inventory "$prefix") || inv="" + if [ "$inv" != "${rest#inventory }" ]; then + echo "ERROR: the recovery refs of ${path} changed" \ + "before it was deleted." >&2 + return 1 + fi ;; + *) + echo "ERROR: the checkpoint has no record that ${path}" \ + "was preserved; not deleting it." >&2 + return 1 ;; + esac + case "$kind" in + volume) + cleanup_volume_free "$path" || return 1 + mp=$(cleanup_volume_mountpoint "$path") || { + echo "ERROR: cannot inspect volume ${path}." >&2 + return 1 + } + if [ -n "$mp" ]; then + cleanup_no_mounts "$mp" at || return 1 + fi + cleanup_record "$id" '.state = "deleting"' || return 1 + if ! docker volume rm "$path" >/dev/null 2>&1; then + echo "ERROR: cannot remove volume ${path}." >&2 + return 1 + fi + cleanup_record "$id" '.state = "deleted"' || return 1 ;; + bare) + rc=0 + swarm_preservation_check "$BARE_REPO" "$REPO_ROOT" \ + > "$RETIRE_SCRATCH/e3" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "ERROR: ${BARE_REPO} changed during cleanup." >&2 + return 1 + fi + cleanup_delete_host "$id" "$path" || return 1 ;; + *) + cleanup_delete_host "$id" "$path" || return 1 ;; + esac + echo " deleted ${path}" + done 3< "$RETIRE_SCRATCH/scanned" < /dev/null || return 1 + done + if [ -d "$MIRROR_DIR" ] && ! rmdir "$MIRROR_DIR" 2>/dev/null; then + echo "ERROR: ${MIRROR_DIR} is not empty after cleanup." >&2 + return 1 + fi + # The engagement's records go only once nothing they stand for is + # left, whatever this run did or did not list. + cleanup_sources "$RETIRE_SCRATCH/left" || return 1 + if [ -s "$RETIRE_SCRATCH/left" ]; then + echo "ERROR: cleanup left these in place:" >&2 + cut -f2 "$RETIRE_SCRATCH/left" | sed 's/^/ /' >&2 + echo " Run ./launch.sh cleanup again." >&2 + return 1 + fi + + # The engagement's records go last; the claim of the runtime + # directory and its lock stay. + local f + for f in "$RUNTIME_DIR"/interactive-*.pending; do + [ -e "$f" ] || [ -L "$f" ] || continue + rm -f "$f" || return 1 + done + swarm_state_update "$STATE_FILE" '{schema: .schema, project: .project, + repo_root: .repo_root}' || return 1 + echo "Cleaned up engagement ${engagement}." + echo "Recovery refs: ${base}/ (what each s was: git cat-file" \ + "blob ${sources_ref})." +} + +cmd_cleanup() { + local rc=0 + RETIRE_SCRATCH="" + # shellcheck disable=SC2034 # read by retire_cleanup (lib/retire.sh) + RETIRE_SNAP_REF="" + trap retire_cleanup EXIT + trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 129' HUP + trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 130' INT + trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 143' TERM + if RETIRE_SCRATCH=$(retire_tmpdir); then + cleanup_run "$@" || rc=$? + else + rc=1 + fi + retire_cleanup + trap - EXIT HUP INT TERM + return "$rc" +} diff --git a/lib/preserve-scan.sh b/lib/preserve-scan.sh index 5a3984d..51fe537 100644 --- a/lib/preserve-scan.sh +++ b/lib/preserve-scan.sh @@ -8,11 +8,16 @@ # Usage: preserve-scan.sh worktree ROOT CONTEXT OUT # preserve-scan.sh gitdir ROOT - OUT # preserve-scan.sh userconfig - - OUT +# preserve-scan.sh files ROOT FORMAT OUT # # worktree: ROOT is a checkout (a workspace) whose repository is # ROOT/.git; CONTEXT (full, slim or none) says which deletions under # .claude the engine made itself. # gitdir: ROOT is a repository directory (a bare repo or a mirror). +# files: ROOT is what is left of one that a deletion got part of the way +# through: every file is stored as a blob, those under objects/ +# whatever their size; FORMAT (sha1 or sha256) is the object format of +# the repository the bundle is for. # userconfig: run as a container's user, with its HOME and # XDG_CONFIG_HOME, in a snapshot of that container: reports the global # and system git config settings that keep repository data outside a @@ -69,11 +74,28 @@ fail() { } case "$MODE" in - worktree|gitdir|userconfig) ;; + worktree|gitdir|userconfig|files) ;; *) fail "unknown mode: ${MODE}" ;; esac [ "$MODE" = userconfig ] || [ -d "$ROOT" ] || fail "not a directory: ${ROOT}" [ -d "$OUT" ] || fail "not a directory: ${OUT}" + +# The shell's read ends a loop at a read error just as at the end of +# its list, so a scan could miss what the rest of a list holds. This +# read stops the scan instead; an error is all the builtin writes to +# standard error here. (In canon_nm, which runs in a subshell, the +# failure reaches its callers as a path that does not resolve, which +# they report as a block.) +# shellcheck disable=SC2329,SC2162 # called by every loop below, and +# every caller passes its own options (-r among them) +read() { + local __rc=0 + builtin read "$@" 2> "$OUT/.read-err" || __rc=$? + if [ -s "$OUT/.read-err" ]; then + fail "cannot read a list: $(cat "$OUT/.read-err")" + fi + return "$__rc" +} # Print the resolved form of directory $1. Command substitution would # drop trailing newlines and turn "repo" into "repo", so the x keeps # them; a path with a newline anywhere fails (the scan's lists are @@ -236,6 +258,118 @@ canon_nm() { canon "$1" } +# ---- Files and bundle (gitdir, worktree and files modes) ---------------- + +# Write to OUT/.fkeep, NUL-separated, every path under directory TOP +# that is not a directory, except those listed in file EXEMPT (one per +# line). Paths are read NUL-separated, so any byte in a name survives; +# a path with a newline in it is never exempt (exempt names are hex). +list_files() { + local top="$1" exempt="$2" f + : > "$OUT/.flist" + : > "$OUT/.fnl" + find "$top" ! -type d -print0 2>"$ERR" > "$OUT/.flist0" \ + || fail "cannot list ${top}" + [ ! -s "$ERR" ] || fail "cannot list ${top}: $(cat "$ERR")" + while IFS= read -r -d '' f; do + case "$f" in + *$'\n'*) printf '%s\0' "$f" >> "$OUT/.fnl" ;; + *) printf '%s\n' "$f" >> "$OUT/.flist" ;; + esac + done < "$OUT/.flist0" + sort "$OUT/.flist" -o "$OUT/.flist" + comm -23 "$OUT/.flist" "$exempt" | tr '\n' '\0' > "$OUT/.fkeep" + cat "$OUT/.fnl" >> "$OUT/.fkeep" +} + +# Store every path of OUT/.fkeep (under TOP) in SCRATCH: a symlink as +# its target string, a regular file as its bytes, each as a blob +# referenced by refs/v/0/f/, with the list " " +# (NUL-terminated, paths relative to TOP) as refs/v/0/files. A file +# over the size limits is reported as uncommitted instead, unless its +# path relative to TOP starts with ALWAYS/ (files mode keeps objects/ +# whatever its size); such files do not count toward the total. Any +# other kind of file fails the scan. +store_files() { + local top="$1" always="$2" f rel target blob size mode files keep + local k=0 total=0 + : > "$OUT/.files" + : > "$OUT/.fupd" + while IFS= read -r -d '' f; do + rel="${f#"$top"/}" + if [ -L "$f" ]; then + # readlink -n prints the target as is; the x keeps a + # trailing newline of the target from command substitution. + target=$(readlink -n "$f" && printf x) \ + || fail "cannot read ${f}" + target="${target%x}" + k=$((k + 1)) + blob=$(printf '%s' "$target" \ + | git --git-dir="$SCRATCH" hash-object -w --stdin) \ + || fail "cannot store ${f}" + printf 'update refs/v/0/f/%s %s\n' "$k" "$blob" >> "$OUT/.fupd" + printf '%s 120000 0 %s\0' "$k" "$rel" >> "$OUT/.files" + continue + fi + [ -f "$f" ] || fail "special file: ${f}" + size=$(fsize "$f") || fail "cannot read ${f}" + keep=no + if [ -n "$always" ]; then + case "$rel" in + "$always"/*) keep=yes ;; + esac + fi + if [ "$keep" = no ]; then + if [ "$size" -gt "$FILE_LIMIT" ] \ + || [ $((total + size)) -gt "$TOTAL_LIMIT" ]; then + report "uncommitted large-file ${f#"$ROOT"/} ${size}" + continue + fi + total=$((total + size)) + fi + k=$((k + 1)) + blob=$(git --git-dir="$SCRATCH" hash-object -w --no-filters --stdin \ + < "$f") || fail "cannot store ${f}" + mode=100644 + [ ! -x "$f" ] || mode=100755 + printf 'update refs/v/0/f/%s %s\n' "$k" "$blob" >> "$OUT/.fupd" + printf '%s %s %s %s\0' "$k" "$mode" "$size" "$rel" >> "$OUT/.files" + done < "$OUT/.fkeep" + git --git-dir="$SCRATCH" update-ref --stdin < "$OUT/.fupd" 2>"$ERR" \ + || fail "cannot record the files of ${top}: $(cat "$ERR")" + files=$(git --git-dir="$SCRATCH" hash-object -w --stdin \ + < "$OUT/.files") || fail "cannot write the file list of ${top}" + git --git-dir="$SCRATCH" update-ref refs/v/0/files "$files" +} + +make_bundle() { + git --git-dir="$SCRATCH" -c safe.directory='*' bundle create \ + "$OUT/bundle" --all >/dev/null 2>"$ERR" \ + || fail "cannot create the bundle: $(cat "$ERR")" +} + +# ---- files mode ----------------------------------------------------------- + +# A remnant that is no longer a git dir (a deletion got part of the +# way): every file under ROOT is stored, with no history to inventory. +# Files under objects/ are stored whatever their size: they may hold +# history no earlier snapshot has. CONTEXT is the object format of the +# repository the bundle goes into (sha1 or sha256). +if [ "$MODE" = files ]; then + case "$CONTEXT" in + sha1|sha256) ;; + *) fail "files mode needs the object format, sha1 or sha256" ;; + esac + git init -q --bare --object-format="$CONTEXT" "$SCRATCH" \ + || fail "cannot create ${SCRATCH}" + : > "$OUT/objects" + : > "$OUT/.exempt" + list_files "$ROOT" "$OUT/.exempt" + store_files "$ROOT" objects + make_bundle + exit 0 +fi + # ---- userconfig mode ---------------------------------------------------- if [ "$MODE" = userconfig ]; then @@ -1010,66 +1144,10 @@ while IFS= read -r d; do done < "$GITDIRS" sort -u "$EXEMPT" -o "$EXEMPT" -# Paths are read NUL-separated, so any byte in a name survives. A path -# with a newline in it is never exempt (exempt names are hex). -: > "$OUT/.flist" -: > "$OUT/.fnl" -find "$TOP" ! -type d -print0 2>"$ERR" > "$OUT/.flist0" \ - || fail "cannot list ${TOP}" -[ ! -s "$ERR" ] || fail "cannot list ${TOP}: $(cat "$ERR")" -while IFS= read -r -d '' f; do - case "$f" in - *$'\n'*) printf '%s\0' "$f" >> "$OUT/.fnl" ;; - *) printf '%s\n' "$f" >> "$OUT/.flist" ;; - esac -done < "$OUT/.flist0" -sort "$OUT/.flist" -o "$OUT/.flist" -comm -23 "$OUT/.flist" "$EXEMPT" | tr '\n' '\0' > "$OUT/.fkeep" -cat "$OUT/.fnl" >> "$OUT/.fkeep" -: > "$OUT/.files" -: > "$OUT/.fupd" -k=0 -total=0 -while IFS= read -r -d '' f; do - rel="${f#"$TOP"/}" - if [ -L "$f" ]; then - # readlink -n prints the target as is; the x keeps a trailing - # newline of the target from command substitution. - target=$(readlink -n "$f" && printf x) || fail "cannot read ${f}" - target="${target%x}" - k=$((k + 1)) - blob=$(printf '%s' "$target" \ - | git --git-dir="$SCRATCH" hash-object -w --stdin) \ - || fail "cannot store ${f}" - printf 'update refs/v/0/f/%s %s\n' "$k" "$blob" >> "$OUT/.fupd" - printf '%s 120000 0 %s\0' "$k" "$rel" >> "$OUT/.files" - continue - fi - [ -f "$f" ] || fail "special file in a git dir: ${f}" - size=$(fsize "$f") || fail "cannot read ${f}" - if [ "$size" -gt "$FILE_LIMIT" ] \ - || [ $((total + size)) -gt "$TOTAL_LIMIT" ]; then - report "uncommitted large-file ${f#"$ROOT"/} ${size}" - continue - fi - total=$((total + size)) - k=$((k + 1)) - blob=$(git --git-dir="$SCRATCH" hash-object -w --no-filters --stdin \ - < "$f") || fail "cannot store ${f}" - mode=100644 - [ ! -x "$f" ] || mode=100755 - printf 'update refs/v/0/f/%s %s\n' "$k" "$blob" >> "$OUT/.fupd" - printf '%s %s %s %s\0' "$k" "$mode" "$size" "$rel" >> "$OUT/.files" -done < "$OUT/.fkeep" -git --git-dir="$SCRATCH" update-ref --stdin < "$OUT/.fupd" 2>"$ERR" \ - || fail "cannot record the files of ${TOP}: $(cat "$ERR")" -files=$(git --git-dir="$SCRATCH" hash-object -w --stdin < "$OUT/.files") \ - || fail "cannot write the file list of ${TOP}" -git --git-dir="$SCRATCH" update-ref refs/v/0/files "$files" +list_files "$TOP" "$EXEMPT" +store_files "$TOP" "" # ---- Bundle ------------------------------------------------------------- -git --git-dir="$SCRATCH" -c safe.directory='*' bundle create \ - "$OUT/bundle" --all >/dev/null 2>"$ERR" \ - || fail "cannot create the bundle: $(cat "$ERR")" +make_bundle exit 0 diff --git a/lib/preserve.sh b/lib/preserve.sh index 9b7fa27..d437f51 100644 --- a/lib/preserve.sh +++ b/lib/preserve.sh @@ -194,12 +194,37 @@ swarm_dev_ino() { stat "$SWARM_STAT_FORM" '%d:%i' "$1" } +# Print absolute PATH with the longest leading part of it that can be +# entered resolved (cd -P), and the rest as it is: PATH need not exist +# on this host, but a link on the way to it is still followed. Fails +# on a relative path. +swarm_path_resolve() { + local p="$1" head rest="" resolved + case "$p" in + /*) ;; + *) return 1 ;; + esac + while [ "$p" != / ] && [ "${p%/}" != "$p" ]; do + p="${p%/}" + done + head="$p" + while ! resolved=$(cd -P "$head" 2>/dev/null && pwd -P); do + rest="/${head##*/}${rest}" + head="${head%/*}" + [ -n "$head" ] || head=/ + done + [ "$resolved" != / ] || resolved="" + [ -n "${resolved}${rest}" ] || rest=/ + printf '%s%s\n' "$resolved" "$rest" +} + # Succeed when the checkout at ROOT keeps every object in its own # store, so objects imported there survive the deletion of TARGETS # (directories cleanup is about to delete; may be empty). Refuses a # shallow or partial clone, alternates, symlinks in the object store, # an object store inside RUNTIME or a target, and a target that shares -# a directory or pack file with the object store (bind mounts). +# a directory with the git dir or the checkout's own git dir, or a pack +# file with the object store (bind mounts). swarm_checkout_storage_ok() { local root="$1" runtime="$2" shift 2 @@ -255,34 +280,57 @@ swarm_checkout_storage_ok() { "first." >&2 return 1 fi - canon=$(cd -P "$objects" 2>/dev/null && pwd -P) || { - echo "ERROR: cannot resolve ${objects}." >&2 + local ccanon gcanon x + if ! canon=$(cd -P "$objects" 2>/dev/null && pwd -P) \ + || ! ccanon=$(cd -P "$common" 2>/dev/null && pwd -P); then + echo "ERROR: cannot resolve ${common}." >&2 return 1 - } + fi + # The checkout's own git dir as well (a linked worktree's lives apart + # from the common one). + if ! gcanon=$(swarm_recovery_git -C "$root" rev-parse \ + --absolute-git-dir 2>/dev/null) \ + || ! gcanon=$(cd -P "$gcanon" 2>/dev/null && pwd -P); then + echo "ERROR: cannot resolve the git dir of ${root}." >&2 + return 1 + fi + # Every target, whether or not this host has it (a volume's + # Mountpoint can be a path in Docker's own machine), may not be + # the git dir or its object store, nor lie inside or around either. for t in "$runtime" "$@"; do - [ -e "$t" ] || continue - tcanon=$(cd -P "$t" 2>/dev/null && pwd -P) || { + tcanon=$(swarm_path_resolve "$t") || { echo "ERROR: cannot resolve ${t}." >&2 return 1 } - case "$canon/" in - "$tcanon"/*) - echo "ERROR: the object store of ${root} is inside ${t}." >&2 - return 1 ;; - esac - case "$tcanon/" in - "$canon"/*) - echo "ERROR: ${t} is inside the object store of ${root}." >&2 - return 1 ;; - esac + for x in "$ccanon" "$canon" "$gcanon"; do + case "$x/" in + "$tcanon"/*) + echo "ERROR: the git dir of ${root} is at or inside" \ + "${t}." >&2 + return 1 ;; + esac + case "$tcanon/" in + "$x"/*) + echo "ERROR: ${t} is inside the git dir of ${root}." >&2 + return 1 ;; + esac + done done [ "$#" -gt 0 ] || return 0 # The same directory or pack file seen through a bind mount has the - # same device and inode under both paths. + # same device and inode under both paths. Every directory of the + # checkout's own git dir counts (it is the common one, or a linked + # worktree's, which lives apart); of the common one, those of the + # object store and the refs. local tmp p rc=0 tmp=$(mktemp -d) || return 1 { - find "$objects" "$common/refs" -type d -print || rc=1 + if [ "$gcanon" = "$ccanon" ]; then + find "$gcanon" -type d -print || rc=1 + else + find "$objects" "$common/refs" "$gcanon" -type d -print \ + || rc=1 + fi for p in "$objects/pack" "$objects/info"; do [ ! -d "$p" ] || find "$p" -type f -print || rc=1 done @@ -303,14 +351,14 @@ swarm_checkout_storage_ok() { done if [ "$rc" -ne 0 ]; then rm -rf "$tmp" - echo "ERROR: cannot list the object store of ${root} or what" \ + echo "ERROR: cannot list the git dir of ${root} or what" \ "cleanup deletes." >&2 return 1 fi comm -12 "$tmp/mine.ids" "$tmp/theirs.ids" > "$tmp/both" || rc=1 if [ "$rc" -ne 0 ] || [ -s "$tmp/both" ]; then rm -rf "$tmp" - echo "ERROR: the object store of ${root} shares files with what" \ + echo "ERROR: the git dir of ${root} shares files with what" \ "cleanup deletes (a bind mount?)." >&2 return 1 fi @@ -454,7 +502,7 @@ swarm_docker_own_group() { } # More docker run arguments for swarm_scan_volume (retire's helper -# label); empty by default. +# label; cleanup's name and label); empty by default. SWARM_SCAN_DOCKER_ARGS=() # Scan workspace volume VOL with image IMAGE (context CONTEXT) into the diff --git a/lib/project.sh b/lib/project.sh index e00ca9d..11c8957 100644 --- a/lib/project.sh +++ b/lib/project.sh @@ -345,6 +345,7 @@ swarm_state_update() { # another checkout. Callers run swarm_runtime_check first. swarm_runtime_owner_check() { local dir="$1" root="$2" project="$3" file canonical state owner + local recorded file="$dir/state.json" if [ ! -e "$file" ] && [ ! -L "$file" ]; then return 0 @@ -363,9 +364,20 @@ swarm_runtime_owner_check() { echo " owner: ${owner}" >&2 echo " this one: ${canonical}" >&2 echo "Give this checkout its own id with SWARM_PROJECT. If it" \ - "is the same checkout at a new path, harvest from the old" \ - "path first, or remove the runtime directory once nothing" \ - "in it is needed." >&2 + "is the same checkout at a new path, run harvest.sh and" \ + "./launch.sh cleanup from the old path." >&2 + return 1 + fi + # The runtime directory can be named apart from the project id + # (SWARM_RUNTIME_DIR), and cleanup finds containers and volumes by + # the project: a runtime of another project is not this one's. + recorded=$(printf '%s' "$state" | jq -r '.project // empty') \ + || return 1 + if [ -n "$recorded" ] && [ "$recorded" != "$project" ]; then + echo "ERROR: ${dir} belongs to project id '${recorded}', not" \ + "'${project}'." >&2 + echo "Use that project's SWARM_PROJECT with it, or another" \ + "SWARM_RUNTIME_DIR." >&2 return 1 fi } @@ -625,6 +637,24 @@ swarm_engagement_recorded() { printf '%s' "$state" | jq -r '.engagement_id // empty' } +# Fail when state file FILE records a cleanup that has not finished: +# the engagement is half deleted, and only cleanup may go on. +swarm_cleanup_pending_check() { + local file="$1" state pending + if [ ! -e "$file" ] && [ ! -L "$file" ]; then + return 0 + fi + if ! state=$(swarm_state_read "$file") \ + || ! pending=$(printf '%s' "$state" | jq -r 'has("cleanup")'); then + echo "ERROR: cannot read ${file}." >&2 + return 1 + fi + if [ "$pending" = true ]; then + echo "ERROR: cleanup in progress: run ./launch.sh cleanup again." >&2 + return 1 + fi +} + # A later phase joins the recorded engagement. Fail when the caller # names a different one, or names one while none is recorded. swarm_engagement_join() { diff --git a/lib/retire.sh b/lib/retire.sh index 6e5605e..7597a5d 100644 --- a/lib/retire.sh +++ b/lib/retire.sh @@ -750,8 +750,9 @@ Usage: $0 retire [--post-process] Remove this checkout's stopped engine containers (or only the post-process container) after checking that nothing lives only in -them. Their workspace volumes stay. Run costs.sh first if you need -the containers' stats. +them. Their workspace volumes stay (./launch.sh cleanup deletes them +after its own checks). Run costs.sh first if you need the containers' +stats. HELP return 0 ;; *) echo "ERROR: unknown retire option: $1" >&2; return 1 ;; @@ -762,6 +763,9 @@ HELP return 1 fi swarm_lock "$RUNTIME_DIR" || return 1 + # Checked before the lock too, but another checkout of this project + # id could have claimed the runtime and started in between. + swarm_runtime_verify "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || return 1 swarm_engagement_join "$STATE_FILE" || return 1 retire_require_daemon || return 1 retire_sweep_snapshots || return 1 @@ -826,7 +830,8 @@ EOF failed=1 fi done 3< "$checked" < /dev/null - echo "Workspace volumes stay, with the files of the removed containers." + echo "Workspace volumes stay, with the files of the removed containers;" \ + "./launch.sh cleanup deletes them after its checks." return "$failed" } diff --git a/tests/runtime_cleanup.sh b/tests/runtime_cleanup.sh new file mode 100755 index 0000000..09996b8 --- /dev/null +++ b/tests/runtime_cleanup.sh @@ -0,0 +1,218 @@ +#!/bin/bash +set -uo pipefail + +# shellcheck source=_test_env.sh +source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" + +# tests/runtime_cleanup.sh +# +# launch.sh cleanup under real Docker: a bare repo a container wrote +# into as its own user (on Linux another UID than this host user's, so +# only the Docker helper can delete those files), a mirror, and a real +# workspace volume of this checkout. Cleanup refuses while a container +# of the project remains (before workspace volumes, retire refuses +# every container), then copies all three into the checkout as recovery +# refs and deletes them. +# +# Requires Docker. No API key needed. + +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$TESTS_DIR/.." && pwd)" +LAUNCH_SH="${REPO_ROOT}/launch.sh" + +if ! command -v docker >/dev/null 2>&1; then + echo "SKIP: docker not found" + exit 0 +fi +if ! docker info >/dev/null 2>&1; then + echo "SKIP: docker daemon not running" + exit 0 +fi +if ! git var GIT_CONFIG_SYSTEM >/dev/null 2>&1; then + echo "SKIP: needs Git 2.42 or later (git var)" + exit 0 +fi + +# shellcheck source=../lib/project.sh +source "$REPO_ROOT/lib/project.sh" + +BASE_IMAGE="swarm-core-runtime-test" +P="swarm-core-cleanup-test" +# Cleanup's helpers run the engine image, -agent. +ENGINE_IMAGE="${P}-agent" +NAME="${P}-agent-1" +VOL="${NAME}-ws-00000000" +WORKDIR=$(mktemp -d /tmp/swarm-core-cleanup-XXXXXX) + +drop_all() { + local c + for c in $(docker ps -aq --filter "label=${SWARM_PROJECT_LABEL}=${P}"); do + docker rm -f "$c" >/dev/null 2>&1 || true + done + for c in $(docker ps -aq --filter "label=swarm-core.cleanup=${P}"); do + docker rm -f "$c" >/dev/null 2>&1 || true + done + docker volume rm "$VOL" >/dev/null 2>&1 || true +} +cleanup_test() { + drop_all + # Files the container wrote may belong to its user: remove them as + # root with the test image, which needs no pull. + docker run --rm --user 0 --entrypoint rm \ + -v "$WORKDIR:/workdir" "$BASE_IMAGE" -rf /workdir/runtime \ + >/dev/null 2>&1 || true + docker rmi "$ENGINE_IMAGE" >/dev/null 2>&1 || true + rm -rf "$WORKDIR" +} +trap cleanup_test EXIT + +PASS=0 +FAIL=0 +check() { + local label="$1" actual="$2" expected="$3" + if [ "$actual" = "$expected" ]; then + echo " PASS: ${label}" + PASS=$((PASS + 1)) + else + echo " FAIL: ${label} expected=${expected} got=${actual}" + FAIL=$((FAIL + 1)) + fi +} +present() { + if [ -e "$1" ] || [ -L "$1" ]; then echo yes; else echo no; fi +} +gitc() { + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false "$@" +} +run_cleanup() { + (cd "$CHECKOUT" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + "$BASH" "$LAUNCH_SH" cleanup) +} + +drop_all +echo "--- Building ${BASE_IMAGE} (fake driver only) ---" +docker build --quiet --tag "$BASE_IMAGE" \ + --build-arg SWARM_AGENTS=fake "$REPO_ROOT" >/dev/null || exit 1 +docker tag "$BASE_IMAGE" "$ENGINE_IMAGE" || exit 1 + +# The checkout, its runtime directory and engagement on this daemon. +CHECKOUT="$WORKDIR/checkout" +git init -q "$CHECKOUT" +echo a > "$CHECKOUT/a" +gitc -C "$CHECKOUT" add a +gitc -C "$CHECKOUT" commit -qm init +CK=$(swarm_checkout_path "$CHECKOUT") +RT="$WORKDIR/runtime" +mkdir -m 700 "$RT" +swarm_runtime_claim "$RT" "$CHECKOUT" "$P" || exit 1 +DAEMON=$(swarm_docker_daemon_id) || exit 1 +swarm_state_update "$RT/state.json" \ + '.engagement_id = "e1" | .docker_daemon = $d' --arg d "$DAEMON" || exit 1 + +# The bare repo, shared as start shares it, and a commit a container +# pushes into it as its own user. +git clone -q --bare --no-hardlinks "$CHECKOUT" "$RT/upstream.git" +git -C "$RT/upstream.git" update-ref refs/heads/agent-work \ + "$(git -C "$CHECKOUT" rev-parse HEAD)" +git -C "$RT/upstream.git" config core.sharedRepository world +chmod -R a+rwX "$RT/upstream.git" +if ! docker run --rm -v "$RT/upstream.git:/upstream:rw" \ + --entrypoint bash "$BASE_IMAGE" -c ' + set -e + git clone -q /upstream /tmp/w + cd /tmp/w + git checkout -q agent-work + echo by-agent > agent.txt + git add agent.txt + git -c user.name=agent -c user.email=a@a commit -qm by-agent + git push -q origin agent-work + # A directory only its owner may change: on Linux, when this + # host user is someone else, only the Docker helper can + # delete what is in it. + mkdir -m 755 /upstream/agent-dir + echo notes > /upstream/agent-dir/notes + ' > "$WORKDIR/push.out" 2>&1; then + echo "FAIL: the container could not push"; tail -5 "$WORKDIR/push.out" + exit 1 +fi +AGENT_COMMIT=$(git -C "$RT/upstream.git" rev-parse agent-work) +# Harvest: the checkout has the agent's work. +git -C "$CHECKOUT" fetch -q "$RT/upstream.git" agent-work +gitc -C "$CHECKOUT" merge -q --ff-only FETCH_HEAD +# A mirror. +mkdir -p "$RT/mirrors" +git init -q --bare "$RT/mirrors/sub.git" +gitc -C "$CHECKOUT" push -q "$RT/mirrors/sub.git" HEAD:refs/heads/main +# A workspace volume of this checkout with a committed repository. +docker volume create \ + --label "${SWARM_PROJECT_LABEL}=${P}" \ + --label "${SWARM_CHECKOUT_LABEL}=${CK}" \ + --label "${SWARM_CONTAINER_LABEL}=${NAME}" \ + --label "${SWARM_KIND_LABEL}=agent" \ + --label "${SWARM_CONTEXT_LABEL}=full" \ + --label "${SWARM_ENGAGEMENT_LABEL}=e1" "$VOL" >/dev/null || exit 1 +docker run --rm --mount "type=volume,src=${VOL},dst=/workspace" \ + --entrypoint bash "$BASE_IMAGE" -c ' + set -e + cd /workspace + git init -q + echo w > w + git add w + git -c user.name=t -c user.email=t@t commit -qm volume-work + ' >/dev/null 2>&1 || { echo "FAIL: cannot fill the volume"; exit 1; } + +echo "--- a stopped container of the project ---" +docker run --name "$NAME" \ + --label "${SWARM_PROJECT_LABEL}=${P}" \ + --label "${SWARM_CHECKOUT_LABEL}=${CK}" \ + --entrypoint true "$BASE_IMAGE" >/dev/null 2>&1 +rc=0 +out=$(run_cleanup 2>&1) || rc=$? +check "cleanup refuses while it remains" "$rc" "1" +check "because retire refuses it" \ + "$(printf '%s\n' "$out" | grep -c 'every container must be retired' \ + | awk '{ print ($1 > 0) }')" "1" +check "the bare repo stays" "$(present "$RT/upstream.git")" "yes" +check "the volume stays" \ + "$(docker volume inspect "$VOL" >/dev/null 2>&1 && echo yes || echo no)" \ + "yes" +docker rm "$NAME" >/dev/null 2>&1 + +# Whether this user could delete what the container wrote (on macOS +# Docker maps the container's user to this one; on Linux it may not). +if [ -w "$RT/upstream.git/agent-dir" ]; then + echo "--- this user may delete the container's files itself ---" +else + echo "--- only the Docker helper can delete the container's files ---" +fi + +echo "--- cleanup ---" +rc=0 +out=$(run_cleanup 2>&1) || rc=$? +check "cleanup succeeds" "$rc" "0" +[ "$rc" -eq 0 ] || printf '%s\n' "$out" | tail -8 | sed 's/^/ /' +check "the bare repo is gone" "$(present "$RT/upstream.git")" "no" +check "no remnant is left" \ + "$(find "$RT" -name '*.removing' | grep -c . || true)" "0" +check "the mirrors are gone" "$(present "$RT/mirrors")" "no" +check "the volume is gone" \ + "$(docker volume inspect "$VOL" >/dev/null 2>&1 && echo yes || echo no)" \ + "no" +check "the lock and the runtime directory stay" \ + "$(present "$RT/engagement.lock")$(present "$RT")" "yesyes" +check "the state keeps only the claim" \ + "$(jq -c 'keys' "$RT/state.json")" '["project","repo_root","schema"]' +check "the agent's commit is in the recovery refs" \ + "$(git -C "$CHECKOUT" for-each-ref --contains "$AGENT_COMMIT" \ + --format='%(refname)' refs/swarm-recovery/ | grep -c . \ + | awk '{ print ($1 > 0) }')" "1" +check "the volume's work is in the recovery refs" \ + "$(git -C "$CHECKOUT" log --all --format=%s | grep -c '^volume-work$' \ + | awk '{ print ($1 > 0) }')" "1" +check "no helper container is left" \ + "$(docker ps -aq --filter "label=swarm-core.cleanup=${P}" | grep -c . \ + || true)" "0" + +echo "" +echo " ${PASS} passed, ${FAIL} failed" +[ "$FAIL" -eq 0 ] diff --git a/tests/test.sh b/tests/test.sh index 85f67ec..f228e6b 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -18,8 +18,8 @@ fi # # Options: # --unit Run unit tests only (no Docker or API key needed). -# --runtime Runtime tests: emergency push and retire (Docker -# only, no API key). +# --runtime Runtime tests: emergency push, retire and cleanup +# (Docker only, no API key). # --all Unit + runtime, then full integration matrix. # --oauth Integration test using OAuth token (needs Docker + # CLAUDE_CODE_OAUTH_TOKEN). @@ -74,8 +74,9 @@ run_all_tests() { # actually fires; this phase does. Skips cleanly when Docker # is unavailable so --all stays useful on hosts without it. # Also runs the real Pi CLI against a local test API, without keys, - # and launch.sh retire on real containers, whose snapshots and - # docker diff test_retire.sh only imitates. + # launch.sh retire on real containers, whose snapshots and docker + # diff test_retire.sh only imitates, and launch.sh cleanup on files + # a container wrote as its own user. echo "=== Phase 1.5: Docker runtime tests ===" echo "" if ! command -v docker >/dev/null 2>&1; then @@ -87,7 +88,7 @@ run_all_tests() { else local rt_test rt_start rt_elapsed rt_rc for rt_test in runtime_signal_trap.sh runtime_pi.sh \ - runtime_retire.sh; do + runtime_retire.sh runtime_cleanup.sh; do rt_rc=0 rt_start=$(date +%s) "$TESTS_DIR/$rt_test" || rt_rc=$? @@ -402,7 +403,8 @@ Options: --runtime Runtime tests (need Docker, no API key): drives SIGTERM / SIGINT / fatal-exit paths under a real container with a synthetic test-committer driver, - then launch.sh retire on real stopped containers. + then launch.sh retire on real stopped containers and + launch.sh cleanup on what containers wrote. --all Unit + runtime, then full integration matrix. --oauth Integration test using CLAUDE_CODE_OAUTH_TOKEN (needs Docker + OAuth token). @@ -461,6 +463,7 @@ if $RUN_RUNTIME; then rt_rc=0 "$TESTS_DIR/runtime_signal_trap.sh" || rt_rc=1 "$TESTS_DIR/runtime_retire.sh" || rt_rc=1 + "$TESTS_DIR/runtime_cleanup.sh" || rt_rc=1 exit "$rt_rc" fi diff --git a/tests/test_cleanup.sh b/tests/test_cleanup.sh new file mode 100755 index 0000000..427483f --- /dev/null +++ b/tests/test_cleanup.sh @@ -0,0 +1,934 @@ +#!/bin/bash +set -euo pipefail + +# shellcheck source=_test_env.sh +source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" + +# Unit tests for launch.sh cleanup, with a fake docker CLI whose answers +# come from files, and real Git repositories for the checkout, the bare +# repo and the mirrors: the preservation check, the scans, the import of +# recovery refs and git fsck all run for real. No Docker or API key +# required. + +PASS=0 +FAIL=0 +TMPDIR=$(mktemp -d) +# Exported, so that the fake docker and the programs it runs make their +# scratch files here, under the cleanup below. +export TMPDIR +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +LAUNCH="$REPO_ROOT/launch.sh" +trap 'chmod -R u+rwX "$TMPDIR" 2>/dev/null; rm -rf "$TMPDIR"' EXIT + +# The scanner asks git which config files it reads (git var, Git 2.42 +# or later); here it runs on this host's git. +if ! git var GIT_CONFIG_SYSTEM >/dev/null 2>&1; then + echo "SKIP: these tests need Git 2.42 or later (git var)" + exit 0 +fi + +# shellcheck source=../lib/project.sh +source "$REPO_ROOT/lib/project.sh" + +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + echo " PASS: ${label}" + PASS=$((PASS + 1)) + else + echo " FAIL: ${label}" + echo " expected: ${expected}" + echo " actual: ${actual}" + FAIL=$((FAIL + 1)) + fi +} + +gitc() { + git -c user.name=t -c user.email=t@t -c commit.gpgsign=false "$@" +} + +P=proj +_ct=$(cd -P "$TMPDIR" && pwd -P) + +# Fake docker. $F holds the answers: +# containers " " lines of the project's containers +# inspect/ their docker container inspect JSON +# volumes the project's volume names, one per line +# vollabels/ a volume's labels (JSON) +# voldir/ a volume's files (scanned by the real scanner) +# volusers/ ids of containers using a volume +# helpers ids of cleanup helpers an earlier run left +# remote-daemon the daemon sees another file system (bind mounts +# show an empty directory) +# daemon docker info id (default d1) +# helper-mountinfo the mount table a deletion helper sees +# volume-later a volume name that docker volume ls adds from +# its second call on +# log every call +F="$TMPDIR/fake" +mkdir -p "$TMPDIR/bin" +cat > "$TMPDIR/bin/docker" <<'EOF' +#!/bin/bash +F="$FAKE_DOCKER" +printf '%s\n' "$*" >> "$F/log" +case "$1" in + info) cat "$F/daemon" 2>/dev/null || echo d1; exit 0 ;; + context) echo default; exit 0 ;; + images) exit 0 ;; + wait) exit 0 ;; + rm) + shift + [ "$1" != -f ] || shift + [ ! -e "$F/helper-rm-fails" ] || exit 1 + grep -vxF "$1" "$F/helpers" > "$F/helpers.new" 2>/dev/null || true + mv "$F/helpers.new" "$F/helpers" + exit 0 ;; + ps) + case "$*" in + *"label=swarm-core.cleanup="*) + [ ! -e "$F/ps-fails" ] || exit 1 + cat "$F/helpers" + exit 0 ;; + *"ancestor="*) exit 0 ;; + *"volume="*) + v=$(printf '%s\n' "$*" | sed -n 's/.*volume=\([^ ]*\).*/\1/p') + cat "$F/volusers/$v" 2>/dev/null + exit 0 ;; + *"label=swarm-core.session="*) exit 0 ;; + *"label=swarm-core.project="*) + while read -r id name; do + [ -n "$id" ] || continue + jq -nc --arg id "$id" --arg n "$name" --arg o "$FAKE_CK" \ + '{id: $id, name: $n, project: "proj", owner: $o}' + done < "$F/containers" ;; + esac + exit 0 ;; + container) + [ "$2" = inspect ] || exit 1 + if [ "$3" = -f ] || [ "$3" = --format ]; then + # A helper's state, from $F/helper-status. + if grep -qxF "$5" "$F/helpers" 2>/dev/null \ + && [ -f "$F/helper-status" ]; then + cat "$F/helper-status" + exit 0 + fi + [ -f "$F/inspect/$5" ] || exit 1 + jq -r '.[0].Id' "$F/inspect/$5" + exit 0 + fi + [ -f "$F/inspect/$3" ] || exit 1 + cat "$F/inspect/$3" + exit 0 ;; + volume) + case "$2" in + ls) + cat "$F/volumes" + # A volume that shows up after the first listing. + if [ -f "$F/volume-later" ]; then + if [ -f "$F/volume-listed" ]; then + cat "$F/volume-later" + else + : > "$F/volume-listed" + fi + fi + exit 0 ;; + rm) + grep -qxF "$3" "$F/volumes" || exit 1 + grep -vxF "$3" "$F/volumes" > "$F/volumes.new" || true + mv "$F/volumes.new" "$F/volumes" + exit 0 ;; + inspect) + name="${*: -1}" + grep -qxF "$name" "$F/volumes" || exit 1 + case "$*" in + *Mountpoint*) echo "/var/lib/docker/volumes/$name/_data" ;; + *"json .Labels"*) cat "$F/vollabels/$name" ;; + *swarm-core.context*) echo full ;; + *) echo "{}" ;; + esac + exit 0 ;; + esac + exit 1 ;; + run) + d=$(mktemp -d "$F/run.XXXXXX") + # A volume scan: the real scanner over voldir/, its + # script on stdin, its output streamed back as a tar. + vol=$(printf '%s\n' "$*" \ + | sed -n 's/.*type=volume,src=\([^,]*\),.*/\1/p') + if [ -n "$vol" ]; then + cat > "$d/scan.sh" + mkdir "$d/out" + ctx="${*: -1}" + bash "$d/scan.sh" worktree "$F/voldir/$vol" "$ctx" "$d/out" \ + >&2 || exit $? + set -- report + [ ! -f "$d/out/bundle" ] || set -- "$@" bundle + [ ! -f "$d/out/objects" ] || set -- "$@" objects + tar -C "$d/out" -cf - "$@" + exit 0 + fi + # A deletion helper: cleanup's own program, run here with the + # bind mount's source in place of /runtime (an empty directory + # when the daemon sees another file system). + prog=""; prev=""; src="" + for a in "$@"; do + [ "$prev" != -c ] || prog="$a" + case "$a" in + type=bind,src=*) + src="${a#type=bind,src=}" + src="${src%%,dst=*}" ;; + esac + prev="$a" + done + rel="${*: -2:1}" + token="${*: -1}" + if [ -e "$F/remote-daemon" ]; then + src="$d/elsewhere" + mkdir -p "$src" + fi + prog="${prog//\/runtime/$src}" + # Its mount table: helper-mountinfo (what the daemon shows it). + prog="${prog//\/proc\/self\/mountinfo/$F/helper-mountinfo}" + PATH="$ORIG_PATH" sh -c "$prog" sh "$rel" "$token" + exit $? ;; +esac +exit 0 +EOF +chmod +x "$TMPDIR/bin/docker" + +# A shim rm that cannot delete what is under the runtime directory while +# $F/rm-fails exists: files the containers wrote as their own user, which +# only the Docker helper (root) can delete. +mkdir -p "$TMPDIR/bin-rm" +cat > "$TMPDIR/bin-rm/rm" <<'EOF' +#!/bin/bash +if [ -e "$FAKE_DOCKER/rm-fails" ]; then + for a in "$@"; do + case "$a" in + "$FAKE_RT"/*) exit 1 ;; + esac + done +fi +exec /bin/rm "$@" +EOF +chmod +x "$TMPDIR/bin-rm/rm" + +# A checkout with one commit; its bare repo and a mirror whose name has +# a space; the runtime directory, claimed, with engagement EID. +setup() { + local eid="${1:-e1}" head + chmod -R u+rwX "$TMPDIR/case" 2>/dev/null || true + rm -rf "$TMPDIR/case" "$F" + mkdir -p "$TMPDIR/case" "$F/inspect" "$F/vollabels" "$F/voldir" \ + "$F/volusers" + : > "$F/containers" + : > "$F/volumes" + : > "$F/helpers" + : > "$F/log" + echo "1 0 0:1 / / rw - overlay overlay rw" > "$F/helper-mountinfo" + REPO="$TMPDIR/case/proj" + git init -q "$REPO" + echo a > "$REPO/a" + gitc -C "$REPO" add a + gitc -C "$REPO" commit -qm init + CK=$(cd -P "$REPO" && pwd -P) + RT="$TMPDIR/case/runtime" + mkdir -m 700 "$RT" + swarm_runtime_claim "$RT" "$REPO" "$P" + swarm_state_update "$RT/state.json" \ + '.engagement_id = $e | .docker_daemon = "d1"' --arg e "$eid" + git clone -q --bare --no-hardlinks "$REPO" "$RT/upstream.git" + head=$(git -C "$REPO" rev-parse HEAD) + git -C "$RT/upstream.git" update-ref refs/heads/agent-work "$head" + mkdir -p "$RT/mirrors" + git init -q --bare "$RT/mirrors/lib space.git" + gitc -C "$REPO" push -q "$RT/mirrors/lib space.git" HEAD:refs/heads/main + : > "$RT/engagement.lock" +} + +# Run launch.sh cleanup ARGS; print "". SHIM_DIR (when set) +# comes first on PATH. +SHIM_DIR="" +cleanup() { + local rc=0 orig="$PATH" + (cd "$REPO" && PATH="${SHIM_DIR:+$SHIM_DIR:}$TMPDIR/bin:$PATH" \ + ORIG_PATH="$orig" FAKE_DOCKER="$F" FAKE_CK="$CK" FAKE_RT="$RT" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + "$BASH" "$LAUNCH" cleanup "$@" > "$TMPDIR/out" 2>&1) || rc=$? + printf '%s' "$rc" +} +said() { + grep -cF -- "$1" "$TMPDIR/out" | awk '{ print ($1 > 0) }' +} +present() { + if [ -e "$1" ] || [ -L "$1" ]; then echo yes; else echo no; fi +} +recovery_refs() { + git -C "$REPO" for-each-ref --format='%(refname)' refs/swarm-recovery/ \ + | grep -c . || true +} +refused() { + local label="$1" expect="$2" + assert_eq "$label: refused" "1" "$(cleanup "${@:3}")" + assert_eq "$label: for that reason" "1" "$(said "$expect")" + assert_eq "$label: the bare repo stays" "yes" \ + "$(present "$RT/upstream.git")" + [ -z "${SHOW_REASON:-}" ] || grep -m1 '^ERROR' "$TMPDIR/out" \ + | sed 's/^/ reason: /' || true +} + +echo "=== 1. Cleanup preserves, then deletes ===" +setup +# An object only the bare repo has, outside the preservation refs. +blob=$(printf 'only in the bare repo\n' \ + | git -C "$RT/upstream.git" hash-object -w --stdin) +git -C "$RT/upstream.git" update-ref refs/keep/extra "$blob" +assert_eq "cleanup succeeds" "0" "$(cleanup)" +assert_eq "the bare repo is gone" "no" "$(present "$RT/upstream.git")" +assert_eq "no remnant is left" "no" "$(present "$RT/upstream.git.removing")" +assert_eq "the mirror is gone" "no" "$(present "$RT/mirrors")" +assert_eq "the lock stays" "yes" "$(present "$RT/engagement.lock")" +assert_eq "the runtime directory stays" "yes" "$(present "$RT")" +assert_eq "the state keeps only the claim" '["project","repo_root","schema"]' \ + "$(jq -c 'keys' "$RT/state.json")" +assert_eq "recovery refs are written" "1" \ + "$(recovery_refs | awk '{ print ($1 > 0) }')" +e1hex=$(printf 'e1' | od -An -tx1 | tr -d ' \n') +base=$(git -C "$REPO" for-each-ref --format='%(refname)' \ + "refs/swarm-recovery/e${e1hex}/" \ + | sed -n 's|^\(refs/swarm-recovery/e[0-9a-f]*/[^/]*\)/.*|\1|p' \ + | sort -u) +assert_eq "one snapshot, under the engagement's hex" "1" \ + "$(printf '%s\n' "$base" | grep -c .)" +assert_eq "the object only the bare repo had is kept" "1" \ + "$(git -C "$REPO" rev-list --objects --all | grep -c "^$blob" || true)" +map=$(git -C "$REPO" cat-file blob "$base/sources") +assert_eq "the sources map names the bare repo" "1" \ + "$(printf '%s\n' "$map" | grep -c ' bare "' || true)" +assert_eq "and the mirror, its name kept" "1" \ + "$(printf '%s\n' "$map" | grep -cF 'mirror "' || true)" +assert_eq "a mirror name with a space still makes valid refs" "1" \ + "$(printf '%s\n' "$map" | grep -cF 'lib space.git"' || true)" +assert_eq "cleanup says where the refs are" "1" \ + "$(said "Recovery refs: $base/")" +# A second run has nothing left to do. +assert_eq "cleanup again finds no engagement" "0" "$(cleanup)" +assert_eq "and says so" "1" "$(said "nothing to clean up")" + +# An engagement id that is no valid ref component still gives valid refs. +setup 'eng..1' +assert_eq "an engagement id with .. is fine" "0" "$(cleanup)" +assert_eq "its refs are under its hex" "1" \ + "$(git -C "$REPO" for-each-ref "refs/swarm-recovery/e$(printf 'eng..1' \ + | od -An -tx1 | tr -d ' \n')/" | grep -c . | awk '{ print ($1 > 0) }')" + +echo "" +echo "=== 2. What stops cleanup before anything is deleted ===" +setup +rm "$RT/state.json" +assert_eq "no state file: refused" "1" "$(cleanup)" +assert_eq "for that reason" "1" "$(said "no state file")" +setup +echo d2 > "$F/daemon" +refused "another Docker daemon" "runs on Docker daemon d1" +setup +# A runtime directory of another project id (SWARM_RUNTIME_DIR kept, +# SWARM_PROJECT changed): cleanup would look for that project's +# containers and volumes, so it refuses before anything else. +setup +echo "id-c1 proj-agent-1" > "$F/containers" +cp "$RT/state.json" "$TMPDIR/state.before" +rc=0 +_orig="$PATH" +(cd "$REPO" && PATH="$TMPDIR/bin:$PATH" ORIG_PATH="$_orig" \ + FAKE_DOCKER="$F" FAKE_CK="$CK" FAKE_RT="$RT" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT=other SWARM_SKIP_DEP_CHECK=1 \ + "$BASH" "$LAUNCH" cleanup > "$TMPDIR/out" 2>&1) || rc=$? +assert_eq "a runtime of another project id: refused" "1" "$rc" +assert_eq "for that reason" "1" "$(said "belongs to project id 'proj'")" +assert_eq "its bare repo stays" "yes" "$(present "$RT/upstream.git")" +assert_eq "and its state file" "same" \ + "$(cmp -s "$TMPDIR/state.before" "$RT/state.json" && echo same)" +assert_eq "Docker made or removed nothing" "0" \ + "$(grep -cE '^(run|create|rm|volume rm|commit|stop|kill)' "$F/log" \ + || true)" +# Another checkout of the project id claims the runtime and starts +# between this cleanup's first checks and its lock (a flock first on +# PATH writes that claim once, then locks): cleanup checks again under +# the lock and leaves that checkout's engagement alone. +mkdir -p "$TMPDIR/bin-flock" +cat > "$TMPDIR/bin-flock/flock" <<'EOF' +#!/bin/bash +if [ -n "${FLOCK_CLAIM:-}" ] && [ ! -e "$FAKE_RT/claimed" ]; then + : > "$FAKE_RT/claimed" + st="$FAKE_RT/state.json" + jq -c --arg r "$FLOCK_CLAIM" '.repo_root = $r' "$st" > "$st.new" + chmod 600 "$st.new" + mv "$st.new" "$st" +fi +PATH="$ORIG_PATH" exec flock "$@" +EOF +chmod +x "$TMPDIR/bin-flock/flock" +setup +mkdir -p "$TMPDIR/case/other" +_other=$(cd -P "$TMPDIR/case/other" && pwd -P) +swarm_state_update "$RT/state.json" 'del(.repo_root)' +SHIM_DIR="$TMPDIR/bin-flock" +assert_eq "a runtime another checkout claims before the lock: refused" "1" \ + "$(FLOCK_CLAIM="$_other" cleanup)" +SHIM_DIR="" +assert_eq "for that reason" "1" "$(said "belongs to another checkout")" +assert_eq "that checkout's bare repo stays" "yes" \ + "$(present "$RT/upstream.git")" +assert_eq "and its mirror" "yes" "$(present "$RT/mirrors/lib space.git")" +assert_eq "no recovery refs are written" "0" "$(recovery_refs)" +assert_eq "Docker removed nothing" "0" \ + "$(grep -cE '^(rm|volume rm|commit|stop|kill)' "$F/log" || true)" +setup + +# The engagement lock is held by another process. +( + exec 9>>"$RT/engagement.lock" + flock -n 9 + sleep 30 +) & +_holder=$! +sleep 1 +refused "the lock held by another command" "another swarm-core operation" +kill "$_holder" 2>/dev/null || true +wait "$_holder" 2>/dev/null || true +setup +sleep 60 & +_live=$! +swarm_pending_write "$RT" proj-agent-interactive-x "$_live" d1 +refused "an interactive session starting" "interactive sessions are open" +kill "$_live" 2>/dev/null || true +wait "$_live" 2>/dev/null || true +setup +# Agent work the checkout does not have: harvest first. +gitc -C "$REPO" checkout -q -b side +echo b > "$REPO/b" +gitc -C "$REPO" add b +gitc -C "$REPO" commit -qm side +gitc -C "$REPO" push -q "$RT/upstream.git" \ + side:refs/heads/agent-work 2>/dev/null \ + || git -C "$RT/upstream.git" fetch -q "$REPO" "+side:agent-work" +gitc -C "$REPO" checkout -q - +gitc -C "$REPO" branch -q -D side +refused "agent work the checkout lacks" "agent-work" +assert_eq "nothing is copied then" "0" "$(recovery_refs)" +setup +# A container of the project (before workspace volumes: retire refuses). +echo "id-c1 proj-agent-1" > "$F/containers" +jq -n --arg ck "$CK" '[{Id: "id-c1", Name: "/proj-agent-1", + State: {Status: "exited"}, HostConfig: {RestartPolicy: {Name: "no"}}, + Mounts: [{Type: "bind", Source: "/x", Destination: "/workspace"}], + Config: {Labels: {"swarm-core.project": "proj", + "swarm-core.checkout": $ck}}}]' > "$F/inspect/id-c1" +refused "a container that cannot be retired" "every container must be retired" +setup +echo notes > "$RT/mirrors/notes.txt" +refused "a file in mirrors/ that is not a mirror" "holds what is not a mirror" +assert_eq "the file stays" "yes" "$(present "$RT/mirrors/notes.txt")" +setup +mv "$RT/mirrors/lib space.git" "$RT/mirrors/lib"$'\t'"tab.git" +refused "a mirror name with a tab" "names with a tab or a newline" +# A runtime directory whose own path holds a tab. +setup +mv "$RT" "$TMPDIR/case/run"$'\t'"time" +RT="$TMPDIR/case/run"$'\t'"time" +refused "a runtime directory with a tab in its path" "a tab or a newline" +# The home directory, and a runtime directory inside the checkout. +setup +assert_eq "a runtime directory that is HOME: refused" "1" \ + "$(HOME="$RT" cleanup)" +assert_eq "for that reason" "1" "$(said "refusing to clean up in")" +assert_eq "its bare repo stays" "yes" "$(present "$RT/upstream.git")" +setup +mv "$RT" "$REPO/rt" +RT="$REPO/rt" +refused "a runtime directory inside the checkout" \ + "is inside the checkout or its git dir" +# A linked worktree whose own git dir someone moved into mirrors/: its +# checkout must not lose it (the common git dir is elsewhere). +setup +WT="$TMPDIR/case/wt" +git -C "$REPO" worktree add -q "$WT" -b wtbranch +gd=$(git -C "$WT" rev-parse --absolute-git-dir) +mv "$gd" "$RT/mirrors/wt" +printf 'gitdir: %s\n' "$RT/mirrors/wt" > "$WT/.git" +(cd -P "$REPO/.git" && pwd -P) > "$RT/mirrors/wt/commondir" +printf '%s\n' "$WT/.git" > "$RT/mirrors/wt/gitdir" +swarm_state_update "$RT/state.json" '.repo_root = $r' \ + --arg r "$(cd -P "$WT" && pwd -P)" +# Run from the worktree, which the runtime directory now belongs to. +REPO="$WT" +CK=$(cd -P "$WT" && pwd -P) +assert_eq "a checkout whose git dir is in the runtime directory: refused" \ + "1" "$(cleanup)" +assert_eq "for that reason" "1" "$(said "holds the checkout")" +assert_eq "its git dir stays" "yes" "$(present "$RT/mirrors/wt/HEAD")" +assert_eq "and the worktree still works" "wtbranch" \ + "$(git -C "$WT" rev-parse --abbrev-ref HEAD 2>/dev/null)" +# A file over the size limit (lowered through the scanner's environment). +setup +head -c 3000 /dev/urandom > "$RT/upstream.git/rescue.patch" +export SWARM_SCAN_FILE_LIMIT=1000 +refused "a large file in the bare repo" "holds work outside Git's history" +assert_eq "--discard-uncommitted drops exactly that" "0" \ + "$(cleanup --discard-uncommitted)" +assert_eq "and lists it" "1" "$(said "large-file rescue.patch 3000")" +unset SWARM_SCAN_FILE_LIMIT +setup +# A corrupt object only the checkout has (the import never reads it): +# the final git fsck fails, and nothing is deleted. +obj=$(printf 'only in the checkout\n' | git -C "$REPO" hash-object -w --stdin) +f="$REPO/.git/objects/${obj:0:2}/${obj:2}" +chmod u+w "$f" +printf 'garbage' > "$f" +refused "a checkout git fsck finds broken" "git fsck found problems" + +echo "" +echo "=== 3. Workspace volumes ===" +# Add volume NAME of this checkout holding a committed repository (or +# nothing, with EMPTY set). +add_volume() { + local name="$1" empty="${2:-}" + printf '%s\n' "$name" >> "$F/volumes" + jq -n --arg ck "$CK" '{"swarm-core.project": "proj", + "swarm-core.checkout": $ck, "swarm-core.context": "full"}' \ + > "$F/vollabels/$name" + mkdir -p "$F/voldir/$name" + if [ -z "$empty" ]; then + git init -q "$F/voldir/$name" + echo w > "$F/voldir/$name/w" + gitc -C "$F/voldir/$name" add w + gitc -C "$F/voldir/$name" commit -qm work + fi +} +setup +add_volume proj-agent-1-ws-00000001 +add_volume proj-agent-2-ws-00000002 +assert_eq "two volumes: cleanup succeeds" "0" "$(cleanup)" +assert_eq "both are scanned" "2" "$(grep -c 'type=volume,src=' "$F/log")" +assert_eq "both are removed" "0" "$(grep -c . "$F/volumes" || true)" +assert_eq "their work is kept" "1" \ + "$(git -C "$REPO" for-each-ref refs/swarm-recovery/ | grep -c . \ + | awk '{ print ($1 > 0) }')" +setup +add_volume proj-agent-1-ws-00000001 empty +assert_eq "an empty volume: cleanup succeeds" "0" "$(cleanup)" +assert_eq "and removes it" "0" "$(grep -c . "$F/volumes" || true)" +setup +add_volume proj-agent-1-ws-00000001 +echo dirty > "$F/voldir/proj-agent-1-ws-00000001/w" +refused "uncommitted work in a volume" "holds work outside Git's history" +assert_eq "the volume stays" "1" "$(grep -c . "$F/volumes")" +assert_eq "--discard-uncommitted drops it" "0" \ + "$(cleanup --discard-uncommitted)" +setup +add_volume proj-agent-1-ws-00000001 +echo id-x > "$F/volusers/proj-agent-1-ws-00000001" +refused "a volume a container uses" "is in use by containers" +setup +add_volume proj-agent-1-ws-00000001 +jq -n '{"swarm-core.project": "proj", "swarm-core.checkout": "/elsewhere"}' \ + > "$F/vollabels/proj-agent-1-ws-00000001" +refused "a volume of another checkout" "is not this checkout's" + +echo "" +echo "=== 4. A run that stops half way, and the next one ===" +# The first run stops at its first deletion: this user's rm cannot +# delete, and the Docker helper sees another file system. +first_run_stops() { + : > "$F/rm-fails" + : > "$F/remote-daemon" + SHIM_DIR="$TMPDIR/bin-rm" + assert_eq "the first run stops" "1" "$(cleanup)" + assert_eq "because the helper sees another directory" "1" \ + "$(said "not the same directory")" + SHIM_DIR="" + rm -f "$F/rm-fails" "$F/remote-daemon" +} +setup +first_run_stops +assert_eq "the mirror was renamed, not deleted" "yes" \ + "$(present "$RT/mirrors/lib space.git.removing")" +assert_eq "the checkpoint is there" "true" \ + "$(jq 'has("cleanup")' "$RT/state.json")" +rc=0 +swarm_cleanup_pending_check "$RT/state.json" 2>/dev/null || rc=$? +assert_eq "other commands refuse meanwhile" "1" "$rc" +rc=0 +(cd "$REPO" && SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" \ + "$BASH" "$REPO_ROOT/harvest.sh" > "$TMPDIR/hout" 2>&1) || rc=$? +assert_eq "harvest refuses meanwhile" "1" "$rc" +assert_eq "saying a cleanup is in progress" "1" \ + "$(grep -c 'cleanup in progress' "$TMPDIR/hout" || true)" +echo rescue > "$RT/mirrors/lib space.git.removing/rescue.txt" +assert_eq "the next run finishes" "0" "$(cleanup)" +assert_eq "the remnant is gone" "no" \ + "$(present "$RT/mirrors/lib space.git.removing")" +assert_eq "so is the bare repo" "no" "$(present "$RT/upstream.git")" +assert_eq "the checkpoint is gone" "false" \ + "$(jq 'has("cleanup")' "$RT/state.json")" +assert_eq "two snapshots were written" "2" \ + "$(git -C "$REPO" for-each-ref --format='%(refname)' refs/swarm-recovery/ \ + | sed -n 's|^refs/swarm-recovery/e[0-9a-f]*/\([^/]*\)/.*|\1|p' \ + | sort -u | grep -c .)" +assert_eq "a file added to the remnant between runs is kept" "1" \ + "$(git -C "$REPO" rev-list --objects --all \ + | grep -c "^$(printf 'rescue\n' | git hash-object --stdin)" || true)" +# Something mounted under a remnant that the daemon shows the deletion +# helper and this host does not see: the helper finds it in its own +# mount table and deletes nothing. (The table writes the space in the +# path as \040.) +setup +mkdir -p "$RT/mirrors/lib space.git/sub" +echo keep > "$RT/mirrors/lib space.git/sub/sentinel" +: > "$F/rm-fails" +SHIM_DIR="$TMPDIR/bin-rm" +printf '36 25 0:32 / %s rw - tmpfs tmpfs rw\n' \ + "$(printf '%s' "$RT/mirrors/lib space.git.removing/sub" \ + | sed 's/ /\\040/g')" >> "$F/helper-mountinfo" +assert_eq "a mount only the daemon sees: refused" "1" "$(cleanup)" +assert_eq "for that reason" "1" "$(said "something is mounted in it")" +assert_eq "what is mounted there stays" "yes" \ + "$(present "$RT/mirrors/lib space.git.removing/sub/sentinel")" +SHIM_DIR="" +rm -f "$F/rm-fails" +# A mount table the helper cannot read (here: a directory in its +# place): it deletes nothing either. +setup +mkdir -p "$RT/mirrors/lib space.git/sub" +echo keep > "$RT/mirrors/lib space.git/sub/sentinel" +: > "$F/rm-fails" +SHIM_DIR="$TMPDIR/bin-rm" +rm -f "$F/helper-mountinfo" +mkdir "$F/helper-mountinfo" +assert_eq "a mount table that cannot be read: refused" "1" "$(cleanup)" +assert_eq "for that reason" "1" "$(said "cannot read the mount table")" +assert_eq "and the remnant keeps its files" "yes" \ + "$(present "$RT/mirrors/lib space.git.removing/sub/sentinel")" +SHIM_DIR="" +rm -f "$F/rm-fails" +# A link planted in a remnant this user cannot clean (here: rm fails) +# does not redirect cleanup's marker onto a file outside. +setup +first_run_stops +echo precious > "$TMPDIR/case/external" +ln -sf "$TMPDIR/case/external" \ + "$RT/mirrors/lib space.git.removing/.swarm-cleanup-token" +: > "$F/rm-fails" +SHIM_DIR="$TMPDIR/bin-rm" +assert_eq "with a link planted in the remnant: the next run finishes" "0" \ + "$(cleanup)" +SHIM_DIR="" +rm -f "$F/rm-fails" +assert_eq "the file the link points to is untouched" "precious" \ + "$(cat "$TMPDIR/case/external")" +# A remnant that is itself a link is not followed. +setup +mkdir -p "$TMPDIR/case/elsewhere" +ln -s "$TMPDIR/case/elsewhere" "$RT/upstream.git.new.removing" +refused "a remnant that is a link" "is a link; cleanup will not follow it" +assert_eq "what it points to stays" "yes" "$(present "$TMPDIR/case/elsewhere")" +# A remnant a deletion got part of the way through: no HEAD left. +setup +first_run_stops +rm -f "$RT/mirrors/lib space.git.removing/HEAD" +assert_eq "a remnant without HEAD: the next run finishes" "0" "$(cleanup)" +assert_eq "the remnant is gone" "no" \ + "$(present "$RT/mirrors/lib space.git.removing")" +# Recovery refs an earlier run wrote, deleted or changed since. +setup +first_run_stops +ref=$(git -C "$REPO" for-each-ref --format='%(refname)' refs/swarm-recovery/ \ + | grep '/s[0-9]*/' | head -n 1) +git -C "$REPO" update-ref -d "$ref" +refused "a recovery ref of the first run deleted" "changed or went" +assert_eq "the remnant stays" "yes" \ + "$(present "$RT/mirrors/lib space.git.removing")" +setup +first_run_stops +ref=$(git -C "$REPO" for-each-ref --format='%(refname)' refs/swarm-recovery/ \ + | grep '/sources$' | head -n 1) +git -C "$REPO" update-ref "$ref" "$(git -C "$REPO" rev-parse HEAD)" +refused "the sources map retargeted" "changed or went" +# A helper container a killed run left is waited for and removed first. +setup +echo id-helper > "$F/helpers" +assert_eq "a helper left by a killed run: cleanup succeeds" "0" "$(cleanup)" +assert_eq "it was checked" "1" \ + "$(grep -c 'container inspect -f {{.State.Status}} id-helper' "$F/log")" +assert_eq "and removed" "1" "$(grep -c '^rm id-helper$' "$F/log")" +setup +echo id-helper > "$F/helpers" +: > "$F/helper-rm-fails" +refused "a helper that cannot be removed" "an earlier cleanup left remain" +setup +echo id-helper > "$F/helpers" +echo running > "$F/helper-status" +export SWARM_CLEANUP_HELPER_WAIT=1 +refused "a helper still running after the wait" "is still running after 1s" +unset SWARM_CLEANUP_HELPER_WAIT +setup +: > "$F/ps-fails" +refused "helpers that cannot be listed" "cannot list Docker containers" +# A hidden mirror (a submodule named with a leading dot) left half way. +setup +rm -rf "$RT/mirrors/lib space.git" +git init -q --bare "$RT/mirrors/.hidden.git" +gitc -C "$REPO" push -q "$RT/mirrors/.hidden.git" HEAD:refs/heads/main +first_run_stops +assert_eq "a hidden mirror was renamed" "yes" \ + "$(present "$RT/mirrors/.hidden.git.removing")" +assert_eq "the next run finishes it" "0" "$(cleanup)" +assert_eq "the hidden remnant is gone" "no" \ + "$(present "$RT/mirrors/.hidden.git.removing")" +assert_eq "and mirrors/ with it" "no" "$(present "$RT/mirrors")" + +# Run cleanup's function ARGS in a subshell with the case's variables +# and the fake docker; with FAIL_BARE set, writing the bare repo's line +# fails. Print its status. +in_cleanup() { + local rc=0 + ( + # shellcheck source=../lib/preserve.sh + source "$REPO_ROOT/lib/preserve.sh" + # shellcheck source=../lib/retire.sh + source "$REPO_ROOT/lib/retire.sh" + # shellcheck source=../lib/cleanup.sh + source "$REPO_ROOT/lib/cleanup.sh" + # shellcheck disable=SC2034 # read by the functions run here + RUNTIME_DIR="$RT" MIRROR_DIR="$RT/mirrors" \ + STATE_FILE="$RT/state.json" BARE_REPO="$RT/upstream.git" \ + PROJECT="$P" CHECKOUT_PATH="$CK" + # shellcheck disable=SC2329 + docker() { + FAKE_DOCKER="$F" FAKE_CK="$CK" "$TMPDIR/bin/docker" "$@" + } + if [ -n "${FAIL_BARE:-}" ]; then + # shellcheck disable=SC2329 + printf() { + case "$1" in + bare*) return 1 ;; + esac + # shellcheck disable=SC2059 # printf's own arguments + builtin printf "$@" + } + fi + "$@" + ) > "$TMPDIR/out" 2>&1 || rc=$? + builtin printf '%s' "$rc" +} + +# Only what lies in the runtime directory (or its mirrors/) is ever +# renamed or deleted, whatever path the deletion is handed. +setup +mkdir -p "$TMPDIR/case/outside" +echo keep > "$TMPDIR/case/outside/f" +assert_eq "a deletion outside the runtime directory is refused" "1" \ + "$(in_cleanup cleanup_delete_host s1 "$TMPDIR/case/outside")" +assert_eq "for that reason" "1" "$(said "is not in")" +assert_eq "and what is there stays" "yes" "$(present "$TMPDIR/case/outside/f")" + +# A failed write to the list of sources stops cleanup: a location left +# out of it would be neither preserved nor deleted. +setup +assert_eq "the sources are listed" "0" \ + "$(in_cleanup cleanup_sources "$TMPDIR/list")" +assert_eq "the bare repo among them" "1" \ + "$(grep -c "^bare" "$TMPDIR/list" || true)" +assert_eq "a write to that list that fails stops it" "1" \ + "$(FAIL_BARE=1 in_cleanup cleanup_sources "$TMPDIR/list")" +if [ -w /dev/full ]; then + assert_eq "so does a full disk (/dev/full)" "1" \ + "$(in_cleanup cleanup_sources /dev/full)" +fi + +# Before the engagement's records go, cleanup lists the sources again: +# one that showed up meanwhile keeps them. +setup +echo "${P}-late-ws-00000001" > "$F/volume-later" +assert_eq "a volume that shows up during cleanup: refused" "1" "$(cleanup)" +assert_eq "for that reason" "1" "$(said "left these in place")" +assert_eq "the checkpoint stays" "true" \ + "$(jq 'has("cleanup")' "$RT/state.json")" +assert_eq "and so does the engagement" "e1" \ + "$(jq -r '.engagement_id' "$RT/state.json")" + +# A read that fails part of the way through a list (an I/O error) ends +# the loop as the end of the list would. Sourced through BASH_ENV by +# the bash that runs cleanup, this makes the HOOK_AT-th read whose last +# variable is HOOK_VAR fail (HOOK_DO=fail), or first drop the bare +# repo's record from the checkpoint (HOOK_DO=drop); every other read +# works. +cat > "$TMPDIR/hook.sh" <<'EOF' +read() { + local c + if [ "$#" -ge 4 ] && [ "$2" = -u ] && [ "${!#}" = "$HOOK_VAR" ]; then + c=$(( $(cat "$HOOK_COUNT" 2>/dev/null || echo 0) + 1 )) + echo "$c" > "$HOOK_COUNT" + if [ "$c" -eq "$HOOK_AT" ]; then + case "$HOOK_DO" in + fail) return 1 ;; + drop) + jq -c 'del(.cleanup.runs[-1].sources[] + | select(.kind == "bare"))' "$STATE_FILE" \ + > "$STATE_FILE.new" \ + && mv "$STATE_FILE.new" "$STATE_FILE" ;; + esac + fi + fi + builtin read "$@" +} +EOF +# Run cleanup with that hook: hooked VAR AT DO. +hooked() { + rm -f "$TMPDIR/hook.count" + BASH_ENV="$TMPDIR/hook.sh" HOOK_VAR="$1" HOOK_AT="$2" HOOK_DO="$3" \ + HOOK_COUNT="$TMPDIR/hook.count" cleanup +} +# The sources here: s1 the mirror, s2 the bare repo. +setup +assert_eq "a read that fails while preserving: refused" "1" \ + "$(hooked mode 2 fail)" +assert_eq "for that reason" "1" "$(said "preserved 1 of the 2 locations")" +assert_eq "the mirror stays" "yes" "$(present "$RT/mirrors/lib space.git")" +assert_eq "so does the bare repo" "yes" "$(present "$RT/upstream.git")" +assert_eq "and no checkpoint is written" "false" \ + "$(jq 'has("cleanup")' "$RT/state.json")" +# The storage pass reads the list three times (two lines, then its +# end); the fifth read is the scan pass's second. +setup +assert_eq "a read that fails while scanning: refused" "1" \ + "$(hooked path 5 fail)" +assert_eq "for that reason" "1" "$(said "scanned 1 of the 2 locations")" +assert_eq "the bare repo stays" "yes" "$(present "$RT/upstream.git")" +# With a volume as third source, the storage pass's third read is the +# volume's: failing, it would leave the volume's Mountpoint out of the +# storage check. +setup +add_volume proj-agent-1-ws-00000001 +assert_eq "a read that fails while checking the storage: refused" "1" \ + "$(hooked path 3 fail)" +assert_eq "for that reason" "1" \ + "$(said "checked the storage of 2 of the 3 locations")" +assert_eq "the volume stays" "1" \ + "$(grep -c proj-agent-1-ws-00000001 "$F/volumes" || true)" +# The checkpoint loses the bare repo's record before the first deletion +# (the preservation pass reads three times, the deletion pass next). +setup +assert_eq "a source with no record in the checkpoint: refused" "1" \ + "$(hooked mode 4 drop)" +assert_eq "for that reason" "1" "$(said "has no record that")" +assert_eq "the bare repo stays where it is" "yes" \ + "$(present "$RT/upstream.git")" +# A run after one that stopped: a read that fails while checking the +# recovery refs of the earlier run. +setup +first_run_stops +assert_eq "a read that fails while checking earlier runs: refused" "1" \ + "$(hooked want 1 fail)" +assert_eq "for that reason" "1" \ + "$(said "cannot read the recovery records")" +assert_eq "the remnant stays" "yes" \ + "$(present "$RT/mirrors/lib space.git.removing")" + +# Nothing may be mounted in the runtime directory: deleting there would +# delete what the mount shows. +if [ ! -r /proc/self/mountinfo ]; then + # This host lists its mounts with mount(8); a fake one shows one. + mkdir -p "$TMPDIR/bin-mount" + cat > "$TMPDIR/bin-mount/mount" <<'EOF' +#!/bin/bash +echo "/dev/disk9 on $(cd -P "$FAKE_RT" && pwd -P)/mirrors/x (nullfs, local)" +EOF + chmod +x "$TMPDIR/bin-mount/mount" + setup + SHIM_DIR="$TMPDIR/bin-mount" + refused "something mounted in the runtime directory" \ + "something is mounted in" + SHIM_DIR="" +elif unshare -rm true 2>/dev/null; then + # A linked worktree's own git dir, bind-mounted (in a user and mount + # namespace of its own) where a remnant would be. + setup + WT="$TMPDIR/case/wt" + git -C "$REPO" worktree add -q "$WT" -b wtbranch + gd=$(git -C "$WT" rev-parse --absolute-git-dir) + wtck=$(cd -P "$WT" && pwd -P) + swarm_state_update "$RT/state.json" '.repo_root = $r' --arg r "$wtck" + mkdir "$RT/mirrors/w t.removing" + rc=0 + orig="$PATH" + (cd "$WT" && PATH="$TMPDIR/bin:$PATH" ORIG_PATH="$orig" \ + FAKE_DOCKER="$F" FAKE_CK="$wtck" FAKE_RT="$RT" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + unshare -rm "$BASH" -c \ + 'mount --bind "$1" "$2" && exec "$BASH" "$3" cleanup' sh \ + "$gd" "$RT/mirrors/w t.removing" "$LAUNCH") \ + > "$TMPDIR/out" 2>&1 || rc=$? + assert_eq "a bind mount in the runtime directory: refused" "1" "$rc" + assert_eq "for that reason" "1" "$(said "something is mounted in")" + assert_eq "the worktree's git dir keeps its HEAD" "yes" \ + "$(present "$gd/HEAD")" + assert_eq "and the worktree still works" "wtbranch" \ + "$(git -C "$WT" symbolic-ref --short HEAD 2>/dev/null)" +else + echo " SKIP: no bind mount here (unshare -rm)" +fi + +echo "" +echo "=== 5. Commands that wait for a pending cleanup ===" +# Run launch.sh ARGS (another command than cleanup) with a small +# swarmfile; print "". +lifecycle() { + local rc=0 orig="$PATH" + (cd "$REPO" && PATH="$TMPDIR/bin:$PATH" \ + ORIG_PATH="$orig" FAKE_DOCKER="$F" FAKE_CK="$CK" FAKE_RT="$RT" \ + SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ + SWARM_CONFIG="$TMPDIR/case/swarm.json" \ + "$BASH" "$LAUNCH" "$@" > "$TMPDIR/out" 2>&1) || rc=$? + printf '%s' "$rc" +} +# With a cleanup half done, every lifecycle command stops before it +# creates or replaces anything, whether the bare repo is still there, +# renamed or gone. Each command starts from its own half-done cleanup. +for bare in present renamed gone; do + for cmd in start post-process interactive stop; do + setup + first_run_stops + case "$bare" in + renamed) mv "$RT/upstream.git" "$RT/upstream.git.removing" ;; + gone) rm -rf "$RT/upstream.git" ;; + esac + printf '%s\n' '{ "prompt": "p.md", "agents": [{ "count": 1, + "model": "m" }], "post_process": { "prompt": "p.md" } }' \ + > "$TMPDIR/case/swarm.json" + echo p > "$REPO/p.md" + (cd "$RT" && find . | LC_ALL=C sort) > "$TMPDIR/before" + cp "$RT/state.json" "$TMPDIR/state.before" + : > "$F/log" + what="bare repo ${bare}, ${cmd}" + assert_eq "${what}: refused" "1" "$(lifecycle "$cmd")" + assert_eq "${what}: for that reason" "1" \ + "$(said "cleanup in progress")" + (cd "$RT" && find . | LC_ALL=C sort) > "$TMPDIR/after" + assert_eq "${what}: the runtime directory is as it was" "" \ + "$(diff "$TMPDIR/before" "$TMPDIR/after" || true)" + assert_eq "${what}: so is the state file" "same" \ + "$(cmp -s "$TMPDIR/state.before" "$RT/state.json" \ + && echo same)" + assert_eq "${what}: Docker made or removed nothing" "0" \ + "$(grep -cE '^(run|create|rm|volume|build|stop|kill)' \ + "$F/log" || true)" + done +done + +echo "" +echo "===============================" +echo " ${PASS} passed, ${FAIL} failed" +echo "===============================" +[ "$FAIL" -eq 0 ] diff --git a/tests/test_preserve.sh b/tests/test_preserve.sh index 4087f02..cbf0abd 100755 --- a/tests/test_preserve.sh +++ b/tests/test_preserve.sh @@ -1017,6 +1017,167 @@ rm -rf "$RT/upstream.git" mkdir -p "$RT/upstream.git" assert_eq "an unrelated deletion target is fine" "0" \ "$(storage "$RT/upstream.git")" +# Targets are compared with the git dir as well as its object store, +# whether or not this host has them (a volume's Mountpoint may be a path +# in Docker's own machine); a link on the way is followed. +assert_eq "a target inside the git dir, outside objects/, is refused" "1" \ + "$(storage "$ST/.git/hooks")" +assert_eq "a target that does not exist, inside the git dir, is refused" \ + "1" "$(storage "$ST/.git/volumes/x/_data")" +assert_eq "a target around the git dir is refused" "1" "$(storage "$ST")" +assert_eq "a target elsewhere that this host lacks is fine" "0" \ + "$(storage "/nonexistent-$$/var/lib/docker/volumes/x/_data")" +ln -s "$ST/.git" "$TMPDIR/gitalias" +assert_eq "a missing target behind a link into the git dir is refused" "1" \ + "$(storage "$TMPDIR/gitalias/volumes/x/_data")" +rm "$TMPDIR/gitalias" +# A linked worktree's own git dir, moved away from the common one: the +# targets are compared with it too. +WTP="$TMPDIR/wt-admin" +mkdir -p "$WTP" +git -C "$ST" worktree add -q "$TMPDIR/storage-wt" -b storage-wt +_wtgd=$(git -C "$TMPDIR/storage-wt" rev-parse --absolute-git-dir) +mv "$_wtgd" "$WTP/gd" +printf 'gitdir: %s\n' "$WTP/gd" > "$TMPDIR/storage-wt/.git" +(cd -P "$ST/.git" && pwd -P) > "$WTP/gd/commondir" +printf '%s\n' "$TMPDIR/storage-wt/.git" > "$WTP/gd/gitdir" +storage_wt() { + local rc=0 + swarm_checkout_storage_ok "$TMPDIR/storage-wt" "$RT" "$@" \ + 2>/dev/null || rc=$? + printf '%s' "$rc" +} +assert_eq "a linked worktree is fine" "0" "$(storage_wt)" +assert_eq "its own git dir as a target is refused" "1" \ + "$(storage_wt "$WTP/gd")" +assert_eq "a target inside that git dir is refused" "1" \ + "$(storage_wt "$WTP/gd/logs")" +assert_eq "a target around that git dir is refused" "1" \ + "$(storage_wt "$WTP")" +assert_eq "a target that does not exist, inside that git dir, is refused" \ + "1" "$(storage_wt "$WTP/gd/volumes/x/_data")" +assert_eq "and the worktree still works" "storage-wt" \ + "$(git -C "$TMPDIR/storage-wt" symbolic-ref --short HEAD)" +# The same git dir bind-mounted at a target (in a user and mount +# namespace of its own) has another path but the same inodes. +if [ -r /proc/self/mountinfo ] && unshare -rm true 2>/dev/null; then + mkdir "$RT/target-wt" + rc=0 + unshare -rm "$BASH" -c ' + mount --bind "$1" "$2" || exit 9 + # shellcheck source=../lib/preserve.sh + source "$3/lib/preserve.sh" + swarm_checkout_storage_ok "$4" "$5" "$2"' sh \ + "$WTP/gd" "$RT/target-wt" "$REPO_ROOT" "$TMPDIR/storage-wt" "$RT" \ + > "$TMPDIR/bind.out" 2>&1 || rc=$? + assert_eq "that git dir bind-mounted at a target is refused" "1" "$rc" + assert_eq "as sharing files with it" "1" \ + "$(grep -c 'shares files with what cleanup deletes' \ + "$TMPDIR/bind.out" || true)" + rmdir "$RT/target-wt" +else + echo " SKIP: no bind mount here (unshare -rm)" +fi + +echo "" +echo "=== 13. Files mode keeps what is left of a repository ===" + +# What a deletion left of a repository: no history to walk, so every +# file is kept as a blob; files under objects/ whatever their size. +FM="$TMPDIR/remnant" +mkdir -p "$FM/objects/pack" "$FM/objects/ab" +head -c 5000 /dev/urandom > "$FM/objects/pack/pack-1.pack" +head -c 3000 /dev/urandom > "$FM/objects/ab/cdef" +head -c 4000 /dev/urandom > "$FM/rescue.patch" +echo small > "$FM/config" +ln -s ../config "$FM/link" +out=$(scan files "$FM" sha1 SWARM_SCAN_FILE_LIMIT=1000) +assert_eq "files mode succeeds" "no" \ + "$([ -e "$out/.rc" ] && echo yes || echo no)" +assert_eq "a large file outside objects/ is uncommitted" "1" \ + "$(has_line "$out" 'uncommitted large-file rescue.patch 4000')" +assert_eq "nothing else is reported" "1" "$(grep -c . "$out/report" || true)" +dst=$(imported "$out") +fl="$TMPDIR/fm.files" +git -C "$dst" cat-file blob refs/r/0/files | tr '\0' '\n' > "$fl" +for f in config link objects/ab/cdef objects/pack/pack-1.pack; do + assert_eq "files mode keeps ${f}" "1" \ + "$(awk -v f="$f" '$4 == f' "$fl" | grep -c . || true)" +done +k=$(awk '$4 == "objects/pack/pack-1.pack" { print $1 }' "$fl") +assert_eq "a pack over the limit is kept byte for byte" \ + "$(git hash-object --no-filters "$FM/objects/pack/pack-1.pack")" \ + "$(git -C "$dst" rev-parse "refs/r/0/f/$k")" +k=$(awk '$4 == "link" { print $1 }' "$fl") +assert_eq "a link is kept as its target" "../config" \ + "$(git -C "$dst" cat-file blob "refs/r/0/f/$k")" +assert_eq "the inventory of objects is empty" "0" \ + "$(grep -c . "$out/objects" || true)" +out=$(scan files "$FM" sha256) +dst=$(mktemp -d "$TMPDIR/dst.XXXXXX") +git init -q --object-format=sha256 "$dst" +git -C "$dst" fetch -q --no-tags "$out/bundle" 'refs/v/*:refs/r/*' +assert_eq "a sha256 bundle goes into a sha256 repository" "1" \ + "$(git -C "$dst" for-each-ref refs/r/0/files | grep -c . || true)" +out=$(scan files "$FM" "") +assert_eq "files mode needs the object format" "rc=3" "$(cat "$out/.rc")" +mkfifo "$FM/fifo" +out=$(scan files "$FM" sha1) +assert_eq "a special file fails the scan" "rc=3" "$(cat "$out/.rc")" +# A read of one of the scan's own lists that fails stops the scan +# (status 3) instead of ending the loop as the end of the list would: a +# find or a cat first on PATH leaves a directory where the file list, +# or the list of files to keep, was (EISDIR). +RBD="$TMPDIR/rb-remnant" +mkdir -p "$RBD/objects" +echo one > "$RBD/a" +echo two > "$RBD/objects/b" +_rb="$TMPDIR/rb-bin" +mkdir -p "$_rb" +cat > "$_rb/find" <<'EOF' +#!/bin/bash +PATH="$RB_PATH" find "$@" +rc=$? +if [ -n "${RB_FIND:-}" ] && [ -f "$RB_FIND" ]; then + rm -f "$RB_FIND" && mkdir "$RB_FIND" +fi +exit "$rc" +EOF +cat > "$_rb/cat" <<'EOF' +#!/bin/bash +if [ -n "${RB_CAT_SRC:-}" ] && [ "${1:-}" = "$RB_CAT_SRC" ]; then + PATH="$RB_PATH" cat "$1" || exit + rm -f "$RB_CAT" && mkdir "$RB_CAT" + exit 0 +fi +PATH="$RB_PATH" exec cat "$@" +EOF +chmod +x "$_rb/find" "$_rb/cat" +# Scan $RBD in files mode with the read of list WHICH (find, cat or +# none) failing; print the status. +rb_scan() { + local rc=0 out + out=$(mktemp -d "$TMPDIR/rb.XXXXXX") + case "$1" in + none) set -- ;; + find) set -- RB_FIND="$out/.flist0" ;; + cat) set -- RB_CAT_SRC="$out/.fnl" RB_CAT="$out/.fkeep" ;; + esac + env RB_PATH="$PATH" PATH="$_rb:$PATH" "$@" \ + bash "$SCAN" files "$RBD" sha1 "$out" > "$TMPDIR/rb.log" 2>&1 \ + || rc=$? + printf '%s' "$rc" +} +assert_eq "with its lists read in full, the scan works" "0" \ + "$(rb_scan none)" +assert_eq "a read of the file list that fails stops the scan" "3" \ + "$(rb_scan find)" +assert_eq "saying so" "1" \ + "$(grep -c 'cannot read a list' "$TMPDIR/rb.log" || true)" +assert_eq "so does one of the list of files to keep" "3" \ + "$(rb_scan cat)" +assert_eq "saying so" "1" \ + "$(grep -c 'cannot read a list' "$TMPDIR/rb.log" || true)" echo "" echo "===============================" diff --git a/tests/test_retire.sh b/tests/test_retire.sh index 8f8df3b..975f9b8 100755 --- a/tests/test_retire.sh +++ b/tests/test_retire.sh @@ -1334,6 +1334,43 @@ flock -n 8 assert_eq "a held lock is refused" "1" "$(retire 8>&-)" assert_eq "nothing removed then" "0" "$(removed)" exec 8>&- +# Another checkout of the project id claims the runtime between +# retire's first checks and its lock (a flock first on PATH writes that +# claim once, then locks): retire checks again under the lock and +# removes nothing. +mkdir -p "$TMPDIR/bin-flock" "$TMPDIR/other" +cat > "$TMPDIR/bin-flock/flock" <<'EOF' +#!/bin/bash +if [ -n "${FLOCK_CLAIM:-}" ] && [ ! -e "$SWARM_RUNTIME_DIR/claimed" ]; then + : > "$SWARM_RUNTIME_DIR/claimed" + st="$SWARM_RUNTIME_DIR/state.json" + jq -c --arg r "$FLOCK_CLAIM" '.repo_root = $r' "$st" > "$st.new" + chmod 600 "$st.new" + mv "$st.new" "$st" +fi +self=$(cd "$(dirname "$0")" && pwd) +IFS=: +for d in $PATH; do + if [ "$d" != "$self" ] && [ -x "$d/flock" ]; then + exec "$d/flock" "$@" + fi +done +exit 127 +EOF +chmod +x "$TMPDIR/bin-flock/flock" +reset_fake +add proj-agent-1 +cp "$RT/state.json" "$TMPDIR/state.keep" +swarm_state_update "$RT/state.json" 'del(.repo_root)' +SHIM_DIR="$TMPDIR/bin-flock" +assert_eq "a runtime another checkout claims before the lock: refused" "1" \ + "$(FLOCK_CLAIM="$(cd -P "$TMPDIR/other" && pwd -P)" retire)" +SHIM_DIR="" +assert_eq "for that reason" "1" \ + "$(grep -c 'belongs to another checkout' "$TMPDIR/out" || true)" +assert_eq "nothing removed then" "0" "$(removed)" +cp "$TMPDIR/state.keep" "$RT/state.json" +rm -f "$RT/claimed" echo "" echo "===============================" From 91521aa61afc09b8883e7193de3b2692678d37d0 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:35:39 +0100 Subject: [PATCH 09/27] Let the Pi runtime test run on Linux hosts whose UID is not 1000 --- CHANGELOG.md | 6 ++++++ tests/runtime_pi.sh | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c9a145..3267d2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- **Fix: `tests/runtime_pi.sh` on Linux hosts whose UID is not 1000.** + The test handed its Codex auth fixture to the container's user before + the driver read it on the host, so the Codex checks failed, and its + cleanup could not remove what the agents wrote into the bare repo. It + now reads the fixture first and removes the bare repo as root with + its test image. - **Feature: `launch.sh cleanup [--discard-uncommitted]`.** Deletes this checkout's engagement (containers, through retire's checks; workspace volumes; the bare repo; the mirrors; leftovers of an earlier cleanup) diff --git a/tests/runtime_pi.sh b/tests/runtime_pi.sh index a305f02..dd6d8c8 100755 --- a/tests/runtime_pi.sh +++ b/tests/runtime_pi.sh @@ -41,6 +41,15 @@ cleanup() { docker run --rm --user 0 -v "$WORK:$WORK" --entrypoint rm \ "$IMAGE" -rf "$WORK/codex-auth" >/dev/null 2>&1 || true fi + # The agents wrote into the bare repo as the container's user, whose + # files this user cannot delete when its UID is not 1000 (native + # Linux): remove them as root with the test image, before it goes. + # Never through a link: Docker would mount where it leads. + if [ -d "$RT" ] && [ ! -L "$RT" ]; then + docker run --rm --user 0 -v "$RT:/rt" --entrypoint rm \ + "$IMAGE" -rf /rt/upstream.git /rt/upstream.git.new /rt/mirrors \ + >/dev/null 2>&1 || true + fi docker image rm "$IMAGE" >/dev/null 2>&1 || true rm -rf "$WORK" "$RT" } @@ -160,12 +169,15 @@ mkdir -m 700 "$PI_AUTH_DIR" printf '%s\n' '{"openai-codex":{"type":"oauth","access":"expired", "refresh":"fixture-refresh","expires":0}}' > "$PI_AUTH_DIR/auth.json" ) -docker run --rm --user 0 -v "$PI_AUTH_DIR:/auth" \ - --entrypoint chown "$IMAGE" -R 1000:1000 /auth +# The driver reads auth.json on this host, so it runs before the files +# go to the container's user (this user cannot read them then when its +# UID is not 1000). AUTH_ARGS=() while IFS= read -r arg; do AUTH_ARGS+=("$arg") done < <(agent_docker_auth '' '' chatgpt '') +docker run --rm --user 0 -v "$PI_AUTH_DIR:/auth" \ + --entrypoint chown "$IMAGE" -R 1000:1000 /auth for i in 1 2; do docker run -d --network none --name "$IMAGE-codex-$i" \ -v "$BARE:/upstream:rw" \ From 48f57c3dd32cab6b999003346abf9a74eeceb515 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Thu, 5 Nov 2026 06:45:33 +0100 Subject: [PATCH 10/27] Give each engine container a workspace volume and start engagements only on a clean runtime --- CHANGELOG.md | 35 +- README.md | 17 +- USAGE.md | 204 +++- dashboard.sh | 80 +- harvest.sh | 20 +- launch.sh | 619 +++++++++---- lib/cleanup.sh | 296 +++--- lib/preserve-scan.sh | 4 - lib/preserve.sh | 36 +- lib/project.sh | 418 ++++++++- lib/retire.sh | 41 +- tests/runtime_cleanup.sh | 109 ++- tests/runtime_pi.sh | 7 + tests/runtime_signal_trap.sh | 30 +- tests/test.sh | 47 +- tests/test_cleanup.sh | 186 +++- tests/test_dashboard.sh | 174 +++- tests/test_harvest.sh | 90 +- tests/test_launch.sh | 1698 +++++++++++++++++++++++++++------- tests/test_preserve.sh | 28 + tests/test_retire.sh | 34 +- tests/test_runtime.sh | 68 +- 22 files changed, 3273 insertions(+), 968 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3267d2d..695348a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,38 @@ ## Unreleased +- **Breaking: workspace volumes, one Docker daemon per engagement, new + engagements only on a clean runtime directory.** Each engine container + gets a `/workspace` volume of its own (`-ws-`, labeled + with the project, checkout, engagement, engagement token, container, + kind, context and a nonce that is read back), so its work outlives it + for retire and cleanup. The workspace mount, the engagement labels, + `--restart no`, `--rm=false` and the container name come after + `docker_args`. The state file records the engagement's Docker daemon + and a random token; `post-process` and `interactive` joining it, + `stop`, `wait`, `retire` and `cleanup` refuse another daemon, and each + command keeps the Docker context it started with. `stop` takes no + lock, stops by ID only containers that carry the engagement's token + (`--numbered`: the numbered agents only), and fails when one still + runs; `wait` checks the token again before it goes on; `SWARM_ENGAGEMENT_TOKEN` binds `stop`, + `post-process`, `interactive` and `harvest.sh` to one engagement, and + the dashboard's `s` and `p` go through `launch.sh`. `start`, and + `post-process` or `interactive` when they would create the engagement, + refuse while the runtime directory holds anything of one; nothing is + removed by force any more (no `docker rm -f`, no `alpine` helper, no + `rm -rf` recipes): end an engagement with `harvest.sh` and + `./launch.sh cleanup`, which gains `--allow-unharvested` for work a + rewritten upstream left behind and checks the volumes before it + retires anything. `post-process` refuses while numbered agents of the + engagement run, and while an earlier post-process container is there + (`./launch.sh retire --post-process`). Bare repos and mirrors are cloned to `.new` and + renamed once ready, and a mistake in the swarmfile or a failed image + build stops `start` before the engagement is created. The runtime + directory must be a place cleanup can work in. Engagements of earlier + releases are not taken over: every command but `status` and `logs` + refuses them. Finish such an engagement with the release that + started it (check it out, stop the agents, run `harvest.sh`, follow + its cleanup steps), then come back. - **Fix: `tests/runtime_pi.sh` on Linux hosts whose UID is not 1000.** The test handed its Codex auth fixture to the container's user before the driver read it on the host, so the Codex checks failed, and its @@ -51,8 +83,7 @@ change). The checks run in a `docker commit` snapshot of the container. A container that changed since its check is left. Workspace volumes stay. It needs no swarmfile, takes the engagement lock and - checks the engagement's Docker daemon. Until containers get workspace - volumes, it refuses every container. `tests/runtime_retire.sh` (in + checks the engagement's Docker daemon. `tests/runtime_retire.sh` (in `tests/test.sh --runtime` and `--all`) runs it on real stopped containers. - **Feature: recovery snapshots (building block).** diff --git a/README.md b/README.md index 8c0bc36..0221a5e 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Host runtime dir (mode 700) ~/project/ โ”€โ”€ git clone โ”€โ”€> upstream.git (rw) --bare mirrors/*.git (ro) | - | docker volumes + | bind mounts | .-----------.----------+-----------. | | | | @@ -61,6 +61,8 @@ Host runtime dir (mode 700) v v /workspace/ /workspace/ (agent-work) (agent-work) + volume volume + -ws- -ws- ``` All containers mount the same bare repo. Each runs @@ -68,6 +70,15 @@ All containers mount the same bare repo. Each runs run one agent session, push. When one agent pushes, others see the changes on the next fetch. +Each container's `/workspace` is a Docker volume of its own, so its +files outlive the container. One engagement goes from `start` to +`harvest.sh`, which merges the agents' work into the checkout, then to +`./launch.sh retire`, which removes the stopped containers once nothing +lives only in them, and `./launch.sh cleanup`, which copies everything +Git kept in the volumes, the bare repo and the mirrors into the +checkout as recovery refs before it deletes them. The next `start` +needs a runtime directory cleaned up that way. + The runtime directory is private to the user who runs the swarm, `~/.local/state/swarm-core/` by default (see [USAGE.md](USAGE.md#runtime-directory)). @@ -89,6 +100,10 @@ SWARM_CONFIG=swarm.json ./launch.sh wait # Open a human-guided session from a named agent profile: SWARM_CONFIG=swarm.json ./launch.sh interactive hunter +# Once the work is harvested (wait does it), delete the engagement; +# what Git had in it stays in the checkout as recovery refs: +./launch.sh cleanup + # Or place swarm.json in your repo root and launch. ./launch.sh start --dashboard ``` diff --git a/USAGE.md b/USAGE.md index ba7c9a7..fb46737 100644 --- a/USAGE.md +++ b/USAGE.md @@ -8,6 +8,9 @@ SWARM_CONFIG=swarm.json ./launch.sh start --dashboard # Later, after agents are running or have exited: SWARM_CONFIG=swarm.json ./launch.sh wait + +# Then delete the engagement before the next start: +./launch.sh cleanup ``` All configuration lives in the swarmfile (JSON). Place a @@ -53,8 +56,11 @@ names, and existing bind mounts still refer to their original paths. ## Commands ```bash -./launch.sh start [--dashboard] # Launch numbered agents. -./launch.sh stop # Stop all agents. +./launch.sh start [--dashboard] # Start an engagement: launch + # numbered agents. +./launch.sh stop [--numbered] # Stop this engagement's + # containers (or only the + # numbered agents). ./launch.sh status # Show containers. ./launch.sh logs N # Tail agent N logs. ./launch.sh wait # Wait for already-started @@ -66,6 +72,10 @@ names, and existing bind mounts still refer to their original paths. ./launch.sh interactive hunter # Start a human-guided driver # session from agents[].name. ./launch.sh shell --agent hunter # Start a shell in that profile. +./launch.sh retire # Remove stopped containers. +./launch.sh cleanup # Keep what Git had as recovery + # refs, then delete the + # engagement. ``` ## Environment variables @@ -89,12 +99,14 @@ Credentials stay as env vars (not in shell history). | `SWARM_RUNTIME_DIR` | | See [Runtime directory](#runtime-directory). | | `SWARM_PROJECT` | | See [Project identity](#project-identity). | | `SWARM_ENGAGEMENT_ID` | | See [Engagements](#engagements). | +| `SWARM_ENGAGEMENT_TOKEN` | | See [Engagements](#engagements). | | `SWARM_LOCK_FD` | | See [Engagement lock](#engagement-lock). | | `SWARM_SKIP_DEP_CHECK` | | Set to `1` to silence dependency version warnings. | | `SWARM_ACTIVITY_TIMEOUT` | `0` | Seconds of logfile silence before the in-container watchdog SIGTERMs the agent CLI's process group. `0` disables. See [Activity watchdog](#activity-watchdog). | | `SWARM_ACTIVITY_POLL` | `10` | Watchdog mtime-poll interval, in seconds. Rarely needs tuning. | | `SWARM_WATCHDOG_GRACE` | `10` | Grace window between watchdog SIGTERM and SIGKILL. Rarely needs tuning. | | `SWARM_STOP_TIMEOUT` | `60` | Seconds `./launch.sh stop` passes to `docker stop -t`, so the harness's SIGTERM trap has time to ship any in-flight local commits via `_session_end_push` before SIGKILL hits. See [Stopping the swarm](#stopping-the-swarm). | +| `SWARM_WAIT_POLL` | `10` | Seconds between the checks of `./launch.sh wait`. | Per-group credentials (`api_key`, `auth_token`, `base_url`) are set in the swarmfile. Use `$VAR` references to pull @@ -225,9 +237,10 @@ gap. Both rarely need adjustment outside the test suite. ### Stopping the swarm -`./launch.sh stop` sends each numbered agent and post-process -container `docker stop -t 60`. The 60 s grace gives the -harness's SIGTERM trap time to ship +`./launch.sh stop` sends every running container of the engagement +(numbered agents, post-process, interactive sessions) +`docker stop -t 60`; `--numbered` stops only the numbered agents. +The 60 s grace gives the harness's SIGTERM trap time to ship any in-flight local commits (in-place rebase โ†’ scratch worktree โ†’ `agent-parked/*` salvage) before docker forces SIGKILL. Without this window, commits the agent made during @@ -255,6 +268,18 @@ The harness handles SIGTERM and SIGINT identically; an `emergency shutdown complete` indicates the trap fired and the emergency push attempt completed. +`stop` takes no lock, so it can stop a post-process that holds one. +It acts on the engagement the state file records when it starts, +through its token: it lists the running containers that carry it, +reads each one's labels again right before it stops it, and stops it +by ID, so a container of a later engagement is never stopped. It lists +them again after each pass, so agents a start is still launching are +stopped too, and fails when they keep appearing, when one of them +still runs afterwards, or when Docker cannot tell. It refuses an engagement of an earlier release (see +[Engagements of earlier releases](#engagements-of-earlier-releases)), +and stops nothing while Docker talks to another daemon than the +engagement's. + ### Extra Docker arguments Pass arbitrary flags to every `docker run` invocation via the @@ -273,6 +298,11 @@ This is useful for mounting the host Docker socket, adding devices or capabilities, setting network modes, or passing any other flags that the harness does not manage natively. +What the engine sets comes after `docker_args`, and Docker takes the +last value of a flag given twice, so an entry there cannot replace it: +the workspace volume at `/workspace`, the engagement labels, +`--restart no`, `--rm=false` and the container's name. + Docker's `-e` flag accepts two forms: `-e VAR=value` passes a literal value, and `-e VAR` (no `=value`) inherits `VAR` from the caller's environment at `launch.sh start` time, omitting it @@ -374,8 +404,8 @@ do not count toward numbered-agent completion. | `1`-`9` | Logs for agent N. | | `P` | Post-process logs (the `P` row), if it exists. | | `h` | Harvest results. | -| `s` | Stop numbered agents and post-process. | -| `p` | Start post-process after confirmation; replaces an exited run. | +| `s` | Stop the containers of the engagement shown (`./launch.sh stop`). | +| `p` | After confirmation, stop the numbered agents and start post-process; refuses while an earlier post-process container is there. | The Model column appends the agent's reasoning effort as a parenthesised letter: `(h)` high, `(m)` medium or minimal, `(l)` low, @@ -517,6 +547,13 @@ or automatically via `./launch.sh wait` after numbered agents have already been started. `./launch.sh wait` does not launch numbered agents. +`post-process` refuses while numbered agents of the engagement run: +stop them (`./launch.sh stop --numbered`) or wait for them first. A +post-process container is never replaced: while +`-agent-post` is there, `post-process` refuses. Stop it first +if it runs, then remove it with `./launch.sh retire --post-process`; +its workspace volume stays for cleanup. + The post-process agent clones the same bare repo, sees all commits on `agent-work`, runs its prompt, and pushes. @@ -1007,6 +1044,7 @@ Each project keeps its host-side state in one directory: |------|---------| | `upstream.git` | Bare repo that containers mount at `/upstream`. | | `mirrors/.git` | Submodule mirrors (`/mirrors`, read-only). | +| `upstream.git.new`, `mirrors/.git.new` | A bare repo or mirror being cloned, renamed once ready. | | `state.json` | State file for a standalone dashboard. | | `engagement.lock` | Lock that lifecycle commands hold. | @@ -1022,6 +1060,14 @@ elsewhere, and set it the same way for every command of a run `costs.sh`): each one resolves the directory on its own. `launch.sh start` prints the directory it used. +Cleanup has to be able to work there, so `launch.sh` refuses, before +it creates anything, a runtime directory that is `/` or your home +directory, that holds or lies inside the checkout or its git dir, that +is not a directory of yours, or whose path has a comma, a quote, a tab +or a newline. Only an earlier release can have made one in such a +place (see +[Engagements of earlier releases](#engagements-of-earlier-releases)). + `launch.sh` creates the directory with mode 700 and refuses one that is a symlink, belongs to another user, has any group or other permission bits, or (on macOS) has an ACL entry that allows access; @@ -1063,6 +1109,23 @@ because an older release created it (see stopping a container checks its labels again and acts on its ID. `harvest.sh` only looks into containers when the Docker daemon answers. +### Workspace volumes + +Each container's `/workspace` is a Docker volume of its own, +`-ws-<8 hex digits>`, created right before the +container. The volume carries the `swarm-core.project`, +`swarm-core.checkout`, `swarm-core.engagement` and +`swarm-core.engagement-token` labels, plus `swarm-core.container` (its +container's name), `swarm-core.kind` (`agent` for numbered agents and +post-process, `interactive` for a human session), `swarm-core.context` +(`full`, `slim` or `none`) and a random `swarm-core.nonce` that +`launch.sh` reads back after creating the volume. A launch fails before +`docker run` when the volume cannot be created or does not carry those +labels. Containers carry the engagement labels too. Removing a stopped +container (`./launch.sh retire`) leaves its volume, and the files in +it; `./launch.sh cleanup` copies what Git keeps there into the checkout, +then deletes the volumes. + ### Engagements An engagement is one life of the bare repo. It begins when `start` @@ -1082,6 +1145,57 @@ and `refs/swarm/base` point at it, the state file records it as branches are not copied, so branches from an earlier engagement never look like new agent work. +The state file also records the Docker daemon (`docker info` ID) the +engagement runs on, and a token of 16 random hex digits that +`launch.sh` makes for each new engagement (an engagement id can come +from the caller and repeat, the token cannot); containers and volumes +carry it in `swarm-core.engagement-token`. `post-process` and +`interactive` joining an engagement, `stop`, `wait`, `retire` and +`cleanup` refuse when Docker talks to another daemon (another Docker +context or `DOCKER_HOST`): switch back first. Each command uses the +Docker context it found when it started for all its Docker calls, even +if another terminal switches it meanwhile. + +`SWARM_ENGAGEMENT_TOKEN` binds a command to one engagement: `stop`, +`post-process`, `interactive` and `harvest.sh` refuse when it is set +and the state file records another token, and `post-process` and +`interactive` then never create an engagement. `wait` reads the token +once, refuses to go on when the engagement changed while it waited +(a cleanup and a new start in another terminal), and passes the token +to the post-process or harvest it runs; the dashboard passes it to each +`launch.sh` it runs. + +A new engagement starts only on a clean runtime directory. `start`, and +`post-process` or `interactive` when there is no bare repo to join, +refuse while the runtime directory holds a bare repo, an +`upstream.git.new`, `mirrors/` or a remnant of a cleanup, while a +container or a workspace volume of this checkout exists, or while an +interactive session is open. The error lists what it found and, when +the bare repo holds agent work the checkout lacks, what to harvest. +End the engagement first: stop the agents, run `harvest.sh`, then +`./launch.sh cleanup`. Nothing is removed by force. The engagement is +recorded before anything is made, and a bare repo or a mirror is cloned +to `.new` and renamed once ready, so a run stopped half way +leaves at most a `.new`, which `cleanup` preserves and deletes. A +mistake in the swarmfile (an unknown driver or context mode) or a +failed image build stops `start` before the engagement is created. + +### Engagements of earlier releases + +An earlier release's engagement records no Docker daemon in the state +file, and its containers keep their files in their own layers (or in +volumes `docker_args` made); Docker may remove some when they stop +(`--rm` in `docker_args`). This release does not take such an +engagement over: `start`, `post-process`, `interactive`, `stop`, +`wait`, `retire`, `cleanup` and `harvest.sh` refuse it and change +nothing, and only `status` and `logs` work on it. The same holds for +what an engagement leaves (a bare repo, mirrors, containers, volumes) +when the state file records none, and for a runtime directory in a +place this release refuses. Finish it with the release that made it: +check that release out, stop the agents, run `harvest.sh`, then follow +that release's cleanup steps in its USAGE.md, and come back to this +one. + ### Engagement lock `start`, `post-process`, `interactive`, `chat`, `shell` and @@ -1127,44 +1241,27 @@ that the descriptor is open on that lock file and locks through it: Older releases kept this state in `/tmp/-upstream.git`, `/tmp/-mirror-*.git` and `/tmp/-swarm.env`, where -any local user could write to it. It is never reused. While -`/tmp/-upstream.git` exists and belongs to you, `start`, -`post-process`, `interactive`, `chat`, `shell` and `harvest.sh` stop -and print the commands below. +any local user could write to it. It is never reused, and this release +changes nothing there. While `/tmp/-upstream.git` exists and +belongs to you, `start`, `post-process`, `interactive`, `chat`, `shell` +and `harvest.sh` stop. Containers that an older release started carry no labels, so `launch.sh` refuses every command while they exist, `stop` included, before it gets to the `/tmp` check (see [Project identity](#project-identity)). Their agents push to the old -bare repo. Stop the containers that the error lists with the grace -`./launch.sh stop` gives, so the agents can still push their last -commits: - -```bash -docker stop -t "${SWARM_STOP_TIMEOUT:-60}" ... -``` - -To keep agent commits from the old bare repo, fetch its branches and -review them before merging anything: +bare repo. Finish that engagement with the release that made it, as +for [engagements of earlier releases](#engagements-of-earlier-releases): +check it out, stop the agents, harvest, then follow its USAGE.md. To +look at the agents' commits in the old bare repo meanwhile, fetch its +branches and review them before merging anything (this changes nothing +there): ```bash git fetch /tmp/-upstream.git \ '+refs/heads/*:refs/remotes/legacy-swarm/*' ``` -Then remove the old containers with `docker rm`, and the old state -with the commands below. The agents wrote part of the old bare repo as -the container's user, and on Linux a plain `rm -rf` cannot delete those -files, so remove the bare repo through Docker. The error about `/tmp` -lists the exact paths; this form also works in zsh when there are no -mirrors: - -```bash -docker run --rm -v /tmp:/tmp alpine rm -rf /tmp/-upstream.git -rm -rf /tmp/-swarm.env -find -H /tmp -maxdepth 1 -name '-mirror-*.git' -exec rm -rf {} + -``` - ## Retiring containers `./launch.sh retire` removes this checkout's stopped engine containers; @@ -1212,9 +1309,11 @@ reaches `docker diff` altered, so such a name refuses too). The error says what to copy out first. Removal goes by ID, without `-f`, and leaves any container that changed since it was checked. Run `costs.sh` before retire if you need the containers' stats. retire -needs no swarmfile; it takes the engagement lock and refuses when -Docker talks to another daemon than the engagement's (an engagement -that recorded none needs `SWARM_DOCKER_DAEMON=`). +needs no swarmfile; it takes the engagement lock, refuses an engagement +of an earlier release (see +[Engagements of earlier releases](#engagements-of-earlier-releases)), +and removes nothing while Docker talks to another daemon than the +engagement's. ## Cleanup @@ -1226,16 +1325,21 @@ every object, stashes, unfinished operations, and every file of the git dir) into the checkout as recovery refs, and checks them there. ```bash -./launch.sh cleanup [--discard-uncommitted] +./launch.sh cleanup [--discard-uncommitted] [--allow-unharvested] ``` It stops, and deletes nothing, when -- the bare repo holds agent work the checkout does not have yet - (`agent-work`, `swarm/*`, `agent-parked/*`): harvest first; -- a container cannot be retired (see [Retiring - containers](#retiring-containers)), or an interactive session is open - or starting; +- the bare repo holds agent work that HEAD's history does not have yet + (`agent-work`, `swarm/*`, `agent-parked/*`): harvest first. When it + never can (`harvest.sh` refuses a bare repo that does not descend + from HEAD, after a rewritten upstream say), take what you need by + hand and pass `--allow-unharvested`, which keeps that work as + recovery refs only. A read of the bare repo's refs that fails part of + the way refuses either way; +- a container still runs (stop the agents first) or cannot be retired + (see [Retiring containers](#retiring-containers)), or an interactive + session is open or starting; - a workspace, the bare repo or a mirror holds work outside Git's history (uncommitted or ignored files, files too large to copy): commit or copy it out, or pass `--discard-uncommitted`, which drops @@ -1251,7 +1355,19 @@ It stops, and deletes nothing, when quote, a tab or a newline in its path; - something is mounted inside the runtime directory, or at a workspace volume's directory on this host (cleanup never deletes through a - mount). + mount); +- the engagement is an earlier release's (see + [Engagements of earlier releases](#engagements-of-earlier-releases)). + +It checks the workspace volumes while the stopped containers are still +there, so none of these refusals touches a container; then it retires +the containers and checks everything again before it copies and +deletes. A later failure (the copy into the checkout, `git fsck`) +leaves the containers retired and every repository location in place, +so save the containers' logs and stats first if you need them +(`./launch.sh logs N`, `costs.sh`). A `.new` that a run cut off while +it cloned the bare repo or a mirror is copied as it is (every file) +and deleted too. The recovery refs are `refs/swarm-recovery/e//s/...`, `` being the engagement id in hex; diff --git a/dashboard.sh b/dashboard.sh index 7c42588..432fa52 100755 --- a/dashboard.sh +++ b/dashboard.sh @@ -88,7 +88,6 @@ own_ref() { } identity_ok || exit 1 -CHECKOUT_PATH="$(swarm_checkout_path "$REPO_ROOT")" IMAGE_NAME="${PROJECT}-agent" START_TIME=$(date +%s) @@ -119,8 +118,11 @@ state_value() { else "n" end' } +# The engagement the dashboard shows, by its token (empty for none); +# s and p act on it alone. Set by load_state. +DASH_TOKEN="" load_state() { - local state title config agents summary label + local state title config agents summary label token if [ ! -f "$STATE_FILE" ] || ! runtime_ok; then return 1 fi @@ -134,6 +136,7 @@ load_state() { agents=$(state_value "$state" num_agents) || return 1 summary=$(state_value "$state" model_summary clean) || return 1 label=$(state_value "$state" config_label clean) || return 1 + token=$(state_value "$state" engagement_token) || return 1 case "$title" in y*) title="${title#y}"; SWARM_TITLE="${title%x}" ;; esac case "$config" in y*) config="${config#y}"; SWARM_CONFIG="${config%x}" ;; @@ -147,6 +150,12 @@ load_state() { case "$label" in y*) label="${label#y}"; SWARM_CONFIG_LABEL="${label%x}" ;; esac + # Unlike the fields above, a token the state no longer has (the + # engagement was cleaned up) is no engagement. + case "$token" in + y*) token="${token#y}"; DASH_TOKEN="${token%x}" ;; + *) DASH_TOKEN="" ;; + esac } load_state || true @@ -473,17 +482,6 @@ interactive_container_names() { | sort || true } -# Stop this checkout's container NAME by ID after checking its label -# again; another checkout's container under that name is left alone. -stop_own_container() { - local name="$1" id - id=$(swarm_own_container_id "$name" "$CHECKOUT_PATH" "$PROJECT") \ - || return 0 - if [ -n "$id" ] && docker stop "$id" >/dev/null 2>&1; then - echo " stopped ${name}" - fi -} - read_interactive_state_value() { local name="$1" key="$2" local tmpf="${DASH_TMPDIR}/interactive-${name}.state" @@ -948,15 +946,17 @@ while true; do s|S) leave_alt_screen identity_ok || exit 1 - echo "--- Stopping all agents ---" - for i in $(seq 1 "$NUM_AGENTS"); do - stop_own_container "${IMAGE_NAME}-${i}" - done - while IFS= read -r int_name; do - [ -n "$int_name" ] || continue - stop_own_container "$int_name" - done < <(interactive_container_names) - stop_own_container "${IMAGE_NAME}-post" + # launch.sh stop, with its checks, on the engagement + # shown (its token, from the last refresh): never a + # later one, and nothing when none is shown. + _token="$DASH_TOKEN" + if [ -z "$_token" ]; then + echo "(no engagement shown; nothing stopped)" + else + SWARM_CONFIG="$CONFIG_FILE" \ + SWARM_ENGAGEMENT_TOKEN="$_token" \ + "$SWARM_DIR/launch.sh" stop || echo "(stop failed)" + fi echo "" read -rp "Press Enter to return to dashboard..." _ enter_alt_screen @@ -983,7 +983,12 @@ while true; do continue fi if post_process_container_exists; then - echo "(previous post-process container will be replaced)" + echo "(${_pp_name} is still there; retire it first:" \ + "./launch.sh retire --post-process)" + echo "" + read -rp "Press Enter to return to dashboard..." _ + enter_alt_screen + continue fi _pp_prompt="Start post-processing now? [y/N] " read -r -p "$_pp_prompt" _pp_confirm || _pp_confirm="" @@ -998,14 +1003,29 @@ while true; do ;; esac identity_ok || exit 1 + # Both steps on the engagement shown (its token, from + # the last refresh): a cleanup and a new start since make + # them refuse, and nothing runs when none is shown. + _token="$DASH_TOKEN" + if [ -z "$_token" ]; then + echo "(no engagement shown; post-processing not started)" + echo "" + read -rp "Press Enter to return to dashboard..." _ + enter_alt_screen + continue + fi echo "--- Stopping agents ---" - for i in $(seq 1 "$NUM_AGENTS"); do - stop_own_container "${IMAGE_NAME}-${i}" >/dev/null - done - echo "--- Starting post-processing ---" - SWARM_CONFIG="$CONFIG_FILE" \ - "$SWARM_DIR/launch.sh" post-process || \ - echo "(post-processing failed)" + if ! SWARM_CONFIG="$CONFIG_FILE" \ + SWARM_ENGAGEMENT_TOKEN="$_token" \ + "$SWARM_DIR/launch.sh" stop --numbered >/dev/null; then + echo "(agents not stopped; post-processing not started)" + else + echo "--- Starting post-processing ---" + SWARM_CONFIG="$CONFIG_FILE" \ + SWARM_ENGAGEMENT_TOKEN="$_token" \ + "$SWARM_DIR/launch.sh" post-process || \ + echo "(post-processing failed)" + fi echo "" read -rp "Press Enter to return to dashboard..." _ enter_alt_screen diff --git a/harvest.sh b/harvest.sh index bd0fc42..5ecf2c2 100755 --- a/harvest.sh +++ b/harvest.sh @@ -90,6 +90,18 @@ swarm_runtime_check "$RUNTIME_DIR" || exit 1 swarm_runtime_owner_check "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || exit 1 swarm_lock "$RUNTIME_DIR" || exit 1 swarm_cleanup_pending_check "$RUNTIME_DIR/state.json" || exit 1 +# An earlier release's engagement is finished with that release; these +# checks read the state file only, so they hold without Docker too. +# This engine records the engagement before it makes the bare repo, so +# a bare repo with none recorded is an earlier release's as well. +swarm_legacy_refuse "$RUNTIME_DIR" "$RUNTIME_DIR/state.json" || exit 1 +_engaged=$(swarm_engagement_recorded "$RUNTIME_DIR/state.json") || exit 1 +if [ -z "$_engaged" ]; then + swarm_legacy_explain "$RUNTIME_DIR" \ + "a bare repo with no engagement recorded" + exit 1 +fi +swarm_engagement_token_check "$RUNTIME_DIR/state.json" || exit 1 swarm_engagement_env_check || exit 1 swarm_engagement_join "$RUNTIME_DIR/state.json" || exit 1 CHECKOUT_PATH="$(swarm_checkout_path "$REPO_ROOT")" @@ -160,9 +172,11 @@ for branch in "${HARVEST_BRANCHES[@]}"; do echo "the bare repo still has the old ancestry. Merging" >&2 echo "would re-introduce removed commits." >&2 echo "" >&2 - echo "Resolve by wiping and recreating the bare repo:" >&2 - echo " $(swarm_bare_rm_command "$RUNTIME_DIR")" >&2 - echo " ./launch.sh ... # recreates the bare from current HEAD" >&2 + echo "Take what you need from it by hand (git log ${remote_ref}," >&2 + echo "cherry-pick), then end the engagement with" >&2 + echo " ./launch.sh cleanup --allow-unharvested" >&2 + echo "which keeps everything in the bare repo as recovery refs," >&2 + echo "and start again from the current HEAD." >&2 git remote remove "$REMOTE_NAME" exit 1 fi diff --git a/launch.sh b/launch.sh index 9cfa0ac..4c230a7 100755 --- a/launch.sh +++ b/launch.sh @@ -67,6 +67,8 @@ source "$SWARM_DIR/lib/check-deps.sh" check_deps git jq docker # shellcheck source=lib/project.sh source "$SWARM_DIR/lib/project.sh" +# shellcheck source=lib/preserve.sh +source "$SWARM_DIR/lib/preserve.sh" # retire and cleanup read and write Git data: none of the caller's GIT_* # variables may point them at another repository or inject settings @@ -97,6 +99,12 @@ IMAGE_NAME="${PROJECT}-agent" # Every command acts on this project's containers or runtime directory, # so none runs when the directory belongs to another checkout. swarm_runtime_verify "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || exit 1 +# Every Docker call of this run, from this first listing on, goes to one +# daemon, even if another terminal switches the context meanwhile. +if ! swarm_docker_pin; then + echo "ERROR: cannot read the current Docker context." >&2 + exit 1 +fi swarm_containers_verify "$PROJECT" "$REPO_ROOT" || exit 1 CHECKOUT_PATH="$(swarm_checkout_path "$REPO_ROOT")" CHECKOUT_LABEL="${SWARM_CHECKOUT_LABEL}=${CHECKOUT_PATH}" @@ -116,21 +124,6 @@ expand_env_ref() { fi } -# Docker containers may create files owned by a different UID inside -# bind-mounted host directories. Plain rm -rf fails without root. -# Use a throwaway Alpine container (Docker is already required) so -# we never need sudo/su -c. -rm_docker_dir() { - local dir="$1" - [ -d "$dir" ] || return 0 - local parent base - parent="$(dirname "$dir")" - base="$(basename "$dir")" - docker run --rm -v "${parent}:${parent}" alpine \ - rm -rf "${parent}/${base}" 2>/dev/null \ - || rm -rf "$dir" 2>/dev/null || true -} - # Compute the comma-separated SWARM_AGENTS build-arg from a config: # the union of every agent group's driver and, when post_process is # present and not null, the post-process driver. @@ -181,35 +174,6 @@ build_image() { -f "$SWARM_DIR/Dockerfile" "$SWARM_DIR" } -# Remove this checkout's container NAME before the name is reused. The -# label is checked again here, since another checkout can create the -# container after the check at the top, and removal goes by ID; when a -# foreign container takes the name afterwards, docker run fails on it. -remove_own_container() { - local id - id=$(swarm_own_container_id "$1" "$CHECKOUT_PATH" "$PROJECT") || exit 1 - if [ -n "$id" ]; then - docker rm -f "$id" >/dev/null - fi -} - -# Stop this checkout's container NAME by ID, after checking its label -# again; a container of another checkout under that name is skipped. -stop_own_container() { - local name="$1" timeout="$2" id - if ! id=$(swarm_own_container_id "$name" "$CHECKOUT_PATH" \ - "$PROJECT"); then - echo " ${name} skipped: not this checkout's container" - return 0 - fi - if [ -n "$id" ] \ - && docker stop -t "$timeout" "$id" >/dev/null 2>&1; then - echo " stopped ${name}" - else - echo " ${name} not running" - fi -} - # Print the state of this checkout's container NAME: its Docker status, # "not found" when there is none, or "not ours" when a container of # another checkout or project holds the name. @@ -228,23 +192,6 @@ own_container_state() { || printf 'not found' } -# Stop when an interactive session may still be about to start its -# container: Docker makes the mounts at start, so the bare repo and the -# mirrors must stay as they are while such a session lives. -refuse_while_interactive_pending() { - local live - live=$(swarm_pending_live "$RUNTIME_DIR") || exit 1 - [ -n "$live" ] || return 0 - echo "ERROR: interactive sessions of this checkout are open" \ - "(containers below); the bare repo and the mirrors stay" \ - "until they end:" >&2 - printf '%s\n' "$live" | sed 's/^/ /' >&2 - echo "A container that was created but never ran also counts; once" \ - "you are sure it will not start, remove it with" \ - "docker rm ." >&2 - exit 1 -} - # Lifecycle commands call this before they touch the bare repo or the # mirrors: stop on state that older releases left in /tmp (named after # the directory, as they did), create or check the private runtime @@ -256,12 +203,57 @@ refuse_while_interactive_pending() { prepare_runtime() { swarm_refuse_legacy_state "$(swarm_project_id "$PROJECT_RAW")" \ || exit 1 + # Only where cleanup can work later; checked before anything is made. + swarm_runtime_location_ok "$RUNTIME_DIR" "$REPO_ROOT" || exit 1 swarm_runtime_init "$RUNTIME_DIR" || exit 1 swarm_lock "$RUNTIME_DIR" || exit 1 swarm_cleanup_pending_check "$STATE_FILE" || exit 1 + swarm_legacy_refuse "$RUNTIME_DIR" "$STATE_FILE" || exit 1 + swarm_engagement_token_check "$STATE_FILE" || exit 1 swarm_runtime_claim "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || exit 1 } +# Stop unless the runtime directory holds nothing of an engagement: no +# bare repo, no upstream.git.new, no mirrors/, no remnant of a cleanup, +# no container or workspace volume of this checkout, no interactive +# session open or starting. A new engagement starts only there, after +# ./launch.sh cleanup has preserved and removed the last one. Called +# under the lock and the pin, by start, and by post-process and +# interactive when they would create the engagement. +require_clean_runtime() { + local left live engaged rc=0 tmp + left=$(swarm_runtime_leftovers "$RUNTIME_DIR" "$PROJECT" "$REPO_ROOT" \ + "$CHECKOUT_PATH") || exit 1 + live=$(swarm_pending_live "$RUNTIME_DIR") || exit 1 + if [ -n "$live" ]; then + left="${left:+${left}$'\n'} interactive sessions: $(printf '%s\n' \ + "$live" | tr '\n' ' ')" + fi + [ -n "$left" ] || return 0 + engaged=$(swarm_engagement_recorded "$STATE_FILE") || exit 1 + if [ -z "$engaged" ]; then + swarm_legacy_explain "$RUNTIME_DIR" \ + "what is left of an engagement it never recorded" + printf '%s\n' "$left" >&2 + exit 1 + fi + echo "ERROR: a new engagement starts only on a clean runtime" \ + "directory; ${RUNTIME_DIR} still holds engagement" \ + "${engaged}:" >&2 + printf '%s\n' "$left" >&2 + if [ -d "$BARE_REPO" ] && swarm_tmp_file tmp e3; then + swarm_preservation_check "$BARE_REPO" "$REPO_ROOT" > "$tmp" || rc=$? + if [ "$rc" -eq 1 ]; then + swarm_preservation_explain "$BARE_REPO" < "$tmp" + fi + rm -f "$tmp" + fi + echo "Stop the agents (./launch.sh stop), run harvest.sh, then" \ + "./launch.sh cleanup: it keeps everything Git had there as" \ + "recovery refs before it deletes anything." >&2 + exit 1 +} + # Record the run in the state file, so a standalone dashboard can show # its title, swarmfile and agent summary. The file is JSON that # readers parse with jq; nothing sources it. @@ -293,31 +285,40 @@ write_state_file() { } create_bare_repo() { - local label="${1:-bare repo}" base new="${BARE_REPO}.new" + local label="${1:-bare repo}" base daemon new="${BARE_REPO}.new" echo "--- Creating ${label} ---" # The engagement starts from the commit the checkout has now. base=$(git -C "$REPO_ROOT" rev-parse --verify --quiet 'HEAD^{commit}') \ || { echo "ERROR: ${REPO_ROOT} has no commit to start from." >&2 exit 1; } - rm_docker_dir "$BARE_REPO" - rm_docker_dir "$new" - if [ -e "$BARE_REPO" ] || [ -e "$new" ]; then - echo "ERROR: cannot remove the old ${BARE_REPO}." >&2 + # The clean-runtime check came first; nothing is ever replaced. + if [ -e "$BARE_REPO" ] || [ -L "$BARE_REPO" ] \ + || [ -e "$new" ] || [ -L "$new" ]; then + echo "ERROR: ${BARE_REPO} or ${new} is there already." >&2 exit 1 fi + if ! daemon=$(swarm_docker_daemon_id); then + echo "ERROR: cannot read the Docker daemon id." >&2 + exit 1 + fi + # The engagement is recorded first: a run stopped from here on + # leaves at most upstream.git.new under it, which cleanup preserves + # and removes. + begin_engagement "$base" "$daemon" || { + echo "ERROR: cannot record the engagement in ${STATE_FILE}." >&2 + exit 1 + } # A local clone hardlinks object files by default, and the chmod # below would then make the checkout's own objects writable for # every user. Copy them instead, with any objects the checkout - # borrows through alternates. - git clone --bare --no-hardlinks --dissociate "$REPO_ROOT" "$new" - # Later commands join any bare repo they find, so the clone gets its - # name only once it is ready and the engagement is recorded. A - # failure leaves at most the .new directory, which nothing joins. - if ! prepare_bare_repo "$new" "$base" \ - || ! begin_engagement "$base" \ + # borrows through alternates. Later commands join any bare repo + # they find, so the clone gets its name only once it is ready. + if ! git clone --bare --no-hardlinks --dissociate "$REPO_ROOT" "$new" \ + || ! prepare_bare_repo "$new" "$base" \ || ! mv "$new" "$BARE_REPO"; then echo "ERROR: cannot prepare ${BARE_REPO}." >&2 - rm_docker_dir "$new" + # This user's files, which no container has seen yet. + rm -rf "$new" exit 1 fi } @@ -354,28 +355,66 @@ prepare_bare_repo() { # refs/swarm/base, so the state file keeps the copy that later checks # of the agents' branches can trust. Returns 1 if either fails. begin_engagement() { - local base="$1" id + local base="$1" daemon="$2" id token id=$(swarm_engagement_new) || return 1 + token=$(swarm_random_hex 8) || return 1 swarm_state_update "$STATE_FILE" \ '.engagement_id = $id | .engagement_started_at = $at - | .base_commit = $base' \ + | .base_commit = $base | .docker_daemon = $daemon + | .engagement_token = $token' \ --arg id "$id" --arg at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --arg base "$base" \ + --arg base "$base" --arg daemon "$daemon" --arg token "$token" \ || return 1 echo "Engagement: ${id}" } +# Fill WORKSPACE_ARGS with a new workspace volume for container NAME +# (kind KIND, context CONTEXT) and the run arguments every engine +# container gets after docker_args, so that an entry there cannot +# replace them: the engagement and its token as labels, no restart +# policy and no auto-remove (retire and cleanup refuse both). Needs the +# engagement recorded, so it runs once the bare repo was created or +# joined. +workspace_args() { + local name="$1" kind="$2" context="$3" engagement token vol + engagement=$(swarm_engagement_recorded "$STATE_FILE") || exit 1 + token=$(swarm_engagement_token "$STATE_FILE") || exit 1 + if [ -z "$engagement" ] || [ -z "$token" ]; then + echo "ERROR: ${STATE_FILE} records no engagement." >&2 + exit 1 + fi + vol=$(swarm_workspace_volume_create "$name" "$PROJECT" \ + "$CHECKOUT_PATH" "$engagement" "$token" "$kind" "$context") \ + || exit 1 + WORKSPACE_ARGS=( + --mount "type=volume,src=${vol},dst=/workspace" + --label "${SWARM_ENGAGEMENT_LABEL}=${engagement}" + --label "${SWARM_TOKEN_LABEL}=${token}" + --restart no --rm=false + ) +} + # A session that creates the bare repo mirrors the submodules too; one # that joins keeps the mirrors the engagement has, which other sessions -# may be about to mount, and only adds the ones that are missing. +# may be about to mount, and only adds the ones that are missing. The +# image is built once the checks passed, and before an engagement is +# created, so a failed build leaves the runtime directory clean. ensure_bare_repo_for_interactive() { if [ ! -d "$BARE_REPO" ]; then - refuse_while_interactive_pending + # A token names an engagement to join, never a new one. + if [ -n "${SWARM_ENGAGEMENT_TOKEN:-}" ]; then + echo "ERROR: the engagement this session was started for" \ + "has no bare repo any more." >&2 + exit 1 + fi + require_clean_runtime + build_image create_bare_repo "bare repo for interactive session" mirror_submodules return fi swarm_engagement_join "$STATE_FILE" || exit 1 + swarm_engagement_daemon_check "$STATE_FILE" || exit 1 local bare_head local_head bare_head=$(git -C "$BARE_REPO" rev-parse --verify --quiet \ @@ -386,16 +425,22 @@ ensure_bare_repo_for_interactive() { echo "ERROR: ${BARE_REPO} is stale (agent-work" \ "${bare_head:0:7} behind local HEAD" \ "${local_head:0:7})." >&2 - echo " Remove it to start from current HEAD:" >&2 - echo " $(swarm_bare_rm_command "$RUNTIME_DIR")" >&2 + echo " To start from the current HEAD, end this" \ + "engagement: stop the agents, run harvest.sh, then" \ + "./launch.sh cleanup, which keeps everything as recovery" \ + "refs." >&2 exit 1 fi + build_image mirror_submodules missing } -# Mirror every submodule into MIRROR_DIR, replacing old mirrors. With -# "missing", only create the mirrors that do not exist yet and leave the -# others, which a session may be about to mount. +# Mirror every submodule into MIRROR_DIR. A new engagement finds none +# (clean runtime), and one that is there refuses; with "missing", only +# the mirrors that do not exist yet are made and the others, which a +# session may be about to mount, stay. Each is cloned to .git.new +# and renamed once ready, so a run stopped half way leaves only a .new, +# which cleanup preserves and removes. mirror_submodules() { local only_missing="${1:-}" cd "$REPO_ROOT" @@ -405,15 +450,22 @@ mirror_submodules() { local safe_name mirror safe_name="${name//\//_}" mirror="${MIRROR_DIR}/${safe_name}.git" - if [ "$only_missing" = missing ] && [ -d "$mirror" ]; then - continue + if [ -e "$mirror" ] || [ -L "$mirror" ]; then + [ "$only_missing" = missing ] && continue + echo "ERROR: ${mirror} is there already." >&2 + exit 1 + fi + if [ -e "$mirror.new" ] || [ -L "$mirror.new" ]; then + echo "ERROR: ${mirror}.new is left from a run stopped half" \ + "way; ./launch.sh cleanup removes it." >&2 + exit 1 fi - rm_docker_dir "$mirror" echo "--- Mirroring submodule: ${name} ---" # Copy objects, as create_bare_repo does, so the chmod below # cannot reach the checkout's .git/modules through hardlinks. - git clone --bare --no-hardlinks --dissociate "$gitdir" "$mirror" - chmod -R a+rwX "$mirror" + git clone --bare --no-hardlinks --dissociate "$gitdir" "$mirror.new" + chmod -R a+rwX "$mirror.new" + mv "$mirror.new" "$mirror" done } @@ -501,6 +553,23 @@ swarm_tmp_file() { printf -v "$1" '%s' "$_tmp" } +# Copy the state file to a temporary file and set the variable VAR to +# its path, for stop and wait, which hold no lock: every check they make +# then reads one version of it, whatever another terminal does +# meanwhile (the state file is replaced by a rename, so a copy holds one +# whole version). With no state file, VAR names a file that does not +# exist. +state_snapshot() { + local _snap + swarm_tmp_file _snap state || return 1 + if [ -e "$STATE_FILE" ] || [ -L "$STATE_FILE" ]; then + cp "$STATE_FILE" "$_snap" || return 1 + else + rm -f "$_snap" || return 1 + fi + printf -v "$1" '%s' "$_snap" +} + CONFIG_FILE="${SWARM_CONFIG:-}" if [ -z "$CONFIG_FILE" ] && [ -f "$REPO_ROOT/swarm.json" ]; then CONFIG_FILE="$REPO_ROOT/swarm.json" @@ -707,7 +776,9 @@ HELP fi safe_profile="$(swarm_project_id "$profile_label")" safe_profile="${safe_profile:0:32}" - short_id="$(date -u +%Y%m%d%H%M%S)-${RANDOM}" + # Random enough that two sessions never meet (a record is never + # written over another; see swarm_pending_write). + short_id="$(date -u +%Y%m%d%H%M%S)-$(swarm_random_hex 4)" || exit 1 branch="swarm/${SWARM_RUN_HASH}/interactive-${safe_profile}-${short_id}" name="${IMAGE_NAME}-interactive-${safe_profile}-${short_id}" @@ -724,6 +795,13 @@ HELP agent_auth=$(jq -r '.auth // empty' "$profile_file") agent_context=$(jq -r '.context // empty' "$profile_file") agent_context="${agent_context:-full}" + case "$agent_context" in + full|slim|none) ;; + *) + echo "ERROR: unknown context mode: ${agent_context}" \ + "(full, slim or none)." >&2 + exit 1 ;; + esac agent_prompt=$(jq -r '.prompt // empty' "$profile_file") agent_auth_token=$(jq -r '.auth_token // empty' "$profile_file") agent_auth_token="$(expand_env_ref "$agent_auth_token")" @@ -754,7 +832,6 @@ HELP prepare_runtime ensure_bare_repo_for_interactive - build_image MIRROR_ARGS=() mirror_volume_args @@ -777,7 +854,9 @@ HELP [ -n "${!_pv:-}" ] && EXTRA_ENV+=(-e "${_pv}=${!_pv}") done - remove_own_container "$name" + # Its workspace volume, made under the lock before the session is + # recorded. + workspace_args "$name" interactive "$agent_context" # The session can last hours and only pushes through git, so it runs # without the lock. It is recorded under the lock with this @@ -785,7 +864,7 @@ HELP # docker run, which makes the container and its mounts. Until this # process ends, or its container after a detach, start and # post-process leave the bare repo and the mirrors alone - # (refuse_while_interactive_pending). One Docker context serves + # (require_clean_runtime). One Docker context serves # the daemon id and docker run, even if another terminal switches # it meanwhile. local daemon @@ -807,6 +886,7 @@ HELP "${MIRROR_ARGS[@]+"${MIRROR_ARGS[@]}"}" \ "${SIGNING_KEY_ARGS[@]+"${SIGNING_KEY_ARGS[@]}"}" \ "${DOCKER_EXTRA_ARGS[@]+"${DOCKER_EXTRA_ARGS[@]}"}" \ + "${WORKSPACE_ARGS[@]}" \ --name "$name" \ --label "${SWARM_SESSION_LABEL}=${name}" \ "${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"}" \ @@ -863,48 +943,6 @@ cmd_start() { fi done <<< "$group_prompts" - prepare_runtime - - # Refuse to overwrite a bare repo that diverges from local HEAD. - # Distinguish two directions so the message leads with the remediation - # that actually works. The `--is-ancestor` check runs in the local - # repo, not in the bare: in the "local ahead" case LOCAL_HEAD is not - # in the bare's object db, so running the check in the bare would - # always return non-zero and collapse the stale case into the - # unharvested branch. Running in local works across all three cases - # because bare's objects are either (a) still in local (inherited at - # clone time, so ancestry is resolvable -> stale) or (b) only in - # bare (agent-produced post-clone, so BARE_HEAD errors out here -> - # unharvested, which also catches the truly-divergent case). - if [ -d "$BARE_REPO" ]; then - BARE_HEAD=$(git -C "$BARE_REPO" rev-parse --verify --quiet refs/heads/agent-work 2>/dev/null || true) - LOCAL_HEAD=$(git rev-parse HEAD 2>/dev/null || true) - if [ -n "$BARE_HEAD" ] && [ "$BARE_HEAD" != "$LOCAL_HEAD" ]; then - if git merge-base --is-ancestor "$BARE_HEAD" HEAD 2>/dev/null; then - echo "ERROR: ${BARE_REPO} is stale (agent-work" \ - "${BARE_HEAD:0:7} behind local HEAD" \ - "${LOCAL_HEAD:0:7})." >&2 - echo " Remove it to start a fresh run from" \ - "current HEAD:" >&2 - echo " $(swarm_bare_rm_command "$RUNTIME_DIR")" >&2 - else - echo "ERROR: ${BARE_REPO} has unharvested agent" \ - "commits (agent-work ${BARE_HEAD:0:7} vs local" \ - "HEAD ${LOCAL_HEAD:0:7})." >&2 - echo " Run harvest.sh first, or if you've" \ - "already integrated those commits:" >&2 - echo " $(swarm_bare_rm_command "$RUNTIME_DIR")" >&2 - fi - exit 1 - fi - fi - - refuse_while_interactive_pending - create_bare_repo "bare repo" - - # Mirror each submodule so containers can init without network. - mirror_submodules - # Build per-agent config (model|base_url|api_key|effort|auth|context|prompt|auth_token|tag|driver per line). # Uses pipe delimiter because bash IFS=$'\t' collapses consecutive tabs. swarm_tmp_file AGENTS_CFG agents || exit 1 @@ -913,10 +951,15 @@ cmd_start() { [.model, (.base_url // ""), (.api_key // ""), (.effort // ""), (.auth // ""), (.context // ""), (.prompt // ""), (.auth_token // ""), (.tag // $dt // ""), (.driver // $dd // "")] | join("|")' \ "$CONFIG_FILE" > "$AGENTS_CFG" - # Preflight: validate all referenced drivers exist before - # spending time on image build and container startup. - local _bad_drivers="" _checked_drivers=" " - while IFS='|' read -r _ _ _ _ _ _ _ _ _ _drv; do + # Preflight: validate all referenced drivers and context modes + # before anything is created, so that a mistake in the swarmfile + # leaves the runtime directory clean. + local _bad_drivers="" _checked_drivers=" " _ctx _bad_contexts="" + while IFS='|' read -r _ _ _ _ _ _ctx _ _ _ _drv; do + case "${_ctx:-full}" in + full|slim|none) ;; + *) _bad_contexts+=" - ${_ctx}\n" ;; + esac _drv="${_drv:-${SWARM_DRIVER_DEFAULT}}" [[ "$_checked_drivers" == *" $_drv "* ]] && continue _checked_drivers+="$_drv " @@ -929,8 +972,23 @@ cmd_start() { echo "Available drivers: $(available_drivers)" >&2 exit 1 fi + if [ -n "$_bad_contexts" ]; then + printf "ERROR: unknown context mode(s):\n%b" "$_bad_contexts" >&2 + echo "Context modes: full, slim, none." >&2 + exit 1 + fi + prepare_runtime + + # A new engagement, only once the last one was cleaned up. The + # image is built before it is created, so a failed build leaves the + # runtime directory clean. + require_clean_runtime build_image + create_bare_repo "bare repo" + + # Mirror each submodule so containers can init without network. + mirror_submodules # Mirror volume mounts, shared across all containers. MIRROR_ARGS=() @@ -940,11 +998,11 @@ cmd_start() { while IFS='|' read -r agent_model agent_base_url agent_api_key agent_effort agent_auth agent_context agent_prompt agent_auth_token agent_tag agent_driver; do AGENT_IDX=$((AGENT_IDX + 1)) NAME="${IMAGE_NAME}-${AGENT_IDX}" - remove_own_container "$NAME" agent_api_key="$(expand_env_ref "$agent_api_key")" agent_auth_token="$(expand_env_ref "$agent_auth_token")" agent_tag="$(expand_env_ref "$agent_tag")" agent_context="${agent_context:-full}" + workspace_args "$NAME" agent "$agent_context" agent_driver="${agent_driver:-${SWARM_DRIVER_DEFAULT}}" # Source the driver to access agent_docker_env. @@ -985,12 +1043,15 @@ cmd_start() { read -r price_input price_output price_cached <<< "$_price" fi + # The workspace, its labels, no restart policy, no auto-remove + # and the name come after docker_args: the last value wins. docker run -d \ - --name "$NAME" \ -v "${BARE_REPO}:/upstream:rw" \ "${MIRROR_ARGS[@]+"${MIRROR_ARGS[@]}"}" \ "${SIGNING_KEY_ARGS[@]+"${SIGNING_KEY_ARGS[@]}"}" \ "${DOCKER_EXTRA_ARGS[@]+"${DOCKER_EXTRA_ARGS[@]}"}" \ + "${WORKSPACE_ARGS[@]}" \ + --name "$NAME" \ "${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"}" \ --label "$CHECKOUT_LABEL" \ --label "$PROJECT_LABEL" \ @@ -1045,21 +1106,110 @@ cmd_stop() { # default that docker ships with cuts the emergency push # mid-rebase on a busy bare repo. Override via env: # SWARM_STOP_TIMEOUT=120 ./launch.sh stop - local stop_timeout="${SWARM_STOP_TIMEOUT:-60}" - swarm_cleanup_pending_check "$STATE_FILE" || exit 1 - swarm_engagement_join "$STATE_FILE" || exit 1 + local stop_timeout="${SWARM_STOP_TIMEOUT:-60}" numbered=false + local snap token list id info name rest state failed=0 pass=0 found + local labels + while [ $# -gt 0 ]; do + case "$1" in + --numbered) numbered=true; shift ;; + *) echo "ERROR: unknown stop option: $1" >&2; exit 1 ;; + esac + done + # stop takes no lock (post-process holds it for its whole run, and + # stop must be able to stop it). It acts on the engagement the + # state file records now, through its token: every check reads the + # same copy of the state, and a container of a later engagement, + # under a reused name or engagement id, is never stopped. + state_snapshot snap || exit 1 + swarm_cleanup_pending_check "$snap" || exit 1 + swarm_legacy_refuse "$RUNTIME_DIR" "$snap" || exit 1 + swarm_engagement_join "$snap" || exit 1 + # A token names an engagement: one that is gone, or replaced, refuses + # (also when none is recorded any more). + swarm_engagement_token_check "$snap" || exit 1 + token=$(swarm_engagement_token "$snap") || exit 1 + if [ -z "$token" ]; then + echo "No engagement recorded; nothing to stop." + return 0 + fi + swarm_engagement_daemon_check "$snap" || exit 1 echo "--- Stopping agents (grace ${stop_timeout}s) ---" - for i in $(seq 1 "$NUM_AGENTS"); do - stop_own_container "${IMAGE_NAME}-${i}" "$stop_timeout" + # What a container's labels must read before it is stopped. + labels="{{.Name}}|{{index .Config.Labels \"${SWARM_PROJECT_LABEL}\"}}" + labels+="|{{index .Config.Labels \"${SWARM_CHECKOUT_LABEL}\"}}" + labels+="|{{index .Config.Labels \"${SWARM_TOKEN_LABEL}\"}}" + # A start that recorded the engagement can still be launching its + # agents (it holds the lock; stop does not), so the listing is taken + # again after each pass, until one finds nothing more to stop: up to + # three passes that stop, then one that only checks. + while :; do + pass=$((pass + 1)) + if ! list=$(docker ps -q \ + --filter "label=${SWARM_TOKEN_LABEL}=${token}" \ + --filter "label=${SWARM_PROJECT_LABEL}=${PROJECT}" \ + 2>/dev/null); then + echo "ERROR: cannot list Docker containers." >&2 + exit 1 + fi + found=false + for id in $list; do + # Its labels again, right before it is stopped by ID. One gone + # since the listing needs nothing; one Docker cannot describe + # fails the stop. + if ! info=$(docker container inspect -f "$labels" "$id" \ + 2>/dev/null); then + if ! swarm_container_gone "$id"; then + echo "ERROR: cannot inspect container ${id}." >&2 + failed=1 + fi + continue + fi + name="${info%%|*}" + name="${name#/}" + if [ "$info" != "/${name}|${PROJECT}|${CHECKOUT_PATH}|${token}" ] + then + echo " ${name} skipped: not this engagement's container" + continue + fi + if $numbered; then + rest="${name#"${IMAGE_NAME}"-}" + if [ "$rest" = "$name" ] || [ -z "$rest" ] \ + || [ -n "${rest//[0-9]/}" ]; then + continue + fi + fi + if [ "$pass" -gt 3 ]; then + echo "ERROR: containers of this engagement keep" \ + "starting; run stop again once start is done." >&2 + exit 1 + fi + found=true + if docker stop -t "$stop_timeout" "$id" >/dev/null 2>&1; then + echo " stopped ${name}" + continue + fi + # docker stop fails too for one that stopped or went meanwhile; + # one that still runs, or that Docker cannot tell, fails the + # stop, so that callers (the dashboard's p) do not go on. + if ! state=$(docker container inspect -f '{{.State.Running}}' \ + "$id" 2>/dev/null); then + state=unknown + swarm_container_gone "$id" && state=gone + fi + case "$state" in + false|gone) echo " ${name} not running" ;; + *) + echo "ERROR: cannot stop ${name}." >&2 + failed=1 ;; + esac + done + if ! $found || [ "$failed" -ne 0 ]; then + break + fi done - while IFS= read -r NAME; do - [ -n "$NAME" ] || continue - stop_own_container "$NAME" "$stop_timeout" - done < <(swarm_interactive_container_names "$PROJECT" 2>/dev/null \ - | sort || true) - stop_own_container "${IMAGE_NAME}-post" "$stop_timeout" # The state file stays: it records which checkout owns the runtime # directory, and later phases read it. + [ "$failed" -eq 0 ] || exit 1 } cmd_logs() { @@ -1106,35 +1256,101 @@ cmd_status() { | sort || true) } +# Print " " for the numbered agents in LISTING, docker +# ps lines " ". No read loop over a here-string: Bash 3.2 +# makes a temporary file for it, which can fail to be made, and a read +# that fails ends a loop like the end of the list; either would count +# running agents as done. Names and states hold no glob characters. +wait_count() { + local line name state rest running=0 exited=0 IFS=$'\n' + for line in $1; do + name="${line%% *}" + state="${line##* }" + # The numbered agents only. + rest="${name#"${IMAGE_NAME}"-}" + if [ "$rest" = "$name" ] || [ -z "$rest" ] \ + || [ -n "${rest//[0-9]/}" ]; then + continue + fi + case "$state" in + running|restarting|created|paused) running=$((running + 1)) ;; + *) exited=$((exited + 1)) ;; + esac + done + printf '%s %s\n' "$running" "$exited" +} + +# Print " " for the numbered agents of the engagement +# with token TOKEN, as Docker lists them now. +wait_agents() { + local states + if ! states=$(docker ps -a \ + --filter "label=${SWARM_TOKEN_LABEL}=${1}" \ + --filter "label=${SWARM_PROJECT_LABEL}=${PROJECT}" \ + --format '{{.Names}} {{.State}}' 2>/dev/null); then + echo "" >&2 + echo "ERROR: cannot list Docker containers." >&2 + return 1 + fi + wait_count "$states" +} + cmd_wait() { - swarm_engagement_join "$STATE_FILE" || exit 1 + local snap token own_lock=true + # wait holds no lock while it polls: every check reads the same copy + # of the state, and the engagement waited for is the one its token + # names there. + state_snapshot snap || exit 1 + swarm_cleanup_pending_check "$snap" || exit 1 + swarm_legacy_refuse "$RUNTIME_DIR" "$snap" || exit 1 + swarm_engagement_join "$snap" || exit 1 + swarm_engagement_token_check "$snap" || exit 1 + token=$(swarm_engagement_token "$snap") || exit 1 + if [ -z "$token" ]; then + echo "ERROR: ${STATE_FILE} records no engagement to wait for." >&2 + exit 1 + fi + swarm_engagement_daemon_check "$snap" || exit 1 + # A lock the caller holds stays held throughout. + [ -z "${SWARM_LOCK_FD:-}" ] || own_lock=false echo "--- Waiting for all agents to finish ---" + local counts running exited while true; do - sleep 10 - local all_done=true running=0 exited=0 - for i in $(seq 1 "$NUM_AGENTS"); do - local state - state=$(own_container_state "${IMAGE_NAME}-${i}") - case "$state" in - running) running=$((running + 1)); all_done=false ;; - exited) exited=$((exited + 1)) ;; - "not ours") - echo "" >&2 - echo "ERROR: ${IMAGE_NAME}-${i} is not this" \ - "checkout's container." >&2 - exit 1 ;; - esac - done - + sleep "${SWARM_WAIT_POLL:-10}" + counts=$(wait_agents "$token") || exit 1 + running="${counts% *}" + exited="${counts#* }" printf "\r %d running, %d exited " "$running" "$exited" - - if $all_done; then + [ "$running" -eq 0 ] || continue + + # What comes next acts on the engagement waited for, and on + # nothing else: under the lock, the state file must still record + # its token (a cleanup and a new start in another terminal + # meanwhile would have changed it), and none of its agents may + # run. A start records the engagement before it launches the + # agents, and holds the lock until they run: agents it launched + # meanwhile are waited for too. + swarm_lock "$RUNTIME_DIR" || exit 1 + swarm_cleanup_pending_check "$STATE_FILE" || exit 1 + if [ "$(swarm_engagement_token "$STATE_FILE")" != "$token" ]; then + echo "" >&2 + echo "ERROR: the engagement changed while waiting (cleaned" \ + "up, or started again); nothing more is done." >&2 + exit 1 + fi + counts=$(wait_agents "$token") || exit 1 + if [ "${counts% *}" -eq 0 ]; then echo "" echo "All agents finished." break fi + if $own_lock; then + swarm_unlock + fi done + SWARM_ENGAGEMENT_TOKEN="$token" + export SWARM_ENGAGEMENT_TOKEN local pp_prompt pp_prompt=$(jq -r '.post_process.prompt // empty' "$CONFIG_FILE") @@ -1181,20 +1397,64 @@ cmd_post_process() { echo "ERROR: post_process.prompt is not set in ${CONFIG_FILE}." >&2 exit 1 fi + if [ ! -f "$SWARM_DIR/lib/drivers/${pp_driver}.sh" ]; then + echo "ERROR: unknown driver: ${pp_driver}" >&2 + echo "Available drivers: $(available_drivers)" >&2 + exit 1 + fi prepare_runtime + local NAME="${IMAGE_NAME}-post" pp_state if [ ! -d "$BARE_REPO" ]; then - refuse_while_interactive_pending + # A token names an engagement to join, never a new one. + if [ -n "${SWARM_ENGAGEMENT_TOKEN:-}" ]; then + echo "ERROR: the engagement post-process was started for" \ + "has no bare repo any more." >&2 + exit 1 + fi + # The image is built before the engagement is created, so a + # failed build leaves the runtime directory clean. + require_clean_runtime + build_image create_bare_repo "bare repo for post-process" mirror_submodules else swarm_engagement_join "$STATE_FILE" || exit 1 + swarm_engagement_daemon_check "$STATE_FILE" || exit 1 + # The numbered agents must be done: post-process works on what + # they pushed, and under the lock no start can be launching more + # of them (the dashboard's p stops them first, without the lock). + local pp_token pp_counts + pp_token=$(swarm_engagement_token "$STATE_FILE") || exit 1 + pp_counts=$(wait_agents "$pp_token") || exit 1 + if [ "${pp_counts% *}" -ne 0 ]; then + echo "ERROR: numbered agents of this engagement still run;" \ + "stop them (./launch.sh stop --numbered) or wait for" \ + "them (./launch.sh wait) first." >&2 + exit 1 + fi + # An earlier post-process container is never replaced: its + # files stay until retire takes it (they are in its volume). + # On a clean runtime there is none of this checkout's. + pp_state=$(own_container_state "$NAME") + case "$pp_state" in + "not found") ;; + running|restarting|paused|created) + echo "ERROR: ${NAME} is ${pp_state}; stop it first" \ + "(./launch.sh stop)." >&2 + exit 1 ;; + "not ours") + echo "ERROR: ${NAME} is not this checkout's container." >&2 + exit 1 ;; + *) + echo "ERROR: ${NAME} is there (${pp_state}); retire it" \ + "first: ./launch.sh retire --post-process." >&2 + exit 1 ;; + esac + build_image fi - build_image - - local NAME="${IMAGE_NAME}-post" - remove_own_container "$NAME" + workspace_args "$NAME" agent full # Build mirror volume args from existing mirrors. local MIRROR_ARGS=() @@ -1231,12 +1491,15 @@ cmd_post_process() { echo "--- Starting post-processing (${pp_model}) ---" local pp_id + # As for the agents: the workspace, its labels, the policies and the + # name come after docker_args. pp_id=$(docker run -d \ - --name "$NAME" \ -v "${BARE_REPO}:/upstream:rw" \ "${MIRROR_ARGS[@]+"${MIRROR_ARGS[@]}"}" \ "${SIGNING_KEY_ARGS[@]+"${SIGNING_KEY_ARGS[@]}"}" \ "${DOCKER_EXTRA_ARGS[@]+"${DOCKER_EXTRA_ARGS[@]}"}" \ + "${WORKSPACE_ARGS[@]}" \ + --name "$NAME" \ "${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"}" \ --label "$CHECKOUT_LABEL" \ --label "$PROJECT_LABEL" \ @@ -1311,7 +1574,7 @@ case "${1:-start}" in exec "$SWARM_DIR/dashboard.sh" fi ;; - stop) cmd_stop ;; + stop) shift; cmd_stop "$@" ;; logs) cmd_logs "${2:-1}" ;; status) cmd_status ;; wait) cmd_wait ;; diff --git a/lib/cleanup.sh b/lib/cleanup.sh index d87f690..3e47159 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -15,6 +15,7 @@ # STATE_FILE, BARE_REPO, MIRROR_DIR and IMAGE_NAME. CLEANUP_DISCARD=false +CLEANUP_ALLOW_UNHARVESTED=false CLEANUP_HELPER_LABEL="swarm-core.cleanup" # Print STRING as lowercase hex: any engagement id becomes a valid ref @@ -60,76 +61,11 @@ cleanup_lines() { printf '%s' "$((count + 0))" } -# Succeed when the runtime directory is safe to delete things in: a -# directory of this user's, and not /, $HOME, the checkout, its git -# dir or anything around them. +# Succeed when the runtime directory is safe to delete things in (the +# rules every lifecycle command applies to it; see +# swarm_runtime_location_ok). cleanup_runtime_sane() { - local rt home root common gitdir x owner - if [ -L "$RUNTIME_DIR" ] || [ ! -d "$RUNTIME_DIR" ]; then - echo "ERROR: ${RUNTIME_DIR} is not a directory." >&2 - return 1 - fi - rt=$(cd -P "$RUNTIME_DIR" && pwd -P) || return 1 - owner=$(swarm_owner_uid "$rt") || return 1 - if [ "$owner" != "$(id -u)" ]; then - echo "ERROR: ${RUNTIME_DIR} does not belong to this user." >&2 - return 1 - fi - home=$(cd -P "${HOME:-/}" 2>/dev/null && pwd -P) || home=/ - # The checkout, its common git dir and its own git dir (a linked - # worktree's lives apart from the common one). - if ! root=$(cd -P "$REPO_ROOT" 2>/dev/null && pwd -P) \ - || ! common=$(swarm_recovery_git -C "$REPO_ROOT" rev-parse \ - --git-common-dir 2>/dev/null) \ - || ! gitdir=$(swarm_recovery_git -C "$REPO_ROOT" rev-parse \ - --absolute-git-dir 2>/dev/null); then - echo "ERROR: cannot resolve the git dir of ${REPO_ROOT}." >&2 - return 1 - fi - case "$common" in - /*) ;; - *) common="$REPO_ROOT/$common" ;; - esac - if ! common=$(cd -P "$common" 2>/dev/null && pwd -P) \ - || ! gitdir=$(cd -P "$gitdir" 2>/dev/null && pwd -P); then - echo "ERROR: cannot resolve the git dir of ${REPO_ROOT}." >&2 - return 1 - fi - if [ "$rt" = / ] || [ "$rt" = "$home" ]; then - echo "ERROR: refusing to clean up in ${RUNTIME_DIR}." >&2 - return 1 - fi - # The Docker helper mounts it with --mount, whose value is - # comma-separated, and cleanup's lists are tab-separated lines. - case "$RUNTIME_DIR" in - *[,\"]*|*$'\n'*|*$'\t'*) - echo "ERROR: ${RUNTIME_DIR} holds a comma, a quote, a tab or a" \ - "newline; cleanup cannot work with it." >&2 - return 1 ;; - esac - for x in "$root" "$common" "$gitdir"; do - case "$x/" in - "$rt"/*) - echo "ERROR: ${RUNTIME_DIR} holds the checkout or its git" \ - "dir (${x})." >&2 - return 1 ;; - esac - case "$rt/" in - "$x"/*) - echo "ERROR: ${RUNTIME_DIR} is inside the checkout or its" \ - "git dir (${x})." >&2 - return 1 ;; - esac - done -} - -# Print the numeric owner of PATH. -swarm_owner_uid() { - if stat -c %u / >/dev/null 2>&1; then - stat -c %u "$1" - else - stat -f %u "$1" - fi + swarm_runtime_location_ok "$RUNTIME_DIR" "$REPO_ROOT" } # Wait for, then remove, the helper containers a killed cleanup of this @@ -196,6 +132,43 @@ cleanup_no_pending() { fi } +# Fail, naming them, while containers of this checkout run (or are +# paused or restarting), or Docker cannot say they do not: their +# workspaces still change, so the scans would judge them too early, and +# retire refuses them anyway. Stopping the agents is the step to take, +# not dropping what a scan finds. +cleanup_none_running() { + local list line name id st live="" IFS + list=$(swarm_own_containers "$PROJECT" "$REPO_ROOT") || { + echo "ERROR: cannot list Docker containers." >&2 + return 1 + } + # " " lines; no read loop over a here-string (see + # wait_count in launch.sh). Names and ids hold no glob characters. + IFS=$'\n' + for line in $list; do + name="${line%% *}" + id="${line#* }" + if ! st=$(docker container inspect "$id" 2>/dev/null \ + | jq -r '.[0].State.Status // empty' 2>/dev/null); then + # One gone since the listing is no longer in the way; one + # Docker cannot describe might still run. + swarm_container_gone "$id" && continue + st="" + fi + case "$st" in + exited|created|dead) ;; + *) live="${live} ${name} (${st:-state unknown})"$'\n' ;; + esac + done + if [ -n "$live" ]; then + echo "ERROR: containers of this engagement still run, or Docker" \ + "cannot tell; stop them first (./launch.sh stop):" >&2 + printf '%s' "$live" >&2 + return 1 + fi +} + # Fail unless every snapshot the checkpoint records is still intact in # the checkout: each source's " " list and each run's sources # blob, from the first run on. A remnant deleted part of the way can @@ -274,6 +247,15 @@ cleanup_sources() { case "$name" in *.removing) continue ;; *$'\t'*|*$'\n'*) odd="${odd} ${e}"$'\n'; continue ;; + *.new) + # A mirror cut off while it was made. + if [ -L "$e" ]; then + echo "ERROR: ${e} is a link; cleanup will not" \ + "follow it." >&2 + return 1 + fi + printf 'new\t%s\n' "$e" >> "$out" || return 1 + continue ;; esac if [ ! -L "$e" ] && [ -d "$e" ] && [ -f "$e/HEAD" ] \ && { [ -d "$e/objects" ] || [ -f "$e/commondir" ]; }; then @@ -421,7 +403,7 @@ cleanup_scan() { SWARM_SCAN_DOCKER_ARGS=() sed 's/^/ /' "$out.err" >&2 return 1 ;; - removing) + removing|new) if [ -f "$path/HEAD" ] \ && { [ -d "$path/objects" ] \ || [ -f "$path/commondir" ]; } \ @@ -601,23 +583,88 @@ cleanup_delete_host() { cleanup_record "$id" '.state = "deleted"' } +# Examine every location cleanup would delete, into directory OUT: the +# list (OUT/sources), the storage check, every scan (OUT/scan.) with +# the policy applied, and the list of what was scanned (OUT/scanned, +# "\t\t\t"). With FREE set, a volume must also +# have no container left that uses it. Sets CLEANUP_TOTAL to the +# number of locations. Fails on any refusal, having changed nothing. +cleanup_examine() { + local out="$1" free="${2:-}" sources targets=() kind path mp seen=0 + local total n=0 dir mode + sources="$out/sources" + cleanup_sources "$sources" || return 1 + + # Storage: the checkout must keep what it gets by itself. + cleanup_no_mounts "$RUNTIME_DIR" || return 1 + while IFS="$(printf '\t')" read -r -u 3 kind path; do + seen=$((seen + 1)) + if [ "$kind" = volume ]; then + if [ -n "$free" ]; then + cleanup_volume_free "$path" || return 1 + fi + mp=$(cleanup_volume_mountpoint "$path") || { + echo "ERROR: cannot inspect volume ${path}." >&2 + return 1 + } + if [ -n "$mp" ]; then + cleanup_no_mounts "$mp" at || return 1 + targets+=("$mp") + fi + else + targets+=("$path") + fi + done 3< "$sources" < /dev/null || return 1 + total=$(cleanup_lines "$sources") || return 1 + if [ "$seen" -ne "$total" ]; then + echo "ERROR: checked the storage of ${seen} of the ${total}" \ + "locations to clean up." >&2 + return 1 + fi + swarm_checkout_storage_ok "$REPO_ROOT" "$RUNTIME_DIR" \ + "${targets[@]+"${targets[@]}"}" || return 1 + + # Scan every source and apply the policy before anything is copied. + : > "$out/scanned" || return 1 + while IFS="$(printf '\t')" read -r -u 3 kind path; do + n=$((n + 1)) + dir="$out/scan.$n" + mkdir "$dir" || return 1 + mode=$(cleanup_scan "$kind" "$path" "$dir") || { + echo "ERROR: cannot scan ${path}." >&2 + return 1 + } + cleanup_policy "$dir/report" "$path" || return 1 + printf '%s\t%s\t%s\t%s\n' "$n" "$kind" "$path" "$mode" \ + >> "$out/scanned" || return 1 + done 3< "$sources" < /dev/null || return 1 + if [ "$n" -ne "$total" ]; then + echo "ERROR: scanned ${n} of the ${total} locations to clean up." >&2 + return 1 + fi + CLEANUP_TOTAL="$total" +} + # The cleanup command; cmd_cleanup runs it with retire's cleanup and # traps in place. cleanup_run() { while [ $# -gt 0 ]; do case "$1" in --discard-uncommitted) CLEANUP_DISCARD=true; shift ;; + --allow-unharvested) CLEANUP_ALLOW_UNHARVESTED=true; shift ;; -h|--help) cat <&2; return 1 ;; @@ -634,24 +681,19 @@ HELP swarm_runtime_verify "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || return 1 swarm_engagement_join "$STATE_FILE" || return 1 cleanup_runtime_sane || return 1 - local engagement e + local engagement engagement=$(swarm_engagement_recorded "$STATE_FILE") || return 1 if [ -z "$engagement" ]; then - # Nothing recorded: a cleaned-up runtime directory, unless engine - # state is left without an engagement to file it under. - for e in "$BARE_REPO" "${BARE_REPO}.new" "$MIRROR_DIR" \ - "$RUNTIME_DIR"/*.removing; do - if [ -e "$e" ] || [ -L "$e" ]; then - echo "ERROR: ${STATE_FILE} records no engagement, but" \ - "${e} is there." >&2 - echo " See USAGE.md (Cleanup) for the manual steps." >&2 - return 1 - fi - done + # Nothing recorded: a cleaned-up runtime directory, unless + # something of an engagement is left without one to file it + # under (an earlier release's). + swarm_unrecorded_refuse "$RUNTIME_DIR" "$PROJECT" "$REPO_ROOT" \ + "$CHECKOUT_PATH" || return 1 echo "No engagement recorded in ${STATE_FILE}; nothing to clean up." return 0 fi - retire_require_daemon || return 1 + swarm_legacy_refuse "$RUNTIME_DIR" "$STATE_FILE" || return 1 + swarm_engagement_daemon_check "$STATE_FILE" || return 1 cleanup_settle_helpers || return 1 cleanup_no_pending || return 1 cleanup_check_evidence || return 1 @@ -665,14 +707,30 @@ HELP case "$rc" in 0) ;; 1) - swarm_preservation_explain "$BARE_REPO" \ - < "$RETIRE_SCRATCH/e3" - return 1 ;; + if [ "$CLEANUP_ALLOW_UNHARVESTED" = true ]; then + echo "Kept only as recovery refs (HEAD's history" \ + "lacks them):" + sed 's/^/ /' "$RETIRE_SCRATCH/e3" + else + swarm_preservation_explain "$BARE_REPO" \ + < "$RETIRE_SCRATCH/e3" + echo " Or, to keep that work only as recovery refs" \ + "(after a rewritten upstream, say), run cleanup" \ + "with --allow-unharvested." >&2 + return 1 + fi ;; *) return 1 ;; esac fi - # Every container goes first, through retire's checks. + # A first pass that only reads, while the stopped containers are + # still there: a refusal here (work outside Git's history, storage) + # leaves every container, its logs and its volume as they were. + cleanup_none_running || return 1 + mkdir "$RETIRE_SCRATCH/first" || return 1 + cleanup_examine "$RETIRE_SCRATCH/first" || return 1 + + # Every container goes next, through retire's checks. retire_run || { echo "Cleanup stops: every container must be retired first." >&2 return 1 @@ -685,56 +743,10 @@ HELP return 1 fi - local sources="$RETIRE_SCRATCH/sources" - cleanup_sources "$sources" || return 1 - - # Storage: the checkout must keep what it gets by itself. - local targets=() kind path mp seen=0 total - cleanup_no_mounts "$RUNTIME_DIR" || return 1 - while IFS="$(printf '\t')" read -r -u 3 kind path; do - seen=$((seen + 1)) - if [ "$kind" = volume ]; then - cleanup_volume_free "$path" || return 1 - mp=$(cleanup_volume_mountpoint "$path") || { - echo "ERROR: cannot inspect volume ${path}." >&2 - return 1 - } - if [ -n "$mp" ]; then - cleanup_no_mounts "$mp" at || return 1 - targets+=("$mp") - fi - else - targets+=("$path") - fi - done 3< "$sources" < /dev/null || return 1 - total=$(cleanup_lines "$sources") || return 1 - if [ "$seen" -ne "$total" ]; then - echo "ERROR: checked the storage of ${seen} of the ${total}" \ - "locations to clean up." >&2 - return 1 - fi - swarm_checkout_storage_ok "$REPO_ROOT" "$RUNTIME_DIR" \ - "${targets[@]+"${targets[@]}"}" || return 1 - - # Scan every source and apply the policy before anything is copied. - local n=0 dir mode - : > "$RETIRE_SCRATCH/scanned" || return 1 - while IFS="$(printf '\t')" read -r -u 3 kind path; do - n=$((n + 1)) - dir="$RETIRE_SCRATCH/scan.$n" - mkdir "$dir" || return 1 - mode=$(cleanup_scan "$kind" "$path" "$dir") || { - echo "ERROR: cannot scan ${path}." >&2 - return 1 - } - cleanup_policy "$dir/report" "$path" || return 1 - printf '%s\t%s\t%s\t%s\n' "$n" "$kind" "$path" "$mode" \ - >> "$RETIRE_SCRATCH/scanned" || return 1 - done 3< "$sources" < /dev/null || return 1 - if [ "$n" -ne "$total" ]; then - echo "ERROR: scanned ${n} of the ${total} locations to clean up." >&2 - return 1 - fi + # Then everything again, the volumes now free: nothing checked + # before retire is trusted after it. + cleanup_examine "$RETIRE_SCRATCH" free || return 1 + local total="$CLEANUP_TOTAL" n kind path mode dir # Preserve every source under a new snapshot, then verify the # checkout's whole object store once. @@ -847,7 +859,9 @@ HELP rc=0 swarm_preservation_check "$BARE_REPO" "$REPO_ROOT" \ > "$RETIRE_SCRATCH/e3" || rc=$? - if [ "$rc" -ne 0 ]; then + if [ "$rc" -ne 0 ] && { [ "$rc" -ne 1 ] \ + || [ "$CLEANUP_ALLOW_UNHARVESTED" != true ]; } + then echo "ERROR: ${BARE_REPO} changed during cleanup." >&2 return 1 fi diff --git a/lib/preserve-scan.sh b/lib/preserve-scan.sh index 51fe537..f182a65 100644 --- a/lib/preserve-scan.sh +++ b/lib/preserve-scan.sh @@ -117,9 +117,6 @@ oneline() { | sed 's/\\n$//' } - - - # Succeed when file $1 holds a NUL byte, or cannot be read. Git stops # reading a path at a NUL, while the shell's read drops it and carries # on, so the two would disagree on what such a file names. @@ -817,7 +814,6 @@ if [ "$MODE" = worktree ]; then done fi - # Engine files and the deletions the context mode makes: in the # workspace's own checkout ($2 = "."), where the engine writes those # paths, entry XY PATH ($1, $3) is not work. In submodules and diff --git a/lib/preserve.sh b/lib/preserve.sh index d437f51..3f23a47 100644 --- a/lib/preserve.sh +++ b/lib/preserve.sh @@ -78,7 +78,7 @@ swarm_tip_contained() { # missing (printed as " " lines), and 2 when the check could # not be made (said on stderr). Callers branch on the status. swarm_preservation_check() { - local bare="$1" root="$2" tmp err sha type ref status rc=0 + local bare="$1" root="$2" tmp err sha type ref status rc=0 seen=0 lines if [ -L "$bare" ] || [ ! -d "$bare" ] \ || [ "$(swarm_preserve_git -C "$bare" rev-parse \ --is-bare-repository 2>/dev/null)" != true ]; then @@ -101,6 +101,7 @@ swarm_preservation_check() { fi rm -f "$err" while read -r sha type ref; do + seen=$((seen + 1)) if [ "$type" != commit ]; then echo "ERROR: ${ref} in ${bare} does not point to a commit." >&2 rc=2 @@ -119,6 +120,15 @@ swarm_preservation_check() { break ;; esac done < "$tmp" + # A read that fails ends the loop like the end of the list: every + # ref must have been looked at (unless the loop broke off above). + if [ "$rc" -ne 2 ]; then + if ! lines=$(wc -l < "$tmp") || [ "$seen" -ne "$((lines + 0))" ] + then + echo "ERROR: cannot read the refs of ${bare} in full." >&2 + rc=2 + fi + fi rm -f "$tmp" return "$rc" } @@ -194,30 +204,6 @@ swarm_dev_ino() { stat "$SWARM_STAT_FORM" '%d:%i' "$1" } -# Print absolute PATH with the longest leading part of it that can be -# entered resolved (cd -P), and the rest as it is: PATH need not exist -# on this host, but a link on the way to it is still followed. Fails -# on a relative path. -swarm_path_resolve() { - local p="$1" head rest="" resolved - case "$p" in - /*) ;; - *) return 1 ;; - esac - while [ "$p" != / ] && [ "${p%/}" != "$p" ]; do - p="${p%/}" - done - head="$p" - while ! resolved=$(cd -P "$head" 2>/dev/null && pwd -P); do - rest="/${head##*/}${rest}" - head="${head%/*}" - [ -n "$head" ] || head=/ - done - [ "$resolved" != / ] || resolved="" - [ -n "${resolved}${rest}" ] || rest=/ - printf '%s%s\n' "$resolved" "$rest" -} - # Succeed when the checkout at ROOT keeps every object in its own # store, so objects imported there survive the deletion of TARGETS # (directories cleanup is about to delete; may be empty). Refuses a diff --git a/lib/project.sh b/lib/project.sh index 11c8957..e845f75 100644 --- a/lib/project.sh +++ b/lib/project.sh @@ -191,6 +191,111 @@ swarm_runtime_check() { fi } +# Print absolute PATH with the longest leading part of it that can be +# entered resolved (cd -P), and the rest as it is: PATH need not exist +# on this host, but a link on the way to it is still followed. Fails +# on a relative path. +swarm_path_resolve() { + local p="$1" head rest="" resolved + case "$p" in + /*) ;; + *) return 1 ;; + esac + while [ "$p" != / ] && [ "${p%/}" != "$p" ]; do + p="${p%/}" + done + head="$p" + while ! resolved=$(cd -P "$head" 2>/dev/null && pwd -P); do + rest="/${head##*/}${rest}" + head="${head%/*}" + [ -n "$head" ] || head=/ + done + [ "$resolved" != / ] || resolved="" + [ -n "${resolved}${rest}" ] || rest=/ + printf '%s%s\n' "$resolved" "$rest" +} + +# Print the numeric owner of PATH. +swarm_owner_uid() { + if stat -c %u / >/dev/null 2>&1; then + stat -c %u "$1" + else + stat -f %u "$1" + fi +} + +# Succeed when DIR (it need not exist yet) can be the runtime directory +# of the checkout at ROOT, in a place cleanup can work in: not /, not +# $HOME; not the checkout, its common git dir or its own git dir (a +# linked worktree's lives apart), nor inside or around any of them; no +# comma, quote, tab or newline in its path (the Docker helper's --mount +# value is comma-separated, cleanup's lists tab-separated); and, when it +# exists, a directory of this user's. +swarm_runtime_location_ok() { + local dir="$1" root="$2" rt home owner croot common gitdir x + case "$dir" in + *[,\"]*|*$'\n'*|*$'\t'*) + echo "ERROR: ${dir} holds a comma, a quote, a tab or a" \ + "newline; it cannot be the runtime directory." >&2 + return 1 ;; + esac + if [ -e "$dir" ] || [ -L "$dir" ]; then + if [ -L "$dir" ] || [ ! -d "$dir" ]; then + echo "ERROR: ${dir} is not a directory." >&2 + return 1 + fi + owner=$(swarm_owner_uid "$dir") || return 1 + if [ "$owner" != "$(id -u)" ]; then + echo "ERROR: ${dir} does not belong to this user." >&2 + return 1 + fi + fi + if ! rt=$(swarm_path_resolve "$dir"); then + echo "ERROR: ${dir} is not an absolute path." >&2 + return 1 + fi + home=$(cd -P "${HOME:-/}" 2>/dev/null && pwd -P) || home=/ + if [ "$rt" = / ] || [ "$rt" = "$home" ]; then + echo "ERROR: refusing ${dir} as the runtime directory." >&2 + return 1 + fi + # The checkout, its common git dir and its own git dir, resolved, + # with no GIT_* variable of the caller's steering git. + if ! croot=$(cd -P "$root" 2>/dev/null && pwd -P) \ + || ! common=$(cd "$root" && env -u GIT_DIR -u GIT_WORK_TREE \ + -u GIT_COMMON_DIR git rev-parse --git-common-dir \ + 2>/dev/null) \ + || ! gitdir=$(cd "$root" && env -u GIT_DIR -u GIT_WORK_TREE \ + -u GIT_COMMON_DIR git rev-parse --absolute-git-dir \ + 2>/dev/null); then + echo "ERROR: cannot resolve the git dir of ${root}." >&2 + return 1 + fi + case "$common" in + /*) ;; + *) common="$root/$common" ;; + esac + if ! common=$(cd -P "$common" 2>/dev/null && pwd -P) \ + || ! gitdir=$(cd -P "$gitdir" 2>/dev/null && pwd -P); then + echo "ERROR: cannot resolve the git dir of ${root}." >&2 + return 1 + fi + for x in "$croot" "$common" "$gitdir"; do + case "$x/" in + "$rt"/*) + echo "ERROR: ${dir} holds the checkout or its git dir" \ + "(${x})." >&2 + return 1 ;; + esac + case "$rt/" in + "$x"/*) + echo "ERROR: ${dir} is inside the checkout or its git" \ + "dir (${x})." >&2 + return 1 ;; + esac + done +} + # Create the runtime directory (owner-only) when missing, then check it. # The directory has to be private from the moment it exists, or another # user could add entries before any check runs. mkdir -m 700 gives that @@ -228,52 +333,27 @@ swarm_runtime_init() { # /tmp with world-writable modes, so any local user could have changed # its refs. That state is never adopted; print how to recover it. # Only a directory owned by the current user counts: anyone can create -# a path in /tmp, and only its owner could remove it. Agents pushed to -# it as the container's user, whose files rm cannot always delete on -# Linux, so the old state goes through Docker as in rm_docker_dir. +# a path in /tmp, and only its owner could remove it. swarm_refuse_legacy_state() { - local project="$1" legacy quoted path remove="" - local grace="${SWARM_STOP_TIMEOUT:-60}" + local project="$1" legacy quoted legacy="/tmp/${project}-upstream.git" if [ -L "$legacy" ] || [ ! -d "$legacy" ] || [ ! -O "$legacy" ]; then return 0 fi quoted=$(printf '%q' "$legacy") - # List the paths that exist instead of printing a glob, which zsh - # rejects when nothing matches. - for path in "$legacy" "/tmp/${project}-mirror-"*.git \ - "/tmp/${project}-swarm.env"; do - if [ -e "$path" ] || [ -L "$path" ]; then - remove+=" $(printf '%q' "$path")" - fi - done cat >&2 <... -To keep agent commits from it, fetch them and review them before you -merge anything: +not reuse the old one, since other local users could write to it; it +leaves the old state and its containers alone. Finish that engagement +with the release that made it (check it out, stop the agents, harvest, +then follow its USAGE.md), then come back to this one. To look at the +agents' commits meanwhile, fetch them (this changes nothing there): git fetch ${quoted} '+refs/heads/*:refs/remotes/legacy-swarm/*' -Then remove the old state. The agents wrote part of it as the -container's user, so remove it through Docker: - docker run --rm -v /tmp:/tmp alpine rm -rf${remove} EOF return 1 } -# Print a command that removes the bare repository of runtime directory -# DIR. The agents wrote files in it as the container's user, which rm -# run by the user cannot always delete on Linux, so rm runs as root in -# a container, as in rm_docker_dir. Only DIR is mounted. -swarm_bare_rm_command() { - printf 'docker run --rm -v %s:/runtime alpine' "$(printf '%q' "$1")" - printf ' rm -rf /runtime/upstream.git' -} - SWARM_STATE_SCHEMA="swarm-core.state/v1" # Print the state file as compact JSON when it holds exactly one @@ -428,6 +508,88 @@ SWARM_KIND_LABEL="swarm-core.kind" SWARM_CONTEXT_LABEL="swarm-core.context" # shellcheck disable=SC2034 SWARM_NONCE_LABEL="swarm-core.nonce" +# They also carry the token the engine made when it created the +# engagement: the engagement id can come from the caller +# (SWARM_ENGAGEMENT_ID) and so repeat, the token cannot. +# shellcheck disable=SC2034 +SWARM_TOKEN_LABEL="swarm-core.engagement-token" + +# Print 2*N random lowercase hex digits. +swarm_random_hex() { + local hex + hex=$(od -An -N"$1" -tx1 /dev/urandom 2>/dev/null | tr -d ' \n') \ + || return 1 + [ "${#hex}" -eq $(( $1 * 2 )) ] || return 1 + printf '%s' "$hex" +} + +# Create the workspace volume for container NAME of the engagement with +# id ENGAGEMENT and token TOKEN, and print its name. Each container gets +# a new volume, so a later container never starts from the files of an +# earlier one, and removing a stopped container leaves its /workspace +# behind for cleanup to preserve. The name has a random part; an +# existing volume of that name is an error, and since docker volume +# create also succeeds for an existing volume, the labels (with a +# nonce) are read back after creating it. +swarm_workspace_volume_create() { + local name="$1" project="$2" checkout="$3" engagement="$4" token="$5" + local kind="$6" context="$7" rand nonce vol err rc=0 labels + case "$kind" in + agent|interactive) ;; + *) echo "ERROR: unknown workspace kind: ${kind}" >&2; return 1 ;; + esac + case "$context" in + full|slim|none) ;; + *) echo "ERROR: unknown context mode: ${context}" >&2; return 1 ;; + esac + if ! rand=$(swarm_random_hex 4) || ! nonce=$(swarm_random_hex 8); then + echo "ERROR: cannot read /dev/urandom." >&2 + return 1 + fi + vol="${name}-ws-${rand}" + err=$(docker volume inspect "$vol" 2>&1 >/dev/null) || rc=$? + if [ "$rc" -eq 0 ]; then + echo "ERROR: Docker volume ${vol} already exists." >&2 + return 1 + fi + case "$err" in + *"o such volume"*) ;; + *) + echo "ERROR: cannot check Docker volume ${vol}: ${err}" >&2 + return 1 ;; + esac + if ! docker volume create \ + --label "${SWARM_PROJECT_LABEL}=${project}" \ + --label "${SWARM_CHECKOUT_LABEL}=${checkout}" \ + --label "${SWARM_ENGAGEMENT_LABEL}=${engagement}" \ + --label "${SWARM_TOKEN_LABEL}=${token}" \ + --label "${SWARM_CONTAINER_LABEL}=${name}" \ + --label "${SWARM_KIND_LABEL}=${kind}" \ + --label "${SWARM_CONTEXT_LABEL}=${context}" \ + --label "${SWARM_NONCE_LABEL}=${nonce}" \ + "$vol" >/dev/null; then + echo "ERROR: cannot create Docker volume ${vol}." >&2 + return 1 + fi + labels=$(docker volume inspect -f '{{json .Labels}}' "$vol" \ + 2>/dev/null) || labels="" + if ! printf '%s' "$labels" | jq -e \ + --arg pk "$SWARM_PROJECT_LABEL" --arg p "$project" \ + --arg ck "$SWARM_CHECKOUT_LABEL" --arg c "$checkout" \ + --arg ek "$SWARM_ENGAGEMENT_LABEL" --arg e "$engagement" \ + --arg tk "$SWARM_TOKEN_LABEL" --arg t "$token" \ + --arg nk "$SWARM_CONTAINER_LABEL" --arg n "$name" \ + --arg kk "$SWARM_KIND_LABEL" --arg k "$kind" \ + --arg xk "$SWARM_CONTEXT_LABEL" --arg x "$context" \ + --arg ok "$SWARM_NONCE_LABEL" --arg o "$nonce" ' + .[$pk] == $p and .[$ck] == $c and .[$ek] == $e and .[$tk] == $t + and .[$nk] == $n and .[$kk] == $k and .[$xk] == $x + and .[$ok] == $o' >/dev/null 2>&1; then + echo "ERROR: Docker volume ${vol} is not the one just created." >&2 + return 1 + fi + printf '%s' "$vol" +} # docker ps format and jq definitions for the project's containers. A # container is the project's when its project label is the id, @@ -518,20 +680,18 @@ swarm_containers_verify() { | jq -rs 'map(select(.owner == "") | .name) | join(" ")') \ || unlabeled="" if [ "$others" != 0 ]; then - echo "Give this checkout its own id with SWARM_PROJECT, or stop" \ - "those containers from the checkout that owns them and" \ - "remove them with docker rm once their work is harvested." >&2 + echo "Give this checkout its own id with SWARM_PROJECT, or" \ + "retire or clean up those containers from the checkout" \ + "that owns them (./launch.sh retire, ./launch.sh cleanup" \ + "there)." >&2 fi - # An older release started these, and its agents push to the bare - # repository in /tmp: stopping them without the grace period can - # lose their last commits. + # An older release started these, and their files may live only in + # their own layers (stopping one started with --rm removes it). if [ -n "$unlabeled" ]; then - echo "Containers without an owner label come from an older" \ - "swarm-core. Stop them with the grace ./launch.sh stop" \ - "gives, so their agents can push their last commits:" >&2 - echo " docker stop -t ${SWARM_STOP_TIMEOUT:-60} ${unlabeled}" >&2 - echo "Then fetch their work and remove them as \"Upgrading from" \ - "/tmp state\" in USAGE.md describes." >&2 + echo "Containers without an owner label (${unlabeled}) come" \ + "from an older swarm-core, which made their engagement:" \ + "finish it with that release, as \"Upgrading from /tmp" \ + "state\" in USAGE.md describes." >&2 fi return 1 } @@ -568,6 +728,14 @@ swarm_own_container_id() { esac } +# Succeed when Docker lists no container with ID (it was removed); fail +# when it does, or cannot tell. +swarm_container_gone() { + local left + left=$(docker ps -aq --no-trunc --filter "id=$1" 2>/dev/null) || return 1 + [ -z "$left" ] +} + # Print "name id" lines for the project's containers that this checkout # owns, so that callers can address each container by ID. swarm_own_containers() { @@ -578,6 +746,55 @@ swarm_own_containers() { 'select(.owner == $root) | "\(.name) \(.id)"' } +# Print what runtime directory DIR holds of an engagement, one entry per +# line: its bare repo and upstream.git.new, mirrors/, any remnant of a +# cleanup (hidden names too), and the containers and workspace volumes +# of the checkout at ROOT (canonical path CHECKOUT) under project id +# PROJECT. Fails, saying so, when Docker cannot list them. +swarm_runtime_leftovers() { + local dir="$1" project="$2" root="$3" checkout="$4" e containers vols + for e in "$dir/upstream.git" "$dir/upstream.git.new" "$dir/mirrors" \ + "$dir"/*.removing "$dir"/.[!.]*.removing \ + "$dir"/..?*.removing; do + if [ -e "$e" ] || [ -L "$e" ]; then + printf ' %s\n' "$e" + fi + done + if ! containers=$(swarm_own_containers "$project" "$root"); then + echo "ERROR: cannot list Docker containers." >&2 + return 1 + fi + if [ -n "$containers" ]; then + printf ' containers: %s\n' "$(printf '%s\n' "$containers" \ + | awk '{ print $1 }' | tr '\n' ' ')" + fi + if ! vols=$(docker volume ls -q \ + --filter "label=${SWARM_PROJECT_LABEL}=${project}" \ + --filter "label=${SWARM_CHECKOUT_LABEL}=${checkout}" \ + 2>/dev/null); then + echo "ERROR: cannot list Docker volumes." >&2 + return 1 + fi + if [ -n "$vols" ]; then + printf ' volumes: %s\n' "$(printf '%s\n' "$vols" | tr '\n' ' ')" + fi +} + +# Fail, saying how to finish it, when runtime directory DIR holds +# something of an engagement although its state file records none +# (see swarm_runtime_leftovers for the other arguments): this engine +# records the engagement before it makes anything, so that comes from +# an earlier release. Also fails when Docker cannot list what is there. +swarm_unrecorded_refuse() { + local dir="$1" left + left=$(swarm_runtime_leftovers "$@") || return 1 + [ -n "$left" ] || return 0 + swarm_legacy_explain "$dir" \ + "what is left of an engagement it never recorded" + printf '%s\n' "$left" >&2 + return 1 +} + # An engagement is one life of the bare repo: it starts when the bare # repo is created, and every later phase (post-process, interactive, # harvest) belongs to it. The id is SWARM_ENGAGEMENT_ID when the @@ -637,6 +854,114 @@ swarm_engagement_recorded() { printf '%s' "$state" | jq -r '.engagement_id // empty' } +# Print the engagement token state file FILE records (empty when none, +# or when there is no state file). Fails when the file cannot be read. +swarm_engagement_token() { + local file="$1" state + if [ ! -e "$file" ] && [ ! -L "$file" ]; then + return 0 + fi + state=$(swarm_state_read "$file") || { + echo "ERROR: not a ${SWARM_STATE_SCHEMA} state file: ${file}" >&2 + return 1 + } + printf '%s' "$state" | jq -r '.engagement_token // empty' +} + +# Fail when SWARM_ENGAGEMENT_TOKEN is set and state file FILE records +# another engagement token, or none: the command was started for an +# engagement that has gone since (cleaned up, maybe started again). +swarm_engagement_token_check() { + local file="$1" token + [ -n "${SWARM_ENGAGEMENT_TOKEN:-}" ] || return 0 + token=$(swarm_engagement_token "$file") || return 1 + if [ "$token" != "$SWARM_ENGAGEMENT_TOKEN" ]; then + echo "ERROR: the engagement this command was started for is" \ + "gone (cleaned up, or started again)." >&2 + return 1 + fi +} + +# Return 0 when state file FILE records an engagement an earlier release +# started (an engagement id without a Docker daemon: its containers keep +# their files in their own layers), 1 when it does not (or there is no +# state file), 2 when the file cannot be read. +swarm_engagement_legacy() { + local file="$1" state + if [ ! -e "$file" ] && [ ! -L "$file" ]; then + return 1 + fi + state=$(swarm_state_read "$file") || return 2 + case "$(printf '%s' "$state" | jq -r ' + if (.engagement_id // "") != "" and (.docker_daemon // "") == "" + then "legacy" else "current" end' 2>/dev/null)" in + legacy) return 0 ;; + current) return 1 ;; + *) return 2 ;; + esac +} + +# Explain on stderr how to finish what an earlier release left in +# runtime directory DIR (WHAT says what was found). +swarm_legacy_explain() { + local dir="$1" what="$2" + cat >&2 <&2 + return 1 ;; + esac +} + +# Stop unless Docker talks to the daemon the engagement in state file +# FILE recorded. The context is pinned first, so every later docker +# call of this run goes to the same daemon. A file that cannot be read +# fails; so does an engagement without a daemon (callers refuse an +# earlier release's engagement before this). +swarm_engagement_daemon_check() { + local file="$1" state recorded current + state=$(swarm_state_read "$file") || { + echo "ERROR: cannot read ${file}." >&2 + return 1 + } + recorded=$(printf '%s' "$state" | jq -r '.docker_daemon // empty') \ + || return 1 + if [ -z "$recorded" ]; then + echo "ERROR: ${file} records no Docker daemon." >&2 + return 1 + fi + if ! swarm_docker_pin || ! current=$(swarm_docker_daemon_id); then + echo "ERROR: cannot read the Docker daemon id." >&2 + return 1 + fi + if [ "$current" != "$recorded" ]; then + echo "ERROR: this engagement runs on Docker daemon ${recorded}," \ + "but Docker now talks to ${current}. Switch back to that" \ + "daemon (docker context use, DOCKER_HOST) first." >&2 + return 1 + fi +} + # Fail when state file FILE records a cleanup that has not finished: # the engagement is half deleted, and only cleanup may go on. swarm_cleanup_pending_check() { @@ -813,6 +1138,11 @@ swarm_pending_write() { return 1 ;; esac file="$dir/interactive-${name}.pending" + # Never over another session's record (callers hold the lock). + if [ -e "$file" ] || [ -L "$file" ]; then + echo "ERROR: a session record ${file} is there already." >&2 + return 1 + fi { printf 'pid=%s\n' "$pid" printf 'started=%s\n' "$(swarm_proc_started "$pid")" diff --git a/lib/retire.sh b/lib/retire.sh index 7597a5d..58768ee 100644 --- a/lib/retire.sh +++ b/lib/retire.sh @@ -767,7 +767,18 @@ HELP # id could have claimed the runtime and started in between. swarm_runtime_verify "$RUNTIME_DIR" "$REPO_ROOT" "$PROJECT" || return 1 swarm_engagement_join "$STATE_FILE" || return 1 - retire_require_daemon || return 1 + swarm_legacy_refuse "$RUNTIME_DIR" "$STATE_FILE" || return 1 + local engagement + engagement=$(swarm_engagement_recorded "$STATE_FILE") || return 1 + if [ -z "$engagement" ]; then + # Containers without a recorded engagement are an earlier + # release's. + swarm_unrecorded_refuse "$RUNTIME_DIR" "$PROJECT" "$REPO_ROOT" \ + "$CHECKOUT_PATH" || return 1 + echo "No engagement recorded in ${STATE_FILE}; nothing to retire." + return 0 + fi + swarm_engagement_daemon_check "$STATE_FILE" || return 1 retire_sweep_snapshots || return 1 local list sel checked id name @@ -834,31 +845,3 @@ EOF "./launch.sh cleanup deletes them after its checks." return "$failed" } - -# Stop unless Docker talks to the daemon the engagement recorded. An -# engagement without a recorded daemon is refused: retire acts on -# containers by ID and must know they are the engagement's. -retire_require_daemon() { - local recorded current - recorded=$(swarm_state_read "$STATE_FILE" 2>/dev/null \ - | jq -r '.docker_daemon // empty' 2>/dev/null) || recorded="" - if [ -z "$recorded" ]; then - if [ -z "${SWARM_DOCKER_DAEMON:-}" ]; then - echo "ERROR: ${STATE_FILE} records no Docker daemon. Name the" \ - "one this engagement runs on:" >&2 - echo " SWARM_DOCKER_DAEMON=\$(docker info -f '{{.ID}}')" \ - "./launch.sh retire" >&2 - return 1 - fi - recorded="$SWARM_DOCKER_DAEMON" - fi - if ! swarm_docker_pin || ! current=$(swarm_docker_daemon_id); then - echo "ERROR: cannot read the Docker daemon id." >&2 - return 1 - fi - if [ "$current" != "$recorded" ]; then - echo "ERROR: this engagement runs on Docker daemon ${recorded}," \ - "but Docker now talks to ${current}." >&2 - return 1 - fi -} diff --git a/tests/runtime_cleanup.sh b/tests/runtime_cleanup.sh index 09996b8..14a0a77 100755 --- a/tests/runtime_cleanup.sh +++ b/tests/runtime_cleanup.sh @@ -10,9 +10,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" # into as its own user (on Linux another UID than this host user's, so # only the Docker helper can delete those files), a mirror, and a real # workspace volume of this checkout. Cleanup refuses while a container -# of the project remains (before workspace volumes, retire refuses -# every container), then copies all three into the checkout as recovery -# refs and deletes them. +# of the project remains that retire refuses (one without its workspace +# volume, as an earlier release made them), then copies all three into +# the checkout as recovery refs and deletes them. Then one whole +# engagement through launch.sh: start, wait, retire, cleanup, and a +# second start on the cleaned runtime directory. # # Requires Docker. No API key needed. @@ -38,6 +40,8 @@ source "$REPO_ROOT/lib/project.sh" BASE_IMAGE="swarm-core-runtime-test" P="swarm-core-cleanup-test" +# The engagement launch.sh starts, under a project id of its own. +P2="swarm-core-cycle-test" # Cleanup's helpers run the engine image, -agent. ENGINE_IMAGE="${P}-agent" NAME="${P}-agent-1" @@ -45,12 +49,19 @@ VOL="${NAME}-ws-00000000" WORKDIR=$(mktemp -d /tmp/swarm-core-cleanup-XXXXXX) drop_all() { - local c - for c in $(docker ps -aq --filter "label=${SWARM_PROJECT_LABEL}=${P}"); do - docker rm -f "$c" >/dev/null 2>&1 || true - done - for c in $(docker ps -aq --filter "label=swarm-core.cleanup=${P}"); do - docker rm -f "$c" >/dev/null 2>&1 || true + local c p + for p in "$P" "$P2"; do + for c in $(docker ps -aq \ + --filter "label=${SWARM_PROJECT_LABEL}=${p}"); do + docker rm -f "$c" >/dev/null 2>&1 || true + done + for c in $(docker ps -aq --filter "label=swarm-core.cleanup=${p}"); do + docker rm -f "$c" >/dev/null 2>&1 || true + done + for c in $(docker volume ls -q \ + --filter "label=${SWARM_PROJECT_LABEL}=${p}"); do + docker volume rm "$c" >/dev/null 2>&1 || true + done done docker volume rm "$VOL" >/dev/null 2>&1 || true } @@ -60,8 +71,8 @@ cleanup_test() { # root with the test image, which needs no pull. docker run --rm --user 0 --entrypoint rm \ -v "$WORKDIR:/workdir" "$BASE_IMAGE" -rf /workdir/runtime \ - >/dev/null 2>&1 || true - docker rmi "$ENGINE_IMAGE" >/dev/null 2>&1 || true + /workdir/runtime2 >/dev/null 2>&1 || true + docker rmi "$ENGINE_IMAGE" "${P2}-agent" >/dev/null 2>&1 || true rm -rf "$WORKDIR" } trap cleanup_test EXIT @@ -213,6 +224,82 @@ check "no helper container is left" \ "$(docker ps -aq --filter "label=swarm-core.cleanup=${P}" | grep -c . \ || true)" "0" +echo "--- a whole engagement: start, wait, retire, cleanup, start ---" +# The fake driver makes no commits, so with max_idle 1 each agent exits +# after one session. docker_args ask for auto-remove, a restart policy +# and another name; what the engine sets comes after and wins. +C2="$WORKDIR/cycle" +RT2="$WORKDIR/runtime2" +git init -q "$C2" +echo "Prompt." > "$C2/prompt.md" +cat > "$C2/swarm.json" <<'EOF' +{ "prompt": "prompt.md", "driver": "fake", "max_idle": 1, + "docker_args": ["--rm", "--restart", "always", "--name", "other"], + "agents": [{ "count": 2, "model": "fake" }] } +EOF +gitc -C "$C2" add -A +gitc -C "$C2" commit -qm init +run2() { + (cd "$C2" && SWARM_RUNTIME_DIR="$RT2" SWARM_PROJECT="$P2" \ + SWARM_SKIP_DEP_CHECK=1 SWARM_WAIT_POLL=2 "$BASH" "$LAUNCH_SH" "$@") +} +vols2() { + docker volume ls -q --filter "label=${SWARM_PROJECT_LABEL}=${P2}" \ + | grep -c . || true +} +rc=0 +out=$(run2 start 2>&1) || rc=$? +check "start succeeds" "$rc" "0" +[ "$rc" -eq 0 ] || printf '%s\n' "$out" | tail -8 | sed 's/^/ /' +TOKEN1=$(jq -r '.engagement_token // empty' "$RT2/state.json") +check "one volume per agent" "$(vols2)" "2" +for i in 1 2; do + check "agent $i: its own name, no restart policy, no auto-remove" \ + "$(docker inspect -f \ + '{{.Name}} {{.HostConfig.RestartPolicy.Name}} {{.HostConfig.AutoRemove}}' \ + "${P2}-agent-${i}" 2>/dev/null)" "/${P2}-agent-${i} no false" + check "agent $i: /workspace is its volume" \ + "$(docker inspect -f '{{range .Mounts}}{{if eq .Destination "/workspace"}}{{.Type}} {{.Name}}{{end}}{{end}}' \ + "${P2}-agent-${i}" 2>/dev/null | grep -cE "^volume ${P2}-agent-${i}-ws-[0-9a-f]{8}$")" "1" +done +rc=0 +out=$(run2 wait 2>&1) || rc=$? +check "wait sees both agents finish and harvests" "$rc" "0" +[ "$rc" -eq 0 ] || printf '%s\n' "$out" | tail -8 | sed 's/^/ /' +rc=0 +run2 start >/dev/null 2>&1 || rc=$? +check "a second start before cleanup is refused" "$rc" "1" +rc=0 +out=$(run2 retire 2>&1) || rc=$? +check "retire takes the stopped agents" "$rc" "0" +[ "$rc" -eq 0 ] || printf '%s\n' "$out" | tail -8 | sed 's/^/ /' +check "no container of the engagement is left" \ + "$(docker ps -aq --filter "label=${SWARM_PROJECT_LABEL}=${P2}" \ + | grep -c . || true)" "0" +check "their volumes stay" "$(vols2)" "2" +rc=0 +out=$(run2 cleanup 2>&1) || rc=$? +check "cleanup succeeds" "$rc" "0" +[ "$rc" -eq 0 ] || printf '%s\n' "$out" | tail -8 | sed 's/^/ /' +check "the volumes are gone" "$(vols2)" "0" +check "so is the bare repo" "$(present "$RT2/upstream.git")" "no" +check "the volumes' history is in the recovery refs" \ + "$(git -C "$C2" for-each-ref refs/swarm-recovery/ | grep -c . \ + | awk '{ print ($1 > 0) }')" "1" +rc=0 +out=$(run2 start 2>&1) || rc=$? +check "start on the cleaned runtime succeeds" "$rc" "0" +[ "$rc" -eq 0 ] || printf '%s\n' "$out" | tail -8 | sed 's/^/ /' +check "as a new engagement" \ + "$(jq -r '.engagement_token // empty' "$RT2/state.json" \ + | grep -cvx "${TOKEN1:-none}" || true)" "1" +check "with new volumes" "$(vols2)" "2" +run2 wait >/dev/null 2>&1 || true +run2 retire >/dev/null 2>&1 || true +rc=0 +run2 cleanup >/dev/null 2>&1 || rc=$? +check "and cleans up again" "$rc" "0" + echo "" echo " ${PASS} passed, ${FAIL} failed" [ "$FAIL" -eq 0 ] diff --git a/tests/runtime_pi.sh b/tests/runtime_pi.sh index dd6d8c8..792b577 100755 --- a/tests/runtime_pi.sh +++ b/tests/runtime_pi.sh @@ -51,6 +51,13 @@ cleanup() { >/dev/null 2>&1 || true fi docker image rm "$IMAGE" >/dev/null 2>&1 || true + # The workspace volumes launch.sh made for this run's containers (the + # project id is this run's own). + local vol + for vol in $(docker volume ls -q \ + --filter "label=swarm-core.project=${PROJECT}" 2>/dev/null); do + docker volume rm "$vol" >/dev/null 2>&1 || true + done rm -rf "$WORK" "$RT" } trap cleanup EXIT diff --git a/tests/runtime_signal_trap.sh b/tests/runtime_signal_trap.sh index 8d0dac4..9715941 100755 --- a/tests/runtime_signal_trap.sh +++ b/tests/runtime_signal_trap.sh @@ -280,23 +280,36 @@ FAKE_DIR="$WORKDIR/fake-bin" mkdir -p "$FAKE_DIR" cat > "$FAKE_DIR/docker" <<'FD' #!/bin/bash +# The daemon cmd_stop finds the engagement on, and no container of it. +case "$1 ${2:-}" in + "info -f") echo d1 ;; + "context show") echo default ;; +esac exit 0 FD chmod +x "$FAKE_DIR/docker" -# cmd_stop checks the engagement in the state file before its banner, -# and stops containers through stop_own_container and lib/project.sh. +# cmd_stop checks the engagement in the state file (its token and +# daemon) before its banner, through lib/project.sh. # shellcheck disable=SC1091 source=../lib/project.sh source "$REPO_ROOT/lib/project.sh" CMD_STOP_BODY=$(awk '/^cmd_stop\(\) \{/,/^\}$/' "$LAUNCH_SH") -STOP_OWN_BODY=$(awk '/^stop_own_container\(\) \{/,/^\}$/' "$LAUNCH_SH") -STATE_FILE="$WORKDIR/no-state.json" +# cmd_stop checks a copy of the state, which these make (in WORKDIR, +# which this test removes). +STOP_HELPERS=$(awk '/^(state_snapshot|swarm_tmp_file)\(\) \{/,/^\}$/' \ + "$LAUNCH_SH") +RUNTIME_DIR="$WORKDIR/stop-runtime" +mkdir -p "$RUNTIME_DIR" +STATE_FILE="$RUNTIME_DIR/state.json" +printf '%s\n' '{"schema":"swarm-core.state/v1","engagement_id":"e1", + "engagement_token":"t1","docker_daemon":"d1"}' > "$STATE_FILE" CHECKOUT_PATH="$WORKDIR" BANNER_DEFAULT=$( eval "$CMD_STOP_BODY" - eval "$STOP_OWN_BODY" - NUM_AGENTS=2 IMAGE_NAME=fake PROJECT=fake \ + eval "$STOP_HELPERS" + SWARM_TMP_FILES=() + NUM_AGENTS=2 IMAGE_NAME=fake PROJECT=fake TMPDIR="$WORKDIR" \ PATH="$FAKE_DIR:$PATH" cmd_stop 2>&1 | head -1 ) check "default banner" \ @@ -304,8 +317,9 @@ check "default banner" \ BANNER_120=$( eval "$CMD_STOP_BODY" - eval "$STOP_OWN_BODY" - NUM_AGENTS=2 IMAGE_NAME=fake PROJECT=fake \ + eval "$STOP_HELPERS" + SWARM_TMP_FILES=() + NUM_AGENTS=2 IMAGE_NAME=fake PROJECT=fake TMPDIR="$WORKDIR" \ PATH="$FAKE_DIR:$PATH" SWARM_STOP_TIMEOUT=120 cmd_stop 2>&1 \ | head -1 ) diff --git a/tests/test.sh b/tests/test.sh index f228e6b..0118d34 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -5,10 +5,20 @@ set -euo pipefail # start real containers, so keep the caller's: it is how a checkout # runs when another one owns the project's containers. _caller_project="${SWARM_PROJECT:-}" +_own_project=false # shellcheck source=_test_env.sh source "$(dirname "${BASH_SOURCE[0]}")/_test_env.sh" if [ -n "$_caller_project" ]; then export SWARM_PROJECT="$_caller_project" +else + _own_project=true + # Each run makes and removes an engagement of its own, under a + # project id of its own, so it never meets an engagement of this + # checkout (a new one needs a clean runtime), and its cleanup never + # touches one. + SWARM_PROJECT="swarm-core-test-$(od -An -N4 -tx1 /dev/urandom \ + | tr -d ' \n')" + export SWARM_PROJECT fi # Smoke test: launch agents with a counting prompt, verify results. @@ -38,6 +48,13 @@ REVIEW_DIR="/tmp/${PROJECT}-test-review" INJECT_DIR="/tmp/${PROJECT}-test-inject" TIMEOUT="${TIMEOUT:-600}" +# The engine image of a project id this run made up goes when the run +# is over (the runs it starts share the id and leave it). +drop_own_image() { + $_own_project || return 0 + docker rmi "${PROJECT}-agent" >/dev/null 2>&1 || true +} + # Docker containers may leave files owned by a different UID. rm_docker_dir() { local dir="$1" @@ -455,8 +472,10 @@ if $RUN_UNIT; then fi if $RUN_OAUTH; then - cmd_oauth - exit $? + rc=0 + cmd_oauth || rc=$? + drop_own_image + exit "$rc" fi if $RUN_RUNTIME; then @@ -468,8 +487,10 @@ if $RUN_RUNTIME; then fi if $RUN_ALL; then - run_all_tests - exit $? + rc=0 + run_all_tests || rc=$? + drop_own_image + exit "$rc" fi if [ -n "$CONFIG_FILE" ] && [ ! -f "$CONFIG_FILE" ]; then @@ -679,11 +700,29 @@ cleanup() { cd "$REPO_ROOT" SWARM_CONFIG="$TEMP_CONFIG" \ "$SWARM_DIR/launch.sh" stop 2>/dev/null || true + # This run's containers and volumes, and nothing else: those with + # its engagement's token, which its own runtime directory records. + local token id vol + token=$(swarm_engagement_token "$TEST_RUNTIME_DIR/state.json" \ + 2>/dev/null) || token="" + if [ -n "$token" ]; then + for id in $(docker ps -aq \ + --filter "label=swarm-core.engagement-token=${token}" \ + 2>/dev/null); do + docker rm -f "$id" >/dev/null 2>&1 || true + done + for vol in $(docker volume ls -q \ + --filter "label=swarm-core.engagement-token=${token}" \ + 2>/dev/null); do + docker volume rm "$vol" >/dev/null 2>&1 || true + done + fi rm -f "$TEMP_CONFIG" "$_AUTO_CONFIG" rm -rf "$REVIEW_DIR" "$INJECT_DIR" rm_docker_dir "$TEST_RUNTIME_DIR" rm -f "$REPO_ROOT/$PROMPT_FILE" "$REPO_ROOT/$SETUP_FILE" \ "$REPO_ROOT/$ALT_PROMPT_FILE" + drop_own_image } trap cleanup EXIT diff --git a/tests/test_cleanup.sh b/tests/test_cleanup.sh index 427483f..c6e306b 100755 --- a/tests/test_cleanup.sh +++ b/tests/test_cleanup.sh @@ -62,8 +62,10 @@ _ct=$(cd -P "$TMPDIR" && pwd -P) # show an empty directory) # daemon docker info id (default d1) # helper-mountinfo the mount table a deletion helper sees -# volume-later a volume name that docker volume ls adds from -# its second call on +# volume-later a volume name that docker volume ls adds once it +# was called volume-later-after times (default 1) +# present-ids ids docker ps --filter id= lists (none by default: +# a container whose inspect fails is gone) # log every call F="$TMPDIR/fake" mkdir -p "$TMPDIR/bin" @@ -95,6 +97,7 @@ case "$1" in cat "$F/volusers/$v" 2>/dev/null exit 0 ;; *"label=swarm-core.session="*) exit 0 ;; + *" id="*) cat "$F/present-ids" 2>/dev/null ;; *"label=swarm-core.project="*) while read -r id name; do [ -n "$id" ] || continue @@ -125,11 +128,13 @@ case "$1" in cat "$F/volumes" # A volume that shows up after the first listing. if [ -f "$F/volume-later" ]; then - if [ -f "$F/volume-listed" ]; then + listed=$(cat "$F/volume-listed" 2>/dev/null || echo 0) + if [ "$listed" -ge \ + "$(cat "$F/volume-later-after" 2>/dev/null \ + || echo 1)" ]; then cat "$F/volume-later" - else - : > "$F/volume-listed" fi + echo $((listed + 1)) > "$F/volume-listed" fi exit 0 ;; rm) @@ -442,7 +447,7 @@ refused "a runtime directory with a tab in its path" "a tab or a newline" setup assert_eq "a runtime directory that is HOME: refused" "1" \ "$(HOME="$RT" cleanup)" -assert_eq "for that reason" "1" "$(said "refusing to clean up in")" +assert_eq "for that reason" "1" "$(said "as the runtime directory")" assert_eq "its bare repo stays" "yes" "$(present "$RT/upstream.git")" setup mv "$RT" "$REPO/rt" @@ -510,7 +515,8 @@ setup add_volume proj-agent-1-ws-00000001 add_volume proj-agent-2-ws-00000002 assert_eq "two volumes: cleanup succeeds" "0" "$(cleanup)" -assert_eq "both are scanned" "2" "$(grep -c 'type=volume,src=' "$F/log")" +# Scanned twice each: before retire, and again once it is done. +assert_eq "both are scanned" "4" "$(grep -c 'type=volume,src=' "$F/log")" assert_eq "both are removed" "0" "$(grep -c . "$F/volumes" || true)" assert_eq "their work is kept" "1" \ "$(git -C "$REPO" for-each-ref refs/swarm-recovery/ | grep -c . \ @@ -751,6 +757,9 @@ fi # one that showed up meanwhile keeps them. setup echo "${P}-late-ws-00000001" > "$F/volume-later" +# Listed first before retire, then after it: it shows up in the last +# listing, the one before the engagement's records go. +echo 2 > "$F/volume-later-after" assert_eq "a volume that shows up during cleanup: refused" "1" "$(cleanup)" assert_eq "for that reason" "1" "$(said "left these in place")" assert_eq "the checkpoint stays" "true" \ @@ -927,6 +936,169 @@ for bare in present renamed gone; do done done +echo "" +echo "=== 6. Engagements of earlier releases, and runs cut off ===" +# An engagement an earlier release started records no Docker daemon: +# cleanup refuses it, says how to finish it, and changes nothing. +docker_changes() { + grep -cE '^(run|create|rm|volume rm|commit|stop|kill)' "$F/log" || true +} +setup +swarm_state_update "$RT/state.json" 'del(.docker_daemon)' +cp "$RT/state.json" "$TMPDIR/state.before" +refused "an earlier release's engagement" \ + "Finish it with the release that made it" +assert_eq "its state file stays as it was" "same" \ + "$(cmp -s "$TMPDIR/state.before" "$RT/state.json" && echo same)" +assert_eq "its mirror stays" "yes" "$(present "$RT/mirrors/lib space.git")" +assert_eq "nothing is copied" "0" "$(recovery_refs)" +assert_eq "Docker made or removed nothing" "0" "$(docker_changes)" +# What an engagement leaves, with none recorded, is an earlier release's +# too: the bare repo, a hidden remnant, a container, a volume. +for _left in bare hidden container volume; do + setup + swarm_state_update "$RT/state.json" 'del(.engagement_id, .docker_daemon)' + rm -rf "$RT/mirrors" + [ "$_left" = bare ] || rm -rf "$RT/upstream.git" + case "$_left" in + hidden) mkdir "$RT/.x.removing" ;; + container) echo "id-c1 proj-agent-1" > "$F/containers" ;; + volume) echo proj-agent-1-ws-00000001 > "$F/volumes" ;; + esac + cp "$RT/state.json" "$TMPDIR/state.before" + assert_eq "$_left without an engagement: refused" "1" "$(cleanup)" + assert_eq "$_left without an engagement: as an earlier release's" "1" \ + "$(said "Finish it with the release that made it")" + assert_eq "$_left without an engagement: state as it was" "same" \ + "$(cmp -s "$TMPDIR/state.before" "$RT/state.json" && echo same)" + assert_eq "$_left without an engagement: Docker changed nothing" "0" \ + "$(docker_changes)" + if [ "$_left" = hidden ]; then + assert_eq "the hidden remnant stays" "yes" \ + "$(present "$RT/.x.removing")" + fi +done + +# Runs cut off while they made the bare repo and a mirror: each left a +# .new without HEAD and with part of the objects, under the recorded +# engagement. cleanup keeps their files (files mode) and removes them. +setup +mv "$RT/upstream.git" "$TMPDIR/case/bare.full" +mkdir -p "$RT/upstream.git.new/objects" +cp -R "$TMPDIR/case/bare.full/objects/." "$RT/upstream.git.new/objects/" +rm -rf "$TMPDIR/case/bare.full" +mkdir -p "$RT/mirrors/.sub.git.new/objects/pack" +head -c 2000 /dev/urandom > "$RT/mirrors/.sub.git.new/objects/pack/tmp_pack_x" +assert_eq "two runs cut off half way: cleanup succeeds" "0" "$(cleanup)" +assert_eq "the bare repo's .new is removed" "no" \ + "$(present "$RT/upstream.git.new")" +assert_eq "so is the hidden mirror's" "no" \ + "$(present "$RT/mirrors/.sub.git.new")" +_base=$(git -C "$REPO" for-each-ref --format='%(refname)' refs/swarm-recovery/ \ + | sed -n 's|^\(refs/swarm-recovery/e[0-9a-f]*/[^/]*\)/.*|\1|p' | sort -u) +_map=$(git -C "$REPO" cat-file blob "$_base/sources" 2>/dev/null || true) +assert_eq "both are recorded as cut-off creations" "2" \ + "$(printf '%s\n' "$_map" | grep -c ' new "' || true)" +assert_eq "their files are kept" "1" \ + "$(git -C "$REPO" for-each-ref "$_base/" | grep -c '/files$' \ + | awk '{ print ($1 > 0) }')" + +# A stopped container whose volume holds uncommitted work: cleanup's +# first pass refuses while the container is still there, so its logs +# and its volume stay (retire never ran). +setup +add_volume proj-agent-1-ws-00000001 +echo dirty > "$F/voldir/proj-agent-1-ws-00000001/w" +echo "id-c1 proj-agent-1" > "$F/containers" +jq -n --arg ck "$CK" '[{Id: "id-c1", Name: "/proj-agent-1", + State: {Status: "exited"}, + HostConfig: {RestartPolicy: {Name: "no"}, AutoRemove: false}, + Mounts: [{Type: "volume", Name: "proj-agent-1-ws-00000001", + Destination: "/workspace"}], + Config: {Labels: {"swarm-core.project": "proj", + "swarm-core.checkout": $ck}}}]' > "$F/inspect/id-c1" +echo id-c1 > "$F/volusers/proj-agent-1-ws-00000001" +refused "uncommitted work while the container is there" \ + "holds work outside Git's history" +assert_eq "retire never ran: no container removed" "0" \ + "$(grep -cE '^(rm|commit|stop|kill)' "$F/log" || true)" +assert_eq "the container is still listed" "1" \ + "$(grep -c proj-agent-1 "$F/containers")" +assert_eq "and its volume stays" "1" "$(grep -c . "$F/volumes")" + +# A container that still runs: refused before any scan, with the step +# to take (stop the agents), not --discard-uncommitted for what its +# changing workspace holds. +setup +add_volume proj-agent-1-ws-00000001 +echo dirty > "$F/voldir/proj-agent-1-ws-00000001/w" +echo "id-c1 proj-agent-1" > "$F/containers" +jq -n --arg ck "$CK" '[{Id: "id-c1", Name: "/proj-agent-1", + State: {Status: "running"}, + HostConfig: {RestartPolicy: {Name: "no"}, AutoRemove: false}, + Mounts: [{Type: "volume", Name: "proj-agent-1-ws-00000001", + Destination: "/workspace"}], + Config: {Labels: {"swarm-core.project": "proj", + "swarm-core.checkout": $ck}}}]' > "$F/inspect/id-c1" +refused "a container that still runs" "still run, or Docker cannot tell" +assert_eq "naming it" "1" "$(said "proj-agent-1 (running)")" +assert_eq "before any scan" "0" "$(grep -c '^run ' "$F/log" || true)" +assert_eq "nothing removed or stopped" "0" \ + "$(grep -cE '^(rm|commit|stop|kill)' "$F/log" || true)" +# Docker cannot describe a container it still lists: refused the same +# way, rather than taken for gone. +rm -f "$F/inspect/id-c1" +echo id-c1 > "$F/present-ids" +: > "$F/log" +refused "a container Docker cannot describe" \ + "still run, or Docker cannot tell" +assert_eq "naming it" "1" "$(said "proj-agent-1 (state unknown)")" +assert_eq "before any scan" "0" "$(grep -c '^run ' "$F/log" || true)" + +# Agent work HEAD's history lacks (after a rewritten upstream, say): +# refused, then kept only as recovery refs with --allow-unharvested. +setup +gitc -C "$REPO" commit -q --amend -m rewritten +_old=$(git -C "$RT/upstream.git" rev-parse agent-work) +refused "work HEAD's history lacks" "agent-work" +assert_eq "the refusal names --allow-unharvested" "1" \ + "$(said "--allow-unharvested")" +assert_eq "nothing is copied then" "0" "$(recovery_refs)" +assert_eq "--allow-unharvested: cleanup goes on" "0" \ + "$(cleanup --allow-unharvested)" +assert_eq "saying what is kept only as recovery refs" "1" \ + "$(said "Kept only as recovery refs")" +assert_eq "the bare repo is gone" "no" "$(present "$RT/upstream.git")" +assert_eq "the old tip is in the recovery refs" "1" \ + "$(git -C "$REPO" for-each-ref --format='%(objectname)' \ + refs/swarm-recovery/ | grep -cx "$_old" | awk '{ print ($1 > 0) }')" +assert_eq "the engagement is cleaned up" "" \ + "$(jq -r '.engagement_id // empty' "$RT/state.json")" + +# A read of the bare repo's refs that fails part of the way (an I/O +# error) is no proof that the checkout has them: refused, with and +# without --allow-unharvested. +cat > "$TMPDIR/hook-ref.sh" <<'EOF' +read() { + if [ "$#" -eq 4 ] && [ "$1" = -r ] && [ "$2" != -u ] \ + && [ "$4" = ref ]; then + return 1 + fi + builtin read "$@" +} +EOF +for _flag in "" --allow-unharvested; do + setup + gitc -C "$REPO" commit -q --amend -m rewritten + assert_eq "a failed read of the refs${_flag:+ ($_flag)}: refused" "1" \ + "$(BASH_ENV="$TMPDIR/hook-ref.sh" cleanup $_flag)" + assert_eq "for that reason${_flag:+ ($_flag)}" "1" \ + "$(said "cannot read the refs of")" + assert_eq "the bare repo stays${_flag:+ ($_flag)}" "yes" \ + "$(present "$RT/upstream.git")" + assert_eq "nothing is copied${_flag:+ ($_flag)}" "0" "$(recovery_refs)" +done + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" diff --git a/tests/test_dashboard.sh b/tests/test_dashboard.sh index a916343..028ef8f 100755 --- a/tests/test_dashboard.sh +++ b/tests/test_dashboard.sh @@ -781,22 +781,30 @@ assert_eq "lowercase p asks for confirmation" "true" \ assert_eq "lowercase p can launch post-process" "1" \ "$(printf '%s\n' "$pp_lower_case" \ | grep -cF '"$SWARM_DIR/launch.sh" post-process' || true)" -assert_eq "lowercase p preserves dashboard config" "1" \ +# Both launch.sh runs get the dashboard's swarmfile and the token of the +# engagement shown, so a cleanup and a new start between them make the +# second refuse. +assert_eq "lowercase p passes the dashboard config to both runs" "2" \ "$(printf '%s\n' "$pp_lower_case" \ | grep -cF 'SWARM_CONFIG="$CONFIG_FILE"' || true)" +assert_eq "lowercase p passes the engagement token to both runs" "2" \ + "$(printf '%s\n' "$pp_lower_case" \ + | grep -cF 'SWARM_ENGAGEMENT_TOKEN="$_token"' || true)" +assert_eq "lowercase p stops the numbered agents through launch.sh" "1" \ + "$(printf '%s\n' "$pp_lower_case" \ + | grep -cF '"$SWARM_DIR/launch.sh" stop --numbered' || true)" assert_eq "lowercase p has cancellation path" "1" \ "$(printf '%s\n' "$pp_lower_case" \ - | grep -cF 'post-processing not started' || true)" + | grep -cF '"(post-processing not started)"' || true)" assert_eq "lowercase p refuses to replace running post-process" "1" \ "$(printf '%s\n' "$pp_lower_case" \ | grep -cF 'post-processing is already running' || true)" assert_eq "lowercase p blocks only running post-process" "1" \ "$(printf '%s\n' "$pp_lower_case" \ | grep -cF 'if post_process_container_running; then' || true)" -assert_eq "lowercase p allows exited post-process replacement" "1" \ +assert_eq "lowercase p sends an exited post-process to retire" "1" \ "$(printf '%s\n' "$pp_lower_case" \ - | grep -cF 'previous post-process container will be replaced' \ - || true)" + | grep -cF './launch.sh retire --post-process' || true)" assert_eq "lowercase p has no replacement prompt" "0" \ "$(printf '%s\n' "$pp_lower_case" \ | grep -cF 'Replace existing' || true)" @@ -817,12 +825,17 @@ assert_eq "footer guards start hint on running container" "1" \ assert_eq "footer keeps logs hint for exited container" "1" \ "$(printf '%s\n' "$help_bar" \ | grep -cF 'if post_process_container_exists; then' || true)" -assert_eq "s stops post-process container" "1" \ +# launch.sh stop covers the agents, post-process and the interactive +# containers of the engagement shown (its token), with its own checks. +assert_eq "s stops through launch.sh stop" "1" \ "$(printf '%s\n' "$s_case" \ - | grep -cF 'stop_own_container "${IMAGE_NAME}-post"' || true)" -assert_eq "s stops interactive containers" "1" \ - "$(printf '%s\n' "$s_case" \ - | grep -cF 'interactive_container_names' || true)" + | grep -cF '"$SWARM_DIR/launch.sh" stop' || true)" +assert_eq "s passes the dashboard config" "1" \ + "$(printf '%s\n' "$s_case" | grep -cF 'SWARM_CONFIG="$CONFIG_FILE"' \ + || true)" +assert_eq "and the engagement token" "1" \ + "$(printf '%s\n' "$s_case" | grep -cF 'SWARM_ENGAGEMENT_TOKEN="$_token"' \ + || true)" # ============================================================ echo "" @@ -916,6 +929,8 @@ done RUNTIME_DIR="$_ls_tmp/rt" STATE_FILE="$RUNTIME_DIR/state.json" +# As dashboard.sh sets it before its first load_state. +DASH_TOKEN="" mkdir -m 700 "$RUNTIME_DIR" jq -n --arg t "x \$(touch $_ls_tmp/pwned)" \ '{schema: "swarm-core.state/v1", title: $t, @@ -933,6 +948,17 @@ assert_eq "config" "/c/swarm.json" "$SWARM_CONFIG" assert_eq "agent count" "3" "$SWARM_NUM_AGENTS" assert_eq "model summary" "3x m" "$SWARM_MODEL_SUMMARY" assert_eq "config label" "swarm.json" "$SWARM_CONFIG_LABEL" +assert_eq "no engagement recorded: no token shown" "" "$DASH_TOKEN" +# The engagement shown is the state's token; once the state has none +# (cleaned up), none is shown, whatever was shown before. +jq '.engagement_token = "t1"' "$STATE_FILE" > "$_ls_tmp/st" \ + && mv "$_ls_tmp/st" "$STATE_FILE" +load_state || true +assert_eq "the engagement's token is shown" "t1" "$DASH_TOKEN" +jq 'del(.engagement_token)' "$STATE_FILE" > "$_ls_tmp/st" \ + && mv "$_ls_tmp/st" "$STATE_FILE" +load_state || true +assert_eq "a cleaned-up engagement shows no token" "" "$DASH_TOKEN" printf 'SWARM_TITLE="from env file"\n' > "$STATE_FILE" SWARM_TITLE="kept" @@ -1067,7 +1093,133 @@ rm -rf "$_id_tmp" # ============================================================ echo "" -echo "=== 17. A dashboard that stops short does not end with 0 ===" +echo "=== 17. s and p act through launch.sh on the engagement shown ===" + +# The s and p arms of the key loop, run with a launch.sh that logs its +# arguments, the swarmfile and the token it got. Its stop can fail, or +# replace the engagement (another token in the state file) as a cleanup +# and a new start in another terminal would. The swarmfile is a custom +# one, as a standalone dashboard knows it from the state file, and the +# engagement shown is DASH_TOKEN (load_state sets it at each refresh). +_kd=$(mktemp -d) +mkdir -p "$_kd/swarm" +cat > "$_kd/swarm/launch.sh" <<'EOF' +#!/bin/bash +printf '%s|%s|%s\n' "$*" "${SWARM_CONFIG:-}" \ + "${SWARM_ENGAGEMENT_TOKEN-unset}" >> "$KEYS_DIR/log" +if [ "$1" = stop ]; then + [ ! -e "$KEYS_DIR/stop-fails" ] || exit 1 + if [ -e "$KEYS_DIR/replace" ]; then + jq -c '.engagement_token = "t2"' "$KEYS_DIR/state.json" \ + > "$KEYS_DIR/state.x" && mv "$KEYS_DIR/state.x" \ + "$KEYS_DIR/state.json" + fi +fi +exit 0 +EOF +chmod +x "$_kd/swarm/launch.sh" +# Run the body of a case arm (ARM, as extracted above) once, with the +# answers on standard input; its output goes to $_kd/out. +run_key() { + local body + body=$(printf '%s\n' "$1" | sed '1d;$d') + ( + SWARM_DIR="$_kd/swarm" + STATE_FILE="$_kd/state.json" + CONFIG_FILE="$_kd/custom.json" + IMAGE_NAME=proj-agent + DASH_TOKEN="$KEY_TOKEN" + KEYS_DIR="$_kd" + export KEYS_DIR + # The dashboard's own helpers, as the key loop finds them. + # shellcheck disable=SC2329 + leave_alt_screen() { :; } + # shellcheck disable=SC2329 + enter_alt_screen() { :; } + # shellcheck disable=SC2329 + identity_ok() { return 0; } + # shellcheck disable=SC2329 + post_process_configured() { return 0; } + # shellcheck disable=SC2329 + post_process_container_running() { [ -e "$_kd/pp-running" ]; } + # shellcheck disable=SC2329 + post_process_container_exists() { [ -e "$_kd/pp-exists" ]; } + # The arms end with continue: a loop of one turn. + eval "for _turn in 1; do +$body +done" + ) < <(printf '%s\n' "${2:-}" "" "" "") > "$_kd/out" 2>&1 +} +key_reset() { + rm -f "$_kd/log" "$_kd/stop-fails" "$_kd/replace" "$_kd/pp-running" \ + "$_kd/pp-exists" + : > "$_kd/log" + KEY_TOKEN=t1 + jq -n '{schema: "swarm-core.state/v1", engagement_id: "e1", + engagement_token: "t1", docker_daemon: "d1"}' > "$_kd/state.json" +} +key_reset +run_key "$s_case" +assert_eq "s runs launch.sh stop with the swarmfile and the token" \ + "stop|$_kd/custom.json|t1" "$(cat "$_kd/log")" +key_reset +run_key "$pp_lower_case" y +assert_eq "p stops the numbered agents, then starts post-process" \ + "stop --numbered|$_kd/custom.json|t1 +post-process|$_kd/custom.json|t1" "$(cat "$_kd/log")" +key_reset +: > "$_kd/replace" +run_key "$pp_lower_case" y +assert_eq "an engagement replaced during p's stop: post-process gets the old token" \ + "post-process|$_kd/custom.json|t1" "$(sed -n 2p "$_kd/log")" +key_reset +: > "$_kd/stop-fails" +run_key "$pp_lower_case" y +assert_eq "a stop that fails: no post-process" "1" \ + "$(grep -c . "$_kd/log")" +assert_eq "saying so" "1" \ + "$(grep -c 'agents not stopped; post-processing not started' \ + "$_kd/out" || true)" +key_reset +: > "$_kd/pp-exists" +run_key "$pp_lower_case" y +assert_eq "an earlier post-process container: p runs nothing" "0" \ + "$(grep -c . "$_kd/log" || true)" +assert_eq "and names the retire step" "1" \ + "$(grep -c 'retire it first: ./launch.sh retire --post-process' \ + "$_kd/out" || true)" +key_reset +run_key "$pp_lower_case" n +assert_eq "p not confirmed: nothing runs" "0" \ + "$(grep -c . "$_kd/log" || true)" +# No engagement shown (cleaned up, or none started): nothing runs, +# rather than a launch.sh bound to no engagement, which would act on +# whatever is there by then. +key_reset +KEY_TOKEN="" +run_key "$s_case" +assert_eq "no engagement shown: s runs nothing" "0" \ + "$(grep -c . "$_kd/log" || true)" +assert_eq "saying so" "1" \ + "$(grep -c 'no engagement shown; nothing stopped' "$_kd/out" || true)" +key_reset +KEY_TOKEN="" +run_key "$pp_lower_case" y +assert_eq "no engagement shown: p runs nothing" "0" \ + "$(grep -c . "$_kd/log" || true)" +# The token is the one shown, not one read at the key press: an +# engagement started since the last refresh is not acted on. +key_reset +jq '.engagement_token = "t9"' "$_kd/state.json" > "$_kd/state.x" \ + && mv "$_kd/state.x" "$_kd/state.json" +run_key "$s_case" +assert_eq "s passes the token shown, not the state's newer one" \ + "stop|$_kd/custom.json|t1" "$(cat "$_kd/log")" +rm -rf "$_kd" + +# ============================================================ +echo "" +echo "=== 17b. A dashboard that stops short does not end with 0 ===" # Bash 3.2 (macOS's /bin/bash, which the dashboard's shebang runs) gives # an EXIT trap status 0 when set -u stops the script. The dashboard's diff --git a/tests/test_harvest.sh b/tests/test_harvest.sh index f54cdef..c1cff71 100755 --- a/tests/test_harvest.sh +++ b/tests/test_harvest.sh @@ -32,11 +32,20 @@ printf '#!/bin/bash\necho "$*" >> "%s"\nexit 1\n' "$_down_log" \ chmod +x "$_down_bin/docker" PATH="$_down_bin:$PATH" -# Point harvest.sh at a private runtime directory below TMPDIR. +# Point harvest.sh at a private runtime directory below TMPDIR, with an +# engagement recorded as this engine records it before it makes the +# bare repo (record_engagement writes it again). use_runtime() { SWARM_RUNTIME_DIR="$TMPDIR/runtime-$1" export SWARM_RUNTIME_DIR mkdir -m 700 "$SWARM_RUNTIME_DIR" + record_engagement +} +record_engagement() { + jq -n '{schema: "swarm-core.state/v1", engagement_id: "eng-1", + engagement_token: "t1", docker_daemon: "d1"}' \ + > "$SWARM_RUNTIME_DIR/state.json" + chmod 600 "$SWARM_RUNTIME_DIR/state.json" } assert_eq() { @@ -654,12 +663,11 @@ assert_eq "guard prints the diagnostic line" "true" \ assert_eq "guard names short HEAD in diagnostic" "true" \ "$(grep -qE "HEAD:[[:space:]]+${HEAD_BEFORE:0:7}" <<< "$guard_output" \ && echo true || echo false)" -guard_hint=$(printf '%s\n' "$guard_output" | grep '^ docker run ' || true) -guard_hint="${guard_hint# docker run }" -eval "set -- $guard_hint" -assert_eq "guard names the bare path in remediation" \ - "$(dirname "$GUARD_BARE"):/runtime /runtime/upstream.git" \ - "${3:-} ${7:-}" +assert_eq "guard sends the user to cleanup --allow-unharvested" "true" \ + "$(grep -q 'launch.sh cleanup --allow-unharvested' <<< "$guard_output" \ + && echo true || echo false)" +assert_eq "guard prints no removal command" "0" \ + "$(grep -cE 'docker run|rm -rf' <<< "$guard_output" || true)" assert_eq "no merge commit produced" "$HEAD_BEFORE" "$HEAD_AFTER" assert_eq "_agent-harvest remote cleaned up after failure" "" \ "$(git remote | grep -E '^_agent-harvest$' || true)" @@ -798,7 +806,7 @@ assert_eq "another checkout's runtime stops harvest" "1" "$owner_rc" assert_eq "message names SWARM_PROJECT" "true" \ "$(grep -q SWARM_PROJECT <<< "$owner_output" \ && echo true || echo false)" -rm -f "$SWARM_RUNTIME_DIR/state.json" +record_engagement # ============================================================ echo "" @@ -1012,7 +1020,8 @@ done echo "" echo "=== 19. Behavioural: harvest.sh checks the engagement ===" -jq -n '{schema: "swarm-core.state/v1", engagement_id: "eng-1"}' \ +jq -n '{schema: "swarm-core.state/v1", engagement_id: "eng-1", + engagement_token: "t1", docker_daemon: "d1"}' \ > "$SWARM_RUNTIME_DIR/state.json" cd "$DOWN_WORK" eng_rc=0 @@ -1023,7 +1032,7 @@ eng_rc=0 SWARM_ENGAGEMENT_ID=eng-1 bash "$HARVEST_SH" --dry >/dev/null 2>&1 \ || eng_rc=$? assert_eq "harvest joins the recorded engagement" "0" "$eng_rc" -rm -f "$SWARM_RUNTIME_DIR/state.json" +record_engagement # ============================================================ echo "" @@ -1042,6 +1051,67 @@ assert_eq "harvest runs under the caller's lock (SWARM_LOCK_FD)" "0" \ "$lock_rc" exec 8>&- +# ============================================================ +echo "" +echo "=== 21. Behavioural: harvest.sh and the engagement it was started for ===" + +# An engagement an earlier release started (no Docker daemon recorded) +# is refused before the checkout is touched, with the way to finish it; +# the Docker daemon is down here, and the check needs only the state +# file. +cd "$DOWN_WORK" +jq -n '{schema: "swarm-core.state/v1", engagement_id: "eng-1"}' \ + > "$SWARM_RUNTIME_DIR/state.json" +refs_before=$(git for-each-ref) +old_rc=0 +old_out=$(bash "$HARVEST_SH" 2>&1) || old_rc=$? +assert_eq "an earlier release's engagement: harvest refused" "1" "$old_rc" +assert_eq "saying how to finish it" "true" \ + "$(grep -qF 'Finish it with the release that made it' <<< "$old_out" \ + && echo true || echo false)" +assert_eq "the checkout's refs are as they were" "$refs_before" \ + "$(git for-each-ref)" +assert_eq "no harvest remote is left" "" \ + "$(git remote | grep -E '^_agent-harvest$' || true)" +# A token names the engagement harvest.sh was started for. +jq -n '{schema: "swarm-core.state/v1", engagement_id: "eng-1", + engagement_token: "t1", docker_daemon: "d1"}' \ + > "$SWARM_RUNTIME_DIR/state.json" +tok_rc=0 +tok_out=$(SWARM_ENGAGEMENT_TOKEN=t2 bash "$HARVEST_SH" --dry 2>&1) \ + || tok_rc=$? +assert_eq "another engagement's token: harvest refused" "1" "$tok_rc" +assert_eq "saying the engagement is gone" "true" \ + "$(grep -qF 'the engagement this command was started for is gone' \ + <<< "$tok_out" && echo true || echo false)" +tok_rc=0 +SWARM_ENGAGEMENT_TOKEN=t1 bash "$HARVEST_SH" --dry >/dev/null 2>&1 \ + || tok_rc=$? +assert_eq "its own token: harvest runs" "0" "$tok_rc" +# This engine records the engagement before it makes the bare repo, so +# a bare repo with none recorded is an earlier release's too: with no +# state file, and with one that holds only the checkout's claim. +for _st in none claim; do + case "$_st" in + none) rm -f "$SWARM_RUNTIME_DIR/state.json" ;; + claim) + jq -n --arg r "$(cd -P "$DOWN_WORK" && pwd -P)" \ + '{schema: "swarm-core.state/v1", repo_root: $r}' \ + > "$SWARM_RUNTIME_DIR/state.json" ;; + esac + refs_before=$(git for-each-ref) + un_rc=0 + un_out=$(bash "$HARVEST_SH" 2>&1) || un_rc=$? + assert_eq "a bare repo, no engagement ($_st): harvest refused" "1" \ + "$un_rc" + assert_eq "as an earlier release's ($_st)" "true" \ + "$(grep -qF 'Finish it with the release that made it' \ + <<< "$un_out" && echo true || echo false)" + assert_eq "the checkout's refs are as they were ($_st)" \ + "$refs_before" "$(git for-each-ref)" +done +record_engagement + # ============================================================ echo "" echo "===============================" diff --git a/tests/test_launch.sh b/tests/test_launch.sh index 3658465..7bc6365 100755 --- a/tests/test_launch.sh +++ b/tests/test_launch.sh @@ -16,6 +16,64 @@ TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" # shellcheck source=../lib/project.sh source "$TESTS_DIR/../lib/project.sh" +# The fake docker CLIs below hand "docker volume" to this script, which +# keeps each volume's labels in a file, so launch.sh can create and +# check workspace volumes. FAKE_VOLUME_FAIL=create makes create fail; +# FAKE_VOLUME_FAIL=labels makes inspect show the labels with another +# nonce, as for a volume of that name another run made; +# FAKE_VOLUME_FAIL=exists makes every name taken; FAKE_VOLUME_FAIL=error +# makes inspect fail as when the daemon is gone. +FAKE_DOCKER_VOLUME="$TMPDIR/fake-docker-volume" +FAKE_DOCKER_VOLUME_DIR="$TMPDIR/fake-volumes" +export FAKE_DOCKER_VOLUME FAKE_DOCKER_VOLUME_DIR +mkdir -p "$FAKE_DOCKER_VOLUME_DIR" +cat > "$FAKE_DOCKER_VOLUME" <<'EOF' +#!/bin/bash +d="$FAKE_DOCKER_VOLUME_DIR" +shift +case "$1" in + inspect) + shift + fmt="" + if [ "$1" = -f ]; then fmt="$2"; shift 2; fi + case "${FAKE_VOLUME_FAIL:-}" in + error) + echo "Cannot connect to the Docker daemon" >&2 + exit 1 ;; + exists) + [ -n "$fmt" ] || { echo "[]"; exit 0; } ;; + esac + if [ ! -f "$d/$1" ]; then + echo "Error response from daemon: get $1: no such volume" >&2 + exit 1 + fi + if [ "${FAKE_VOLUME_FAIL:-}" = labels ]; then + jq -c '.["swarm-core.nonce"] = "other"' "$d/$1" + elif [ -n "$fmt" ]; then + cat "$d/$1" + else + echo "[]" + fi ;; + create) + shift + [ "${FAKE_VOLUME_FAIL:-}" = create ] && exit 1 + labels='{}' + while [ $# -gt 1 ]; do + if [ "$1" = --label ]; then + labels=$(printf '%s' "$labels" | jq -c --arg k "${2%%=*}" \ + --arg v "${2#*=}" '.[$k] = $v') + shift 2 + continue + fi + shift + done + printf '%s\n' "$labels" > "$d/$1" + echo "$1" ;; +esac +exit 0 +EOF +chmod +x "$FAKE_DOCKER_VOLUME" + assert_eq() { local label="$1" expected="$2" actual="$3" if [ "$expected" = "$actual" ]; then @@ -842,20 +900,21 @@ echo "" echo "=== 18. Swarmfile is required ===" # launch.sh start without a config should fail with a clear message. -# Must run from a git repo (launch.sh needs git rev-parse). -# Requires docker in PATH (check_deps runs before config check). -if command -v docker &>/dev/null; then - _no_cfg_dir=$(mktemp -d) - git -C "$_no_cfg_dir" init -q - out=$(cd "$_no_cfg_dir" && SWARM_CONFIG="" bash "$TESTS_DIR/../launch.sh" start 2>&1) \ - && rc=0 || rc=$? - rm -rf "$_no_cfg_dir" - assert_eq "no config exits nonzero" "1" "$rc" - assert_eq "no config says swarmfile" "true" \ - "$([[ "$out" == *"swarmfile"* ]] && echo true || echo false)" -else - echo " SKIP: docker not available (macOS CI)" -fi +# Must run from a git repo (launch.sh needs git rev-parse). The checks +# before it ask Docker for its context and containers: a fake docker +# answers, with none. +_no_cfg_dir=$(mktemp -d) +_no_cfg_bin=$(mktemp -d) +printf '#!/bin/bash\n[ "$1 $2" = "context show" ] && echo default\nexit 0\n' \ + > "$_no_cfg_bin/docker" +chmod +x "$_no_cfg_bin/docker" +git -C "$_no_cfg_dir" init -q +out=$(cd "$_no_cfg_dir" && PATH="$_no_cfg_bin:$PATH" SWARM_CONFIG="" \ + bash "$TESTS_DIR/../launch.sh" start 2>&1) && rc=0 || rc=$? +rm -rf "$_no_cfg_dir" "$_no_cfg_bin" +assert_eq "no config exits nonzero" "1" "$rc" +assert_eq "no config says swarmfile" "true" \ + "$([[ "$out" == *"swarmfile"* ]] && echo true || echo false)" # ============================================================ echo "" @@ -1365,9 +1424,9 @@ _hl_mirror="$_hl_runtime/mirrors/sub.git" ( # shellcheck source=/dev/null source "$TMPDIR/hl_funcs.sh" - # The extracted functions call this stub. + # No Docker here: the daemon id is made up. # shellcheck disable=SC2329 - rm_docker_dir() { rm -rf "$1"; } + swarm_docker_daemon_id() { printf 'TEST:DAEMON'; } REPO_ROOT="$_hl_dir/src" BARE_REPO="$_hl_dir/bare.git" MIRROR_DIR="$_hl_runtime/mirrors" @@ -1534,184 +1593,6 @@ assert_eq "missing signing_key -> non-zero exit" "nonzero" "$_missing_rc" rm -rf "$_sk_dir" "$_sk_home" -# ============================================================ -echo "" -echo "=== 37. bare preflight: stale vs unharvested ===" - -# Mirrors the divergence guard in launch.sh. The `--is-ancestor` -# check runs in the local repo (not in the bare) so the stale -# case -- where LOCAL_HEAD is a commit that only exists in local -# -- resolves correctly; running the check inside the bare would -# fail to resolve LOCAL_HEAD and collapse stale into unharvested. -# Returns 0 when the guard would allow the run, 1 (with an ERROR -# line on stderr) when it would refuse. -check_bare_preflight() { - local bare="$1" local_repo="$2" - [ -d "$bare" ] || return 0 - local bare_head local_head - bare_head=$(git -C "$bare" rev-parse --verify --quiet \ - refs/heads/agent-work 2>/dev/null || true) - local_head=$(git -C "$local_repo" rev-parse HEAD \ - 2>/dev/null || true) - [ -z "$bare_head" ] && return 0 - [ "$bare_head" = "$local_head" ] && return 0 - if git -C "$local_repo" merge-base --is-ancestor \ - "$bare_head" HEAD 2>/dev/null; then - echo "ERROR: ${bare} is stale (agent-work" \ - "${bare_head:0:7} behind local HEAD" \ - "${local_head:0:7})." >&2 - echo " Remove it to start a fresh run from" \ - "current HEAD:" >&2 - echo " $(swarm_bare_rm_command "$(dirname "$bare")")" >&2 - else - echo "ERROR: ${bare} has unharvested agent commits" \ - "(agent-work ${bare_head:0:7} vs local HEAD" \ - "${local_head:0:7})." >&2 - echo " Run harvest.sh first, or if you've" \ - "already integrated those commits:" >&2 - echo " $(swarm_bare_rm_command "$(dirname "$bare")")" >&2 - fi - return 1 -} - -# Build a local repo with commits A, B on HEAD (where B is -# LOCAL_HEAD). A bare clone taken at B mirrors the happy path; -# moving bare's agent-work forward to a fresh commit C created -# in the bare itself models the unharvested case; advancing -# local past B to D models the stale case; combining both models -# divergence. -_bp_local="$TMPDIR/bp-local" -_bp_bare="$TMPDIR/bp-bare.git" -git init -q -b main "$_bp_local" -git -C "$_bp_local" -c user.name=t -c user.email=t@t \ - commit --allow-empty -q -m "A" -git -C "$_bp_local" -c user.name=t -c user.email=t@t \ - commit --allow-empty -q -m "B" -_bp_B=$(git -C "$_bp_local" rev-parse HEAD) - -# ------ 37.1 equal (guard does not fire) ------ -git clone -q --bare "$_bp_local" "$_bp_bare" -git -C "$_bp_bare" branch -q agent-work "$_bp_B" 2>/dev/null \ - || git -C "$_bp_bare" update-ref refs/heads/agent-work "$_bp_B" -if check_bare_preflight "$_bp_bare" "$_bp_local" 2>/dev/null; then - _bp_eq_rc="zero" -else - _bp_eq_rc="nonzero" -fi -assert_eq "equal BARE_HEAD / LOCAL_HEAD -> guard allows run" \ - "zero" "$_bp_eq_rc" - -# ------ 37.2 unharvested (bare has a commit local doesn't) ------ -# Create commit C inside the bare on agent-work, mirroring an -# agent push. Uses `git -C $bare commit-tree` with B's tree so -# the new commit's object lives only in the bare's objects/. -# Pass -c user.{name,email} explicitly: commit-tree refuses to -# stamp without an identity, and CI runners may have neither a -# global git config nor a usable getpwuid gecos fallback. -_bp_C=$(git -c user.name=t -c user.email=t@t \ - -C "$_bp_bare" commit-tree \ - -p "$_bp_B" -m "C (agent)" "${_bp_B}^{tree}") -git -C "$_bp_bare" update-ref refs/heads/agent-work "$_bp_C" -_bp_unh_err=$(check_bare_preflight "$_bp_bare" "$_bp_local" \ - 2>&1 >/dev/null || true) -assert_eq "unharvested -> 'has unharvested agent commits'" "1" \ - "$(echo "$_bp_unh_err" \ - | grep -cE 'has unharvested agent commits' || true)" -assert_eq "unharvested -> names short BARE_HEAD" "1" \ - "$(echo "$_bp_unh_err" \ - | grep -cE "agent-work ${_bp_C:0:7}" || true)" -assert_eq "unharvested -> names short LOCAL_HEAD" "1" \ - "$(echo "$_bp_unh_err" \ - | grep -cE "local HEAD ${_bp_B:0:7}" || true)" -assert_eq "unharvested -> removal through Docker still offered" "1" \ - "$(echo "$_bp_unh_err" \ - | grep -cF "alpine rm -rf /runtime/upstream.git" || true)" -if check_bare_preflight "$_bp_bare" "$_bp_local" \ - >/dev/null 2>&1; then - _bp_unh_rc="zero" -else - _bp_unh_rc="nonzero" -fi -assert_eq "unharvested -> non-zero exit" "nonzero" "$_bp_unh_rc" - -# ------ 37.3 stale (local has commit D that bare doesn't) ------ -# Reset bare back to B; advance local to D. -git -C "$_bp_bare" update-ref refs/heads/agent-work "$_bp_B" -git -C "$_bp_local" -c user.name=t -c user.email=t@t \ - commit --allow-empty -q -m "D (local-only)" -_bp_D=$(git -C "$_bp_local" rev-parse HEAD) -_bp_stale_err=$(check_bare_preflight "$_bp_bare" "$_bp_local" \ - 2>&1 >/dev/null || true) -assert_eq "stale -> 'is stale' wording" "1" \ - "$(echo "$_bp_stale_err" \ - | grep -cE 'is stale \(agent-work' || true)" -assert_eq "stale -> 'behind local HEAD' wording" "1" \ - "$(echo "$_bp_stale_err" \ - | grep -cE 'behind local HEAD' || true)" -assert_eq "stale -> leads with removal as remediation" "1" \ - "$(echo "$_bp_stale_err" \ - | grep -cE 'Remove it to start a fresh run' || true)" -assert_eq "stale -> does NOT instruct to run harvest.sh" "0" \ - "$(echo "$_bp_stale_err" \ - | grep -cE 'Run harvest\.sh first' || true)" -assert_eq "stale -> names short BARE_HEAD" "1" \ - "$(echo "$_bp_stale_err" \ - | grep -cE "agent-work ${_bp_B:0:7}" || true)" -assert_eq "stale -> names short LOCAL_HEAD" "1" \ - "$(echo "$_bp_stale_err" \ - | grep -cE "local HEAD ${_bp_D:0:7}" || true)" -if check_bare_preflight "$_bp_bare" "$_bp_local" \ - >/dev/null 2>&1; then - _bp_stale_rc="zero" -else - _bp_stale_rc="nonzero" -fi -assert_eq "stale -> non-zero exit" "nonzero" "$_bp_stale_rc" - -# ------ 37.4 divergent (each side has commits the other lacks) ------ -# Give bare its own E on top of B while local still sits on D. -_bp_E=$(git -c user.name=t -c user.email=t@t \ - -C "$_bp_bare" commit-tree \ - -p "$_bp_B" -m "E (agent)" "${_bp_B}^{tree}") -git -C "$_bp_bare" update-ref refs/heads/agent-work "$_bp_E" -_bp_div_err=$(check_bare_preflight "$_bp_bare" "$_bp_local" \ - 2>&1 >/dev/null || true) -# Divergent collapses into the unharvested branch per spec -- -# the goal is only that the operator sees both SHAs and knows -# rm -rf is available. -assert_eq "divergent -> falls through to unharvested wording" "1" \ - "$(echo "$_bp_div_err" \ - | grep -cE 'has unharvested agent commits' || true)" -assert_eq "divergent -> names short BARE_HEAD" "1" \ - "$(echo "$_bp_div_err" \ - | grep -cE "agent-work ${_bp_E:0:7}" || true)" -assert_eq "divergent -> names short LOCAL_HEAD" "1" \ - "$(echo "$_bp_div_err" \ - | grep -cE "local HEAD ${_bp_D:0:7}" || true)" - -# ------ 37.5 bare absent (guard is a no-op) ------ -rm -rf "$_bp_bare" -if check_bare_preflight "$_bp_bare" "$_bp_local" \ - >/dev/null 2>&1; then - _bp_abs_rc="zero" -else - _bp_abs_rc="nonzero" -fi -assert_eq "bare missing -> guard is a no-op" "zero" "$_bp_abs_rc" - -# ------ 37.6 bare exists but agent-work ref missing (guard no-op) ------ -git init -q --bare "$_bp_bare" -if check_bare_preflight "$_bp_bare" "$_bp_local" \ - >/dev/null 2>&1; then - _bp_noref_rc="zero" -else - _bp_noref_rc="nonzero" -fi -assert_eq "bare without agent-work ref -> guard is a no-op" \ - "zero" "$_bp_noref_rc" - -rm -rf "$_bp_local" "$_bp_bare" - # ============================================================ echo "" echo "=== 38. compute_swarm_agents helper (live function from launch.sh) ===" @@ -1861,10 +1742,11 @@ echo "=== 39. build_image is wired into launch entrypoints ===" # rebuild the image with build-args derived from its own config so # the layer cache invalidates correctly when the driver set differs # from a prior cmd_start. Catches accidental removal of either -# call site. +# call site. post-process and interactive build in each of their two +# ways (creating the engagement, joining it), start in its one. _call_sites=$(grep -cE '^[[:space:]]+build_image[[:space:]]*$' \ "$_LAUNCH_SH" || true) -assert_eq "build_image has 3 call sites" "3" "$_call_sites" +assert_eq "build_image has 5 call sites" "5" "$_call_sites" _pp_calls=$(awk ' /^cmd_post_process\(\)[[:space:]]*\{/ { p = 1; next } @@ -1872,7 +1754,7 @@ _pp_calls=$(awk ' p && /^[[:space:]]+build_image[[:space:]]*$/ { c++ } END { print c + 0 } ' "$_LAUNCH_SH") -assert_eq "cmd_post_process calls build_image" "1" "$_pp_calls" +assert_eq "cmd_post_process calls build_image in both ways" "2" "$_pp_calls" _start_calls=$(awk ' /^cmd_start\(\)[[:space:]]*\{/ { p = 1; next } @@ -1883,12 +1765,19 @@ _start_calls=$(awk ' assert_eq "cmd_start calls build_image" "1" "$_start_calls" _interactive_calls=$(awk ' - /^cmd_interactive\(\)[[:space:]]*\{/ { p = 1; next } + /^ensure_bare_repo_for_interactive\(\)[[:space:]]*\{/ { p = 1; next } p && /^\}[[:space:]]*$/ { p = 0 } p && /^[[:space:]]+build_image[[:space:]]*$/ { c++ } END { print c + 0 } ' "$_LAUNCH_SH") -assert_eq "cmd_interactive calls build_image" "1" "$_interactive_calls" +assert_eq "interactive calls build_image in both ways" "2" \ + "$_interactive_calls" +assert_eq "cmd_interactive goes through that step" "1" "$(awk ' + /^cmd_interactive\(\)[[:space:]]*\{/ { p = 1; next } + p && /^\}[[:space:]]*$/ { p = 0 } + p && /^[[:space:]]+ensure_bare_repo_for_interactive[[:space:]]*$/ { c++ } + END { print c + 0 } +' "$_LAUNCH_SH")" # build_image must thread both the SWARM_AGENTS union and the # version pins; missing any of these would silently produce a @@ -1927,12 +1816,12 @@ echo "=== 40. cmd_stop gives the harness time to push ===" # flag's presence, the default, and the env-var override. LAUNCH_FILE="$TESTS_DIR/../launch.sh" CMD_STOP_BODY=$(awk '/^cmd_stop\(\) \{/,/^\}$/' "$LAUNCH_FILE") -STOP_OWN_BODY=$(awk '/^stop_own_container\(\) \{/,/^\}$/' "$LAUNCH_FILE") -assert_eq "stop_own_container passes -t to docker stop" "1" \ - "$(printf '%s\n' "$STOP_OWN_BODY" \ - | grep -cF 'docker stop -t "$timeout"')" -assert_eq "cmd_stop passes the grace to every stop" "3" \ - "$(printf '%s\n' "$CMD_STOP_BODY" | grep -cF '"$stop_timeout"')" +# cmd_stop checks a copy of the state, which these make. +STOP_HELPERS=$(awk '/^(state_snapshot|swarm_tmp_file)\(\) \{/,/^\}$/' \ + "$LAUNCH_FILE") +assert_eq "cmd_stop passes the grace to docker stop" "1" \ + "$(printf '%s\n' "$CMD_STOP_BODY" \ + | grep -cF 'docker stop -t "$stop_timeout"')" assert_eq "cmd_stop defaults grace to 60s" "1" \ "$(printf '%s\n' "$CMD_STOP_BODY" \ | grep -cF 'stop_timeout="${SWARM_STOP_TIMEOUT:-60}"')" @@ -1948,125 +1837,163 @@ exit 0 FAKE chmod +x "$FAKE_DOCKER_DIR/docker" -# Pull cmd_stop and its helper out of launch.sh into sourceable text -# so we do not have to run the launch.sh argv dispatcher. -cmd_stop_src=$(printf '%s\n%s\n' "$CMD_STOP_BODY" "$STOP_OWN_BODY") - -# Run cmd_stop with a docker function in front of the fake CLI. It -# answers the container list (STOP_LIST, "name project" pairs, all owned -# by this checkout) and the label lookups: every container is this -# checkout's and this project's, with ID "id-", except the names -# in STOP_FOREIGN. +# Run cmd_stop ARGS on the engagement of $_sr/state.json (token tok1 on +# daemon d1 unless set otherwise), with a docker function in front of +# the fake CLI. The running containers are the lines of $_sr/list, +# " ", each with ID "id-"; each +# run starts from them, and a container it stops no longer runs. +# STOP_DAEMON names the daemon Docker talks to (d1); with STOP_SWITCH +# set, the first docker stop makes a new engagement (token tok2, same +# names) take the place of this one. +_sr="$TMPDIR/stop-rt" +mkdir -m 700 "$_sr" +stop_state() { + printf '%s\n' "$1" > "$_sr/state.json" +} run_stop() { + # The containers that run, from $_sr/list; stop takes them out. + cp "$_sr/list" "$_sr/running" ( - eval "$cmd_stop_src" + eval "$CMD_STOP_BODY" + eval "$STOP_HELPERS" + SWARM_TMP_FILES=() CHECKOUT_PATH="/checkout" - STATE_FILE="$TMPDIR/no-such-state.json" + RUNTIME_DIR="$_sr" + STATE_FILE="$_sr/state.json" # Called by cmd_stop and the lib/project.sh helpers. # shellcheck disable=SC2329 docker() { - local last owner + local last a want_t="" want_p="" line n pr ck tk for last; do :; done case "${1:-} ${2:-}" in - "ps -a") - # shellcheck disable=SC2086 - set -- ${STOP_LIST:-} - while [ $# -gt 1 ]; do - jq -nc --arg n "$1" --arg p "$2" \ - --arg o "$CHECKOUT_PATH" \ - '{name: $n, project: $p, owner: $o}' - shift 2 + "info -f") echo "${STOP_DAEMON:-d1}"; return 0 ;; + "context show") echo default; return 0 ;; + "ps -q") + for a; do + case "$a" in + label=swarm-core.engagement-token=*) + want_t="${a#*=}"; want_t="${want_t#*=}" ;; + label=swarm-core.project=*) + want_p="${a#*=}"; want_p="${want_p#*=}" ;; + esac done + while read -r n pr ck tk; do + [ -n "$n" ] || continue + [ "$tk" = "$want_t" ] && [ "$pr" = "$want_p" ] \ + && echo "id-$n" + done < "$_sr/running" return 0 ;; "container inspect") - owner="$CHECKOUT_PATH" - case " ${STOP_FOREIGN:-} " in - *" ${last} "*) owner="/elsewhere" ;; - esac - jq -nc --arg id "id-${last}" --arg o "$owner" \ - --arg p "$PROJECT" \ - '{id: $id, labels: {"swarm-core.checkout": $o, - "swarm-core.project": $p}}' - return 0 ;; + while read -r n pr ck tk; do + if [ "id-$n" = "$last" ]; then + echo "/$n|$pr|$ck|$tk" + return 0 + fi + done < "$_sr/running" + return 1 ;; + "stop -t") + # Stopped, it no longer runs. + grep -v "^${last#id-} " "$_sr/running" \ + > "$_sr/running.stopped" || true + mv "$_sr/running.stopped" "$_sr/running" + if [ -n "${STOP_SWITCH:-}" ] \ + && [ ! -e "$_sr/switched" ]; then + : > "$_sr/switched" + sed 's/ tok1$/ tok2/' "$_sr/running" > "$_sr/running.new" + mv "$_sr/running.new" "$_sr/running" + jq -c '.engagement_token = "tok2"' \ + "$_sr/state.json" > "$_sr/state.new" + mv "$_sr/state.new" "$_sr/state.json" + fi ;; esac command docker "$@" } - PATH="$FAKE_DOCKER_DIR:$PATH" cmd_stop >/dev/null 2>&1 || true + # cmd_stop exits on a refusal: a subshell of its own keeps that + # from ending this one before the status is noted. + (PATH="$FAKE_DOCKER_DIR:$PATH" cmd_stop "$@") > "$TMPDIR/stop.out" \ + 2>&1 || echo "rc=1" >> "$TMPDIR/stop.out" ) } +_stop_engaged='{"schema":"swarm-core.state/v1","engagement_id":"e1","engagement_token":"tok1","docker_daemon":"d1"}' +stopped() { + grep -c "^stop -t ${1:-60} " "$trap_dir/calls" || true +} -# Default grace: 60 for the agents, the post-process container and the -# interactive containers. +# Default grace: 60 for every container of the engagement (numbered +# agents, post-process, interactive). +stop_state "$_stop_engaged" +printf '%s\n' "fake-agent-1 fake /checkout tok1" \ + "fake-agent-2 fake /checkout tok1" "fake-agent-post fake /checkout tok1" \ + "fake-agent-interactive-codex-1 fake /checkout tok1" > "$_sr/list" : > "$trap_dir/calls" -NUM_AGENTS=3 PROJECT="claude-swarm-fake" IMAGE_NAME="claude-swarm-fake-agent" \ - STOP_LIST="claude-swarm-fake-agent-interactive-codex-1 claude-swarm-fake -claude-swarm-fake-agent-interactive-claude-2 claude-swarm-fake" run_stop -default_calls=$(wc -l < "$trap_dir/calls" | tr -d ' ') -assert_eq "default grace: 3 agents plus 2 interactive plus post" \ - "6" "$default_calls" -default_flag=$(head -1 "$trap_dir/calls") -assert_eq "default grace: -t 60 in flags" "1" \ - "$(printf '%s\n' "$default_flag" | grep -cF 'stop -t 60')" -assert_eq "default grace: post-process is stopped by ID" "1" \ - "$(grep -cF 'stop -t 60 id-claude-swarm-fake-agent-post' \ - "$trap_dir/calls" || true)" -assert_eq "default grace: interactive containers are stopped" "2" \ - "$(grep -cF 'stop -t 60 id-claude-swarm-fake-agent-interactive-' \ - "$trap_dir/calls" || true)" - -# New project name with env override: SWARM_STOP_TIMEOUT=120. +PROJECT=fake IMAGE_NAME=fake-agent run_stop +assert_eq "default grace: every container of the engagement" "4" \ + "$(stopped 60)" +assert_eq "default grace: by ID" "1" \ + "$(grep -cF 'stop -t 60 id-fake-agent-post' "$trap_dir/calls" || true)" : > "$trap_dir/calls" -NUM_AGENTS=2 PROJECT="swarm-core-fake" IMAGE_NAME="swarm-core-fake-agent" \ - STOP_LIST="swarm-core-fake-agent-interactive-codex-1 swarm-core-fake" \ - SWARM_STOP_TIMEOUT=120 run_stop -override_flag=$(head -1 "$trap_dir/calls") -assert_eq "SWARM_STOP_TIMEOUT=120 propagates to docker stop -t 120" "1" \ - "$(printf '%s\n' "$override_flag" | grep -cF 'stop -t 120')" -assert_eq "SWARM_STOP_TIMEOUT=120 propagates to post-process" "1" \ - "$(grep -cF 'stop -t 120 id-swarm-core-fake-agent-post' \ - "$trap_dir/calls" || true)" -assert_eq "SWARM_STOP_TIMEOUT=120 propagates to interactive" "1" \ - "$(grep -cF 'stop -t 120 id-swarm-core-fake-agent-interactive-codex-1' \ - "$trap_dir/calls" || true)" - -# A '.' in the project id matches only itself: axb's interactive -# container is not a.b's. +PROJECT=fake IMAGE_NAME=fake-agent SWARM_STOP_TIMEOUT=120 run_stop +assert_eq "SWARM_STOP_TIMEOUT=120 propagates to docker stop -t 120" "4" \ + "$(stopped 120)" +# --numbered: only -. : > "$trap_dir/calls" -NUM_AGENTS=1 PROJECT="a.b" IMAGE_NAME="a.b-agent" \ - STOP_LIST="a.b-agent-interactive-x a.b axb-agent-interactive-y axb" \ - run_stop -assert_eq "dotted id: its own interactive container is stopped" "1" \ - "$(grep -cF 'stop -t 60 id-a.b-agent-interactive-x' \ - "$trap_dir/calls" || true)" -assert_eq "dotted id: a look-alike name is not stopped" "0" \ - "$(grep -cF 'axb' "$trap_dir/calls" || true)" - -# A container that another checkout created under one of the names -# after the check at the top is skipped, by its label. +PROJECT=fake IMAGE_NAME=fake-agent run_stop --numbered +assert_eq "--numbered stops the numbered agents only" "2" "$(stopped 60)" +assert_eq "--numbered leaves post-process running" "0" \ + "$(grep -cF 'id-fake-agent-post' "$trap_dir/calls" || true)" +# Only this checkout's and this engagement's. +printf '%s\n' "a.b-agent-1 a.b /checkout tok1" \ + "a.b-agent-2 a.b /elsewhere tok1" "a.b-agent-3 a.b /checkout tok9" \ + > "$_sr/list" : > "$trap_dir/calls" -NUM_AGENTS=2 PROJECT="fake" IMAGE_NAME="fake-agent" \ - STOP_FOREIGN="fake-agent-2" run_stop -assert_eq "foreign container under the name is not stopped" "0" \ - "$(grep -cF 'id-fake-agent-2' "$trap_dir/calls" || true)" -assert_eq "own containers are still stopped" "1" \ - "$(grep -cF 'stop -t 60 id-fake-agent-1' "$trap_dir/calls" || true)" - -# Ids can overlap as names: project app-agent-interactive-tools names -# its first agent app-agent-interactive-tools-agent-1. Even from the -# same checkout, app's stop must leave that container alone. +PROJECT=a.b IMAGE_NAME=a.b-agent run_stop +assert_eq "another checkout's container is not stopped" "0" \ + "$(grep -cF 'id-a.b-agent-2' "$trap_dir/calls" || true)" +assert_eq "another engagement's container is not stopped" "0" \ + "$(grep -cF 'id-a.b-agent-3' "$trap_dir/calls" || true)" +assert_eq "this engagement's container is" "1" "$(stopped 60)" +# A new engagement with the same names takes this one's place after the +# first docker stop: none of its containers is stopped. +printf '%s\n' "fake-agent-1 fake /checkout tok1" \ + "fake-agent-2 fake /checkout tok1" "fake-agent-3 fake /checkout tok1" \ + > "$_sr/list" +rm -f "$_sr/switched" : > "$trap_dir/calls" -_tools_name="app-agent-interactive-tools-agent-1" -NUM_AGENTS=1 PROJECT="app" IMAGE_NAME="app-agent" \ - STOP_LIST="${_tools_name} app-agent-interactive-tools" run_stop -assert_eq "overlapping id: the other project's container is not stopped" \ - "0" "$(grep -cF 'interactive-tools' "$trap_dir/calls" || true)" +PROJECT=fake IMAGE_NAME=fake-agent STOP_SWITCH=1 run_stop +assert_eq "an engagement that takes this one's place mid-stop: one stop" \ + "1" "$(stopped 60)" +rm -f "$_sr/switched" +# Refusals, with nothing stopped. +printf '%s\n' "fake-agent-1 fake /checkout tok1" > "$_sr/list" +stop_state "$_stop_engaged" +for _sv in daemon token legacy; do + : > "$trap_dir/calls" + case "$_sv" in + daemon) PROJECT=fake IMAGE_NAME=fake-agent STOP_DAEMON=d2 run_stop ;; + token) PROJECT=fake IMAGE_NAME=fake-agent \ + SWARM_ENGAGEMENT_TOKEN=tok7 run_stop ;; + legacy) + stop_state '{"schema":"swarm-core.state/v1","engagement_id":"e1"}' + PROJECT=fake IMAGE_NAME=fake-agent run_stop ;; + esac + assert_eq "stop refuses (${_sv})" "1" \ + "$(grep -c '^rc=1$' "$TMPDIR/stop.out" || true)" + assert_eq "and stops nothing (${_sv})" "0" "$(stopped 60)" +done +assert_eq "an earlier release's engagement: says how to finish it" "1" \ + "$(grep -c 'release that made it' "$TMPDIR/stop.out" || true)" +stop_state '{"schema":"swarm-core.state/v1"}' +: > "$trap_dir/calls" +PROJECT=fake IMAGE_NAME=fake-agent run_stop +assert_eq "no engagement: nothing to stop" "0" "$(stopped 60)" # stop never creates the runtime directory and leaves the state file in # place, since it names the owner. run_stop_with_runtime() { ( - eval "$cmd_stop_src" - NUM_AGENTS=1 + eval "$CMD_STOP_BODY" + eval "$STOP_HELPERS" + SWARM_TMP_FILES=() IMAGE_NAME="stop-fake" PROJECT="fake" RUNTIME_DIR="$1" @@ -2076,7 +2003,7 @@ run_stop_with_runtime() { # shellcheck disable=SC2329 docker() { return 0; } cmd_stop >/dev/null 2>&1 || true - ) + ) || true } _stop_target="$TMPDIR/stop-target" _stop_rt="$TMPDIR/stop-runtime" @@ -2326,8 +2253,9 @@ assert_eq "no fixed /tmp paths" "" \ echo "" echo "=== 46. Recovery hints quote the bare repo path ===" -# The runtime directory can contain shell metacharacters, so a printed -# removal hint must name the same path when it is pasted. +# A bare repo whose agent-work is behind the checkout's HEAD (stale): +# interactive refuses and points to cleanup, with no removal command, +# even in a runtime directory with shell metacharacters in its path. _hint_fn="$TMPDIR/ensure_bare_repo_for_interactive.sh" awk '/^ensure_bare_repo_for_interactive\(\)[[:space:]]*\{/ { p = 1 } p { print } @@ -2350,14 +2278,24 @@ _hint_out=$( RUNTIME_DIR="$(dirname "$_hint_bare")" BARE_REPO="$_hint_bare" STATE_FILE="$TMPDIR/hint-state.json" + printf '%s\n' '{"schema":"swarm-core.state/v1","engagement_id":"e1", + "engagement_token":"t1","docker_daemon":"d1"}' > "$STATE_FILE" + # Called by the engagement's daemon check. + # shellcheck disable=SC2329 + docker() { + # (Patterns with their opening parenthesis: Bash 3.2 cannot + # parse a case inside $(...) otherwise.) + case "$1 ${2:-}" in + ("info -f") echo d1 ;; + ("context show") echo default ;; + esac + } ensure_bare_repo_for_interactive 2>&1 ) || true -_hint=$(printf '%s\n' "$_hint_out" | grep 'docker run ' || true) -_hint="${_hint#*docker run }" -eval "set -- $_hint" -assert_eq "stale bare hint quotes the path for the shell" \ - "$(dirname "$_hint_bare"):/runtime /runtime/upstream.git" \ - "${3:-} ${7:-}" +assert_eq "a stale bare repo: the hint names cleanup" "1" \ + "$(printf '%s\n' "$_hint_out" | grep -c 'launch.sh cleanup' || true)" +assert_eq "and offers no removal command" "0" \ + "$(printf '%s\n' "$_hint_out" | grep -cE 'docker run|rm -rf' || true)" # ============================================================ echo "" @@ -2414,22 +2352,17 @@ _st_bin="$TMPDIR/start-bin" _st_repo="$TMPDIR/start-repo" _st_rt="$TMPDIR/start-runtime" mkdir -p "$_st_bin" "$_st_repo/sub/configs" -# Like the real CLI, inspecting a missing container fails, info and -# context show answer, and the alpine rm -rf that rm_docker_dir runs -# removes its last argument. +# Like the real CLI, inspecting a missing container fails, and info and +# context show answer. cat > "$_st_bin/docker" <<'EOF' #!/bin/bash +[ "$1" = volume ] && exec "$FAKE_DOCKER_VOLUME" "$@" if [ "$1 $2" = "container inspect" ]; then echo "Error: No such container: ${*: -1}" >&2 exit 1 fi [ "$1" = info ] && echo d1 [ "$1 $2" = "context show" ] && echo default -case " $* " in - " run --rm "*" alpine rm -rf "*) - for _a in "$@"; do _last="$_a"; done - rm -rf "$_last" ;; -esac exit 0 EOF chmod +x "$_st_bin/docker" @@ -2510,6 +2443,9 @@ _lbl_log="$TMPDIR/label-docker.log" mkdir -p "$_lbl_bin" : > "$_lbl_log" printf '#!/bin/bash\nprintf "%%s\\n" "$*" >> "%s" +[ "$1" = volume ] && exec "$FAKE_DOCKER_VOLUME" "$@" +[ "$1" = info ] && { echo d1; exit 0; } +[ "$1 $2" = "context show" ] && { echo default; exit 0; } [ "$1 $2" = "container inspect" ] && exit 1\nexit 0\n' \ "$_lbl_log" > "$_lbl_bin/docker" chmod +x "$_lbl_bin/docker" @@ -2572,6 +2508,9 @@ mkdir -p "$_id_bin" : > "$_id_log" cat > "$_id_bin/docker" <> "$_id_log" if [ "\$1" = ps ]; then jq -nc --arg o "$(cd -P "$_id_a" && pwd -P)" \ @@ -2598,6 +2537,15 @@ assert_eq "B creates no runtime directory" "false" \ # another checkout owns it. start must not remove it. cat > "$_id_bin/docker" <> "$_id_log" + echo "docker: Error response from daemon: Conflict." >&2 + exit 125 +fi echo "\$*" >> "$_id_log" if [ "\$1 \$2" = "container inspect" ]; then jq -nc --arg o "/elsewhere/start-repo" \ @@ -2612,7 +2560,8 @@ _id_rc=0 SWARM_RUNTIME_DIR="$TMPDIR/id-b-runtime-2" \ SWARM_CONFIG=sub/configs/swarm.json SWARM_SKIP_DEP_CHECK=1 \ bash "$_LAUNCH_SH" start >/dev/null 2>&1) || _id_rc=$? -assert_eq "a container that appears later stops start" "1" "$_id_rc" +assert_eq "a container that appears later stops start" "1" \ + "$([ "$_id_rc" -ne 0 ] && echo 1 || echo 0)" assert_eq "the late container is not removed" "0" \ "$(grep -c '^rm ' "$_id_log" || true)" assert_eq "no agent container is started" "0" \ @@ -2623,6 +2572,9 @@ assert_eq "no agent container is started" "0" \ # logs, status and wait must not read it. cat > "$_id_bin/docker" <> "$_id_log" if [ "\$1 \$2" = "container inspect" ]; then jq -nc '{id: "other-id", labels: {"swarm-core.project": "other", @@ -2691,6 +2643,9 @@ _eng_log="$TMPDIR/eng-docker.log" _eng_bin="$TMPDIR/eng-bin" mkdir -p "$_eng_bin" printf '#!/bin/bash\necho "$*" >> "%s" +[ "$1" = volume ] && exec "$FAKE_DOCKER_VOLUME" "$@" +[ "$1" = info ] && { echo d1; exit 0; } +[ "$1 $2" = "context show" ] && { echo default; exit 0; } [ "$1 $2" = "container inspect" ] && exit 1\nexit 0\n' \ "$_eng_log" > "$_eng_bin/docker" chmod +x "$_eng_bin/docker" @@ -2719,6 +2674,9 @@ mkdir -p "$_lk_bin" mkdir -m 700 "$_lk_rt" cat > "$_lk_bin/docker" <> "$_lk_log" [ "\$1 \$2" = "container inspect" ] && exit 1 # docker run -it checks that the session record for its --name names @@ -2807,8 +2765,8 @@ assert_eq "docker_args reach docker run unchanged" "1" \ assert_eq "the container gets the recorded name" "1" \ "$(grep -c 'record names the session' "$_lk_log" || true)" assert_eq "the session label follows docker_args too" "1" \ - "$(grep '^run -it ' "$_lk_log" \ - | grep -c -- ' --name custom-session --name [^ ]* --label swarm-core.session=' \ + "$(grep '^run -it ' "$_lk_log" | grep -cE -- \ + ' --name custom-session .* --name [^ ]+ --label swarm-core\.session=' \ || true)" rm -f "$_lk_cfg" @@ -2886,9 +2844,9 @@ _br_run() { ( # shellcheck source=/dev/null source "$_br_dir/funcs.sh" - # The extracted functions call this stub. + # No Docker here: the daemon id is made up. # shellcheck disable=SC2329 - rm_docker_dir() { rm -rf "$1"; } + swarm_docker_daemon_id() { printf 'TEST:DAEMON'; } REPO_ROOT="$1" BARE_REPO="$_br_dir/bare-$2.git" STATE_FILE="$_br_dir/state-$2.json" @@ -2912,19 +2870,26 @@ assert_eq "other branches are copied as before" "$_br_old" \ "$(git -C "$_br_bare" rev-parse feature 2>/dev/null)" assert_eq "the bare repo's HEAD is agent-work" "refs/heads/agent-work" \ "$(git -C "$_br_bare" symbolic-ref HEAD 2>/dev/null)" - -# A failure while the clone is prepared must not leave a bare repo that -# later commands would join with old agent branches in it, even when -# removing the clone fails too: rm_docker_dir is a no-op here. FAIL -# names the step that fails. +assert_eq "the state file records the Docker daemon" "TEST:DAEMON" \ + "$(jq -r .docker_daemon "$_br_dir/state-1.json" 2>/dev/null)" +assert_eq "and a token of 16 hex digits" "true" \ + "$(jq -r .engagement_token "$_br_dir/state-1.json" 2>/dev/null \ + | grep -qE '^[0-9a-f]{16}$' && echo true || echo false)" +assert_eq "no upstream.git.new is left" "false" \ + "$([ -e "$_br_bare.new" ] && echo true || echo false)" + +# The engagement is recorded before the clone starts. A failure while +# the clone is prepared must not leave a bare repo that later commands +# would join with old agent branches in it, nor the half-made clone: +# cleanup is the next step. FAIL names the step that fails. _br_fail() { local n="$1" fail="$2" ( # shellcheck source=/dev/null source "$_br_dir/funcs.sh" - # The extracted functions call these stubs. + # No Docker here: the daemon id is made up. # shellcheck disable=SC2329 - rm_docker_dir() { :; } + swarm_docker_daemon_id() { printf 'TEST:DAEMON'; } # shellcheck disable=SC2329 git() { case "$fail $*" in @@ -2933,36 +2898,53 @@ _br_fail() { esac command git "$@" } - # shellcheck disable=SC2329 - swarm_state_update() { - [ "$fail" != state ] - } + if [ "$fail" = state ]; then + # shellcheck disable=SC2329 + swarm_state_update() { return 1; } + fi REPO_ROOT="$_br_src" BARE_REPO="$_br_dir/bare-$n.git" STATE_FILE="$_br_dir/state-$n.json" create_bare_repo ) >/dev/null 2>&1 } -for _br_case in 3:list 4:delete 5:state; do +for _br_case in 3:list:true 4:delete:true 5:state:false; do _br_n="${_br_case%%:*}" _br_step="${_br_case#*:}" + _br_recorded="${_br_step#*:}" + _br_step="${_br_step%%:*}" _br_rc=0 _br_fail "$_br_n" "$_br_step" || _br_rc=$? assert_eq "$_br_step failure stops bare creation" "1" "$_br_rc" assert_eq "$_br_step failure leaves no bare repo to join" "false" \ "$([ -e "$_br_dir/bare-$_br_n.git" ] && echo true || echo false)" - assert_eq "$_br_step failure records no engagement" "false" \ - "$([ -e "$_br_dir/state-$_br_n.json" ] && echo true || echo false)" + assert_eq "$_br_step failure leaves no upstream.git.new" "false" \ + "$([ -e "$_br_dir/bare-$_br_n.git.new" ] && echo true || echo false)" + assert_eq "$_br_step failure: engagement recorded is $_br_recorded" \ + "$_br_recorded" \ + "$([ -n "$(jq -r '.engagement_id // empty' \ + "$_br_dir/state-$_br_n.json" 2>/dev/null)" ] \ + && echo true || echo false)" done -# An old bare repo that cannot be removed is refused, not cloned into. -mkdir -p "$_br_dir/bare-6.git" -touch "$_br_dir/bare-6.git/marker" -_br_rc=0 -_br_fail 6 none || _br_rc=$? -assert_eq "an old bare repo that stays is refused" "1" "$_br_rc" -assert_eq "the old bare repo is left as it was" "true" \ +# A bare repo that is there already is refused, never replaced; so is +# an upstream.git.new that a run stopped half way left. +mkdir -p "$_br_dir/bare-6.git" "$_br_dir/bare-7.git.new" +touch "$_br_dir/bare-6.git/marker" "$_br_dir/bare-7.git.new/marker" +for _br_n in 6 7; do + _br_rc=0 + _br_fail "$_br_n" none || _br_rc=$? + assert_eq "an existing bare repo or .new ($_br_n) is refused" "1" \ + "$_br_rc" + assert_eq "and records no engagement ($_br_n)" "false" \ + "$([ -e "$_br_dir/state-$_br_n.json" ] && echo true || echo false)" +done +assert_eq "the bare repo is left as it was" "true" \ "$([ -e "$_br_dir/bare-6.git/marker" ] && echo true || echo false)" +assert_eq "so is the .new" "true" \ + "$([ -e "$_br_dir/bare-7.git.new/marker" ] && echo true || echo false)" +assert_eq "and no bare repo is made next to the .new" "false" \ + "$([ -e "$_br_dir/bare-7.git" ] && echo true || echo false)" # A checkout without a commit has nothing to start from. git init -q "$_br_dir/empty" @@ -3208,11 +3190,15 @@ rm -f "$_pd_dir/interactive-$_pd_id_b.pending" unset -f docker -# Every place that creates a bare repo checks for open sessions first, -# and interactive mirrors only when it creates the bare repo. -assert_eq "each bare repo creation checks for open sessions" "3" \ +# Every place that creates a bare repo checks for a clean runtime +# (which counts open sessions) first, then builds the image, and +# interactive mirrors only when it creates the bare repo. +assert_eq "each bare repo creation checks for a clean runtime" "3" \ + "$(grep -B2 '^ *create_bare_repo "' "$_LAUNCH_SH" \ + | grep -c '^ *require_clean_runtime$')" +assert_eq "and builds the image between the two" "3" \ "$(grep -B1 '^ *create_bare_repo "' "$_LAUNCH_SH" \ - | grep -c 'refuse_while_interactive_pending')" + | grep -c '^ *build_image$')" assert_eq "cmd_interactive mirrors only through the bare repo step" "0" \ "$(printf '%s\n' "$CMD_INTERACTIVE_BODY" \ | grep -c 'mirror_submodules' || true)" @@ -3239,9 +3225,6 @@ _mm_run() { ( # shellcheck source=/dev/null source "$_mm_dir/funcs.sh" - # The extracted function calls this stub. - # shellcheck disable=SC2329 - rm_docker_dir() { rm -rf "$1"; } REPO_ROOT="$_mm_dir/src" MIRROR_DIR="$_mm_dir/mirrors" mirror_submodules "$@" @@ -3259,14 +3242,27 @@ _mm_run missing assert_eq "a joining session adds a missing mirror" "true" \ "$(git -C "$_mm_dir/mirrors/lib.git" rev-parse --is-bare-repository \ 2>/dev/null || echo false)" +assert_eq "a mirror is made under .new and then renamed" "false" \ + "$([ -e "$_mm_dir/mirrors/lib.git.new" ] && echo true || echo false)" touch "$_mm_dir/mirrors/lib.git/keep" -_mm_run -assert_eq "a new engagement replaces the mirrors" "false" \ +_mm_rc=0 +_mm_run || _mm_rc=$? +assert_eq "a new engagement never replaces a mirror: refused" "1" \ + "$([ "$_mm_rc" -ne 0 ] && echo 1 || echo 0)" +assert_eq "and the mirror stays" "true" \ "$([ -e "$_mm_dir/mirrors/lib.git/keep" ] && echo true || echo false)" +rm -rf "$_mm_dir/mirrors/lib.git" +mkdir "$_mm_dir/mirrors/lib.git.new" +_mm_rc=0 +_mm_run missing || _mm_rc=$? +assert_eq "a .new a stopped run left: refused" "1" \ + "$([ "$_mm_rc" -ne 0 ] && echo 1 || echo 0)" +rmdir "$_mm_dir/mirrors/lib.git.new" # Through launch.sh: with a session open, start leaves the bare repo # alone, and another interactive session still joins it. Once the -# session ends, start replaces the bare repo as before. +# session ends, start still refuses: a new engagement waits for +# ./launch.sh cleanup. _pd_rt="$TMPDIR/pending-runtime" run_pd() { (cd "$_st_repo" && PATH="$_st_bin:$PATH" SWARM_RUNTIME_DIR="$_pd_rt" \ @@ -3290,9 +3286,12 @@ assert_eq "another interactive session joins meanwhile" "0" "$_pd_rc" kill "$_pd_live" 2>/dev/null || true wait "$_pd_live" 2>/dev/null || true _pd_rc=0 -run_pd start >/dev/null || _pd_rc=$? -assert_eq "after the session ends start goes ahead" "0" "$_pd_rc" -assert_eq "and replaces the bare repo" "false" \ +_pd_out=$(run_pd start) || _pd_rc=$? +assert_eq "after the session ends start still refuses" "1" "$_pd_rc" +assert_eq "and says cleanup comes first" "true" \ + "$(grep -q 'launch.sh cleanup' <<< "$_pd_out" \ + && echo true || echo false)" +assert_eq "and the bare repo stays" "true" \ "$([ -e "$_pd_rt/upstream.git/keep" ] && echo true || echo false)" assert_eq "the old record is gone" "false" \ "$([ -e "$_pd_rt/interactive-$_pd_id_a.pending" ] \ @@ -3416,6 +3415,999 @@ else tf_drop fi +# ============================================================ +echo "" +echo "=== 55. Workspace volumes, one daemon, clean runtimes ===" + +# start, post-process, interactive, stop and wait against a fake docker +# that keeps its world in files under V_DIR, one directory per case: +# context the current context, a or b (DOCKER_CONTEXT overrides +# it); the daemon behind context X has id daemon-X +# switch when there, the first "context show" switches the +# current context to b right after it answers, as another +# terminal could +# containers one JSON object per line: id, name, state, labels +# volumes the names "volume ls" prints +# fail calls that fail, one per line: ps (every listing), ps3 +# (from the third listing on), volume-ls, run, build, +# stop (the container keeps running), stop-exits (it +# stops anyway), stop-gone (it is removed meanwhile), +# inspect +# spawn a count: while it is above 0, each container docker stop +# stops makes a new running one of the same engagement +# appear (as a start launching its agents would), and the +# count goes down +# newtoken when there, wait's first listing gives the state file +# (V_STATE) another engagement token, as a cleanup and a +# new start in another terminal would +# at.N containers that replace containers right before wait's +# Nth listing (the polls and the recount under the lock) +# Every call is logged to V_DIR/log as "|". +# volume create and inspect go to the shared fake, which keeps this +# case's volumes in V_DIR/vols. +unset DOCKER_HOST DOCKER_CONTEXT SWARM_ENGAGEMENT_TOKEN +_v_bin="$TMPDIR/v55-bin" +_v_root="$TMPDIR/v55" +mkdir -p "$_v_bin" "$_v_root" +cat > "$_v_bin/docker" <<'EOF' +#!/bin/bash +d="$V_DIR" +ctx="${DOCKER_CONTEXT:-$(cat "$d/context")}" +printf '%s|%s\n' "${DOCKER_CONTEXT:-}" "$*" >> "$d/log" +fails() { grep -qx "$1" "$d/fail" 2>/dev/null; } +case "$1" in + context) + cat "$d/context" + if [ -e "$d/switch" ]; then + rm -f "$d/switch" + echo b > "$d/context" + fi ;; + info) echo "daemon-$ctx" ;; + volume) + if [ "$2" = ls ]; then + fails volume-ls && exit 1 + cat "$d/volumes" 2>/dev/null + exit 0 + fi + FAKE_DOCKER_VOLUME_DIR="$d/vols" exec "$FAKE_DOCKER_VOLUME" "$@" ;; + ps) + n=$(( $(cat "$d/ps-count" 2>/dev/null || echo 0) + 1 )) + echo "$n" > "$d/ps-count" + fails ps && exit 1 + if fails ps3 && [ "$n" -ge 3 ]; then exit 1; fi + shift + all=false quiet=false fmt="" filters="" + while [ $# -gt 0 ]; do + case "$1" in + -a) all=true ;; + -q) quiet=true ;; + --filter) filters="${filters}$2"$'\n'; shift ;; + --format) fmt="$2"; shift ;; + esac + shift + done + if [ "$fmt" = '{{.Names}} {{.State}}' ]; then + w=$(( $(cat "$d/wait-listed" 2>/dev/null || echo 0) + 1 )) + echo "$w" > "$d/wait-listed" + if [ -f "$d/at.$w" ]; then + mv "$d/at.$w" "$d/containers" + fi + fi + if [ "$fmt" = '{{.Names}} {{.State}}' ] && [ -e "$d/newtoken" ]; then + rm -f "$d/newtoken" + jq -c '.engagement_token = "ffffffffffffffff"' "$V_STATE" \ + > "$V_STATE.x" && mv "$V_STATE.x" "$V_STATE" + fi + [ -s "$d/containers" ] || exit 0 + jq -r --argjson all "$all" --argjson quiet "$quiet" \ + --arg fmt "$fmt" --arg filters "$filters" ' + ($filters | split("\n") | map(select(length > 0))) as $f + | select($all or .state == "running") + | . as $c + | select(all($f[]; . as $x + | if startswith("label=") then + $x[6:] as $kv | ($kv | index("=")) as $i + | if $i == null then ($c.labels | has($kv)) + else ($c.labels[$kv[:$i]] // null) == $kv[$i + 1:] + end + elif startswith("name=") then ($c.name | contains($x[5:])) + elif startswith("id=") then ($c.id | startswith($x[3:])) + else false end)) + | if $quiet then .id + elif ($fmt | contains("\"owner\":")) then + {id, name, + project: (.labels["swarm-core.project"] // ""), + owner: (.labels["swarm-core.checkout"] // "")} | tojson + elif $fmt == "{{.State}}" then .state + elif $fmt == "{{.Names}} {{.State}}" then "\(.name) \(.state)" + else .name end' "$d/containers" ;; + container|inspect) + if fails inspect; then + echo "Cannot connect to the Docker daemon" >&2 + exit 1 + fi + [ "$1" = container ] && shift + shift + fmt="" + if [ "$1" = -f ] || [ "$1" = --format ]; then + fmt="$2" + shift 2 + fi + c=$(jq -c --arg t "$1" 'select(.name == $t or .id == $t)' \ + "$d/containers" 2>/dev/null | head -n 1) + if [ -z "$c" ]; then + echo "Error: No such container: $1" >&2 + exit 1 + fi + case "$fmt" in + *'"labels":'*) jq -c '{id, labels}' <<< "$c" ;; + *.State.Status*) jq -r .state <<< "$c" ;; + *.State.Running*) jq -r '.state == "running"' <<< "$c" ;; + *.State.ExitCode*) echo 0 ;; + *'{{.Name}}|'*) + jq -r '"/\(.name)|\(.labels["swarm-core.project"] // "")|\(.labels["swarm-core.checkout"] // "")|\(.labels["swarm-core.engagement-token"] // "")"' <<< "$c" ;; + *) printf '%s\n' "$c" ;; + esac ;; + run) + fails run && exit 125 + case " $* " in + *" -d "*) echo "cid-run" ;; + esac ;; + build) fails build && exit 1 ;; + stop) + id="${*: -1}" + if fails stop-gone; then + jq -c --arg i "$id" 'select(.id != $i)' "$d/containers" \ + > "$d/containers.x" && mv "$d/containers.x" "$d/containers" + exit 42 + fi + if fails stop-exits; then + jq -c --arg i "$id" \ + 'if .id == $i then .state = "exited" else . end' \ + "$d/containers" > "$d/containers.x" \ + && mv "$d/containers.x" "$d/containers" + exit 42 + fi + fails stop && exit 42 + jq -c --arg i "$id" \ + 'if .id == $i then .state = "exited" else . end' \ + "$d/containers" > "$d/containers.x" \ + && mv "$d/containers.x" "$d/containers" + left=$(cat "$d/spawn" 2>/dev/null || echo 0) + if [ "$left" -gt 0 ]; then + echo $((left - 1)) > "$d/spawn" + n=$(( $(wc -l < "$d/containers") + 1 )) + jq -c --arg i "$id" --arg n "$n" 'select(.id == $i) + | .id = ("id-spawn-" + $n) | .state = "running" + | .name = (.name | sub("-[0-9]+$"; "-" + $n))' \ + "$d/containers" > "$d/containers.x" \ + && cat "$d/containers.x" >> "$d/containers" + fi ;; +esac +exit 0 +EOF +chmod +x "$_v_bin/docker" + +_v_checkout=$(swarm_checkout_path "$_st_repo") +_v_img="start-repo-agent" +# Two agents in slim context, a post-process, and docker_args that try +# to replace what the engine sets: auto-remove, a restart policy, a +# name. +V_CFG="$_v_root/swarm.json" +cat > "$V_CFG" <<'EOF' +{ "prompt": "prompt.md", "title": "v55", "driver": "fake", + "docker_args": ["--rm", "--restart", "always", "--name", "other"], + "agents": [{ "count": 2, "model": "fake", "context": "slim" }], + "post_process": { "prompt": "prompt.md", "model": "fake" } } +EOF +_v_n=0 +# A new case: a fresh fake Docker world on context a and a fresh +# runtime directory V_RT. +v_new() { + _v_n=$((_v_n + 1)) + V_DIR="$_v_root/case-$_v_n" + V_RT="$_v_root/rt-$_v_n" + V_STATE="$V_RT/state.json" + mkdir -p "$V_DIR/vols" + echo a > "$V_DIR/context" + : > "$V_DIR/log" +} +# Run launch.sh with ARGS in the start tests' checkout against this +# case's fake Docker; print its output. Variables set on the call +# reach it; the lock descriptor of the test, 8, does not. +v_run() { + (cd "$_st_repo" && PATH="$_v_bin:$PATH" V_DIR="$V_DIR" \ + V_STATE="$V_STATE" SWARM_RUNTIME_DIR="$V_RT" \ + SWARM_CONFIG="$V_CFG" SWARM_SKIP_DEP_CHECK=1 SWARM_WAIT_POLL=0 \ + "$BASH" "$_LAUNCH_SH" "$@" 8>&- 2>&1) +} +# What a refused command must leave as it was: the runtime directory's +# entries (the lock file aside, which taking the lock creates) and the +# state file. +v_snap() { + (cd "$V_RT" 2>/dev/null && find . ! -name engagement.lock -print \ + | LC_ALL=C sort) + cat "$V_STATE" 2>/dev/null || true +} +# Count the logged Docker calls that make, change or remove something. +v_writes() { + grep -cE '^[^|]*[|](run|start|stop|kill|rm|commit|build|volume create|volume rm|container (start|stop|kill|rm)) ' \ + "$V_DIR/log" || true +} +v_has() { + grep -qF -- "$1" <<< "$2" && echo true || echo false +} +# Print the names of this case's volumes that match extended regular +# expression RE. +v_volume() { + local f + for f in "$V_DIR/vols"/*; do + [ -e "$f" ] || continue + f="${f##*/}" + if grep -qE -- "$1" <<< "$f"; then + printf '%s\n' "$f" + fi + done +} +v_container() { + # NAME STATE TOKEN: one container of this checkout. + jq -cn --arg n "$1" --arg s "$2" --arg t "$3" --arg c "$_v_checkout" \ + '{id: ("id-" + $n), name: $n, state: $s, + labels: ({"swarm-core.project": "start-repo", + "swarm-core.checkout": $c} + + (if $t == "" then {} + else {"swarm-core.engagement-token": $t} end))}' \ + >> "$V_DIR/containers" +} + +# --- start: one volume per agent, the pin, the recorded daemon ------- +v_new +touch "$V_DIR/switch" +_v_rc=0 +_v_out=$(v_run start) || _v_rc=$? +assert_eq "start on a clean runtime succeeds" "0" "$_v_rc" +assert_eq "the first Docker call reads the context" "|context show" \ + "$(head -n 1 "$V_DIR/log")" +assert_eq "every later call goes to that context" "0" \ + "$(awk 'NR > 1 && !/^a[|]/ { n++ } END { print n + 0 }' "$V_DIR/log")" +assert_eq "even though the current context changed after that call" "b" \ + "$(cat "$V_DIR/context")" +assert_eq "the engagement records the daemon of the pinned context" \ + "daemon-a" "$(jq -r .docker_daemon "$V_STATE" 2>/dev/null)" +_v_eng=$(jq -r .engagement_id "$V_STATE" 2>/dev/null) +_v_tok=$(jq -r .engagement_token "$V_STATE" 2>/dev/null) +assert_eq "and a token of 16 hex digits" "true" \ + "$(grep -qE '^[0-9a-f]{16}$' <<< "$_v_tok" && echo true || echo false)" +assert_eq "one volume per agent" "2" \ + "$(find "$V_DIR/vols" -type f | wc -l | tr -d ' ')" +for _v_i in 1 2; do + _v_vol=$(v_volume "^${_v_img}-${_v_i}-ws-[0-9a-f]{8}$") + assert_eq "agent $_v_i: its volume is named after it" "true" \ + "$([ -n "$_v_vol" ] && echo true || echo false)" + assert_eq "agent $_v_i: the volume carries every label" "true" \ + "$(jq -e --arg c "$_v_checkout" --arg e "$_v_eng" \ + --arg t "$_v_tok" --arg n "${_v_img}-${_v_i}" ' + .["swarm-core.project"] == "start-repo" + and .["swarm-core.checkout"] == $c + and .["swarm-core.engagement"] == $e + and .["swarm-core.engagement-token"] == $t + and .["swarm-core.container"] == $n + and .["swarm-core.kind"] == "agent" + and .["swarm-core.context"] == "slim" + and (.["swarm-core.nonce"] | test("^[0-9a-f]{16}$"))' \ + "$V_DIR/vols/${_v_vol:-none}" >/dev/null 2>&1 \ + && echo true || echo false)" + assert_eq "agent $_v_i: mount, labels, policies and name follow docker_args" \ + "1" "$(grep -F -- '|run -d ' "$V_DIR/log" | grep -cF -- " --rm --restart always --name other --mount type=volume,src=${_v_vol:-none},dst=/workspace --label swarm-core.engagement=${_v_eng} --label swarm-core.engagement-token=${_v_tok} --restart no --rm=false --name ${_v_img}-${_v_i} " || true)" +done +assert_eq "start builds the image once" "1" \ + "$(grep -c '|build ' "$V_DIR/log" || true)" +_v_start_dir="$V_DIR" +_v_start_rt="$V_RT" + +# --- interactive and post-process join it, each with its volume ------ +# The current context is b now: switch back first. +echo a > "$V_DIR/context" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run interactive --agent-index 1) || _v_rc=$? +assert_eq "interactive joins the engagement" "0" "$_v_rc" +_v_sess=$(grep -F -- '|run -it ' "$V_DIR/log" \ + | sed -n 's/.* --rm=false --name \([^ ]*\) .*/\1/p') +assert_eq "the session's name ends in UTC time and 8 hex digits" "true" \ + "$(grep -qE "^${_v_img}-interactive-[a-z0-9._-]+-[0-9]{14}-[0-9a-f]{8}$" \ + <<< "$_v_sess" && echo true || echo false)" +_v_vol=$(v_volume "^${_v_sess:-none}-ws-[0-9a-f]{8}$") +assert_eq "the session gets a volume of its own" "true" \ + "$(jq -e --arg n "$_v_sess" --arg t "$_v_tok" ' + .["swarm-core.container"] == $n + and .["swarm-core.kind"] == "interactive" + and .["swarm-core.context"] == "slim" + and .["swarm-core.engagement-token"] == $t' \ + "$V_DIR/vols/${_v_vol:-none}" >/dev/null 2>&1 \ + && echo true || echo false)" +assert_eq "mounted after docker_args, with the policies and its name" "1" \ + "$(grep -F -- '|run -it ' "$V_DIR/log" | grep -cF -- " --rm --restart always --name other --mount type=volume,src=${_v_vol:-none},dst=/workspace --label swarm-core.engagement=${_v_eng} --label swarm-core.engagement-token=${_v_tok} --restart no --rm=false --name ${_v_sess} " || true)" +assert_eq "its session record names the daemon" "daemon=daemon-a" \ + "$(grep '^daemon=' "$V_RT/interactive-${_v_sess}.pending" 2>/dev/null)" +assert_eq "joining builds the image too" "1" \ + "$(grep -c '|build ' "$V_DIR/log" || true)" + +# post-process: docker run fails here, which ends it before it waits. +: > "$V_DIR/log" +echo run > "$V_DIR/fail" +_v_rc=0 +_v_out=$(v_run post-process) || _v_rc=$? +rm -f "$V_DIR/fail" +assert_eq "post-process reached docker run" "125" "$_v_rc" +_v_vol=$(v_volume "^${_v_img}-post-ws-[0-9a-f]{8}$") +assert_eq "post-process gets a volume of its own" "true" \ + "$(jq -e --arg t "$_v_tok" ' + .["swarm-core.container"] == "start-repo-agent-post" + and .["swarm-core.kind"] == "agent" + and .["swarm-core.context"] == "full" + and .["swarm-core.engagement-token"] == $t' \ + "$V_DIR/vols/${_v_vol:-none}" >/dev/null 2>&1 \ + && echo true || echo false)" +assert_eq "mounted after docker_args, with the policies and its name" "1" \ + "$(grep -F -- '|run -d ' "$V_DIR/log" | grep -cF -- " --rm --restart always --name other --mount type=volume,src=${_v_vol:-none},dst=/workspace --label swarm-core.engagement=${_v_eng} --label swarm-core.engagement-token=${_v_tok} --restart no --rm=false --name ${_v_img}-post " || true)" + +# --- volume failures stop before any docker run ---------------------- +# Creation fails; the labels read back are not the ones set (another +# volume of that name); the name is taken; Docker cannot say. +for _v_case in "create:cannot create Docker volume" \ + "labels:is not the one just created" \ + "exists:already exists" "error:cannot check Docker volume"; do + v_new + _v_rc=0 + _v_out=$(FAKE_VOLUME_FAIL="${_v_case%%:*}" v_run start) || _v_rc=$? + assert_eq "volume ${_v_case%%:*}: start fails" "1" "$_v_rc" + assert_eq "volume ${_v_case%%:*}: saying why" "true" \ + "$(v_has "${_v_case#*:}" "$_v_out")" + assert_eq "volume ${_v_case%%:*}: before any docker run" "0" \ + "$(grep -c '|run ' "$V_DIR/log" || true)" +done + +# An unknown context mode stops start before anything is made. +_v_cfg_ok="$V_CFG" +V_CFG="$_v_root/bad-context.json" +jq '.agents[0].context = "weird"' "$_v_cfg_ok" > "$V_CFG" +v_new +_v_rc=0 +_v_out=$(v_run start) || _v_rc=$? +assert_eq "an unknown context mode stops start" "1" "$_v_rc" +assert_eq "saying so" "true" "$(v_has "unknown context mode" "$_v_out")" +assert_eq "before the runtime directory is made" "false" \ + "$([ -e "$V_RT" ] && echo true || echo false)" +assert_eq "and before any Docker write" "0" "$(v_writes)" +v_new +_v_rc=0 +_v_out=$(v_run interactive --agent-index 1) || _v_rc=$? +assert_eq "an unknown context mode stops interactive" "1" "$_v_rc" +assert_eq "saying so" "true" "$(v_has "unknown context mode" "$_v_out")" +assert_eq "before the runtime directory is made" "false" \ + "$([ -e "$V_RT" ] && echo true || echo false)" +V_CFG="$_v_cfg_ok" + +# The volume helper itself refuses an unknown kind or context before it +# asks Docker anything. +_v_vlog="$_v_root/helper.log" +: > "$_v_vlog" +_v_rc=0 +(docker() { echo "$*" >> "$_v_vlog"; } + swarm_workspace_volume_create n p /c e t other full) >/dev/null 2>&1 \ + || _v_rc=$? +assert_eq "an unknown volume kind is refused" "1" "$_v_rc" +_v_rc=0 +(docker() { echo "$*" >> "$_v_vlog"; } + swarm_workspace_volume_create n p /c e t agent weird) >/dev/null 2>&1 \ + || _v_rc=$? +assert_eq "an unknown context is refused" "1" "$_v_rc" +assert_eq "both before any Docker call" "0" \ + "$(wc -l < "$_v_vlog" | tr -d ' ')" + +# --- another daemon: nothing joins, nothing is stopped --------------- +V_DIR="$_v_start_dir" +V_RT="$_v_start_rt" +V_STATE="$V_RT/state.json" +# The ended session's record would be pruned by a later check. +rm -f "$V_RT"/interactive-*.pending +# The agents as stop and wait would see them: stop has one to stop; wait +# would go on at once to post-process (it would poll a running one +# forever). +v_agents() { + : > "$V_DIR/containers" + v_container "${_v_img}-1" "$1" "$_v_tok" + v_container "${_v_img}-2" exited "$_v_tok" +} +echo b > "$V_DIR/context" +_v_before=$(v_snap) +for _v_cmd in stop wait post-process "interactive --agent-index 1"; do + if [ "$_v_cmd" = stop ]; then v_agents running; else v_agents exited; fi + : > "$V_DIR/log" + _v_rc=0 + # shellcheck disable=SC2086 + _v_out=$(v_run $_v_cmd) || _v_rc=$? + assert_eq "another daemon: $_v_cmd refused" "1" "$_v_rc" + assert_eq "another daemon: $_v_cmd says which" "true" \ + "$(v_has "runs on Docker daemon daemon-a, but Docker now talks to daemon-b" "$_v_out")" + assert_eq "another daemon: $_v_cmd changes nothing" "0" "$(v_writes)" +done +assert_eq "another daemon: runtime and state as they were" "$_v_before" \ + "$(v_snap)" +echo a > "$V_DIR/context" + +# The daemon check refuses a state file it cannot read, and one with no +# daemon, without asking Docker. +printf '{' > "$_v_root/bad-state.json" +_v_rc=0 +(docker() { echo "$*" >> "$_v_vlog"; } + swarm_engagement_daemon_check "$_v_root/bad-state.json") >/dev/null 2>&1 \ + || _v_rc=$? +assert_eq "an unreadable state file fails the daemon check" "1" "$_v_rc" +jq -n '{schema: "swarm-core.state/v1", engagement_id: "e"}' \ + > "$_v_root/no-daemon.json" +_v_rc=0 +(docker() { echo "$*" >> "$_v_vlog"; } + swarm_engagement_daemon_check "$_v_root/no-daemon.json") >/dev/null 2>&1 \ + || _v_rc=$? +assert_eq "so does a state file with no daemon" "1" "$_v_rc" +assert_eq "neither asks Docker" "0" "$(wc -l < "$_v_vlog" | tr -d ' ')" + +# --- stop takes no lock; wait follows one engagement ------------------ +# A post-process holds the lock for its whole run; stop still stops. +v_agents running +: > "$V_DIR/log" +exec 8>>"$V_RT/engagement.lock" +flock -n 8 +_v_rc=0 +_v_out=$(v_run stop) || _v_rc=$? +exec 8>&- +assert_eq "stop works while another command holds the lock" "0" "$_v_rc" +assert_eq "and stops the running agent by its id" "1" \ + "$(grep -c "^a|stop -t 60 id-${_v_img}-1$" "$V_DIR/log" || true)" + +# stop and wait take every check from one copy of the state: the state +# file replaced right after that copy (a cleanup and a new start with +# the same engagement id elsewhere) changes neither the engagement they +# check nor the token they act on. A cp first on PATH replaces it. +_v_cp="$TMPDIR/v55-cp" +mkdir -p "$_v_cp" +cat > "$_v_cp/cp" <<'EOF' +#!/bin/bash +/bin/cp "$@" || exit $? +if [ "${*: -2:1}" = "$V_STATE" ] && [ -e "$V_DIR/replace-state" ]; then + rm -f "$V_DIR/replace-state" + jq -c '.engagement_token = "ffffffffffffffff"' "$V_STATE" \ + > "$V_STATE.x" && mv "$V_STATE.x" "$V_STATE" +fi +EOF +chmod +x "$_v_cp/cp" +cp "$V_STATE" "$_v_root/state.saved" +v_agents running +jq -c --arg c "$_v_checkout" '{id: "id-new-1", name: "start-repo-agent-1", + state: "running", labels: {"swarm-core.project": "start-repo", + "swarm-core.checkout": $c, + "swarm-core.engagement-token": "ffffffffffffffff"}}' \ + -n >> "$V_DIR/containers" +: > "$V_DIR/replace-state" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(PATH="$_v_cp:$PATH" SWARM_ENGAGEMENT_ID="$_v_eng" v_run stop) \ + || _v_rc=$? +assert_eq "stop: the state replaced after its copy: still stops" "0" \ + "$_v_rc" +assert_eq "the engagement it checked" "1" \ + "$(grep -c "|stop -t 60 id-${_v_img}-1$" "$V_DIR/log" || true)" +assert_eq "and not the new one's container" "0" \ + "$(grep -c '|stop -t 60 id-new-1$' "$V_DIR/log" || true)" +cp "$_v_root/state.saved" "$V_STATE" +v_agents exited +: > "$V_DIR/replace-state" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(PATH="$_v_cp:$PATH" SWARM_ENGAGEMENT_ID="$_v_eng" v_run wait) \ + || _v_rc=$? +assert_eq "wait: the state replaced after its copy: refused under the lock" \ + "1" "$_v_rc" +assert_eq "for that reason" "true" \ + "$(v_has "the engagement changed while waiting" "$_v_out")" +assert_eq "with no post-process" "0" "$(grep -c '|run ' "$V_DIR/log" || true)" +cp "$_v_root/state.saved" "$V_STATE" +# Neither command reads the state file itself before its checks. +for _v_fn in cmd_stop cmd_wait; do + assert_eq "$_v_fn checks a copy of the state" "true" \ + "$(awk -v fn="$_v_fn" '$0 ~ "^" fn "\\(\\)" { p = 1 } p { print } + p && /^}/ { exit }' "$_LAUNCH_SH" \ + | awk '/state_snapshot snap/ { s = 1 } + /swarm_engagement_daemon_check/ { exit } + s && /"\$STATE_FILE"/ { bad = 1 } + END { print (s && !bad) ? "true" : "false" }')" +done + +# A start can still be launching agents of the engagement while stop +# runs (it holds the lock, stop does not): stop lists them again after +# each pass and stops those too; when they keep appearing, it fails +# rather than leave one running behind a success. +v_agents running +echo 1 > "$V_DIR/spawn" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run stop) || _v_rc=$? +assert_eq "stop: an agent launched while it stops is stopped too" "0" \ + "$_v_rc" +assert_eq "in a second pass" "2" "$(grep -c '|stop -t 60 ' "$V_DIR/log" || true)" +assert_eq "none of the engagement's agents runs then" "0" \ + "$(jq -r 'select(.state == "running") | .id' "$V_DIR/containers" \ + | grep -c . || true)" +# Three passes that stop, then one that only checks: two more batches +# that appear are stopped too. +v_agents running +echo 2 > "$V_DIR/spawn" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run stop) || _v_rc=$? +assert_eq "stop: two more batches during stop are stopped too" "0" "$_v_rc" +assert_eq "in three passes" "3" "$(grep -c '|stop -t 60 ' "$V_DIR/log" || true)" +v_agents running +echo 99 > "$V_DIR/spawn" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run stop) || _v_rc=$? +rm -f "$V_DIR/spawn" +assert_eq "stop: agents that keep appearing fail it" "1" "$_v_rc" +assert_eq "after three passes" "3" \ + "$(grep -c '|stop -t 60 ' "$V_DIR/log" || true)" +assert_eq "saying so" "true" \ + "$(v_has "keep starting; run stop again once start is done" "$_v_out")" +# post-process, under the lock, finds every numbered agent done (no +# start can be launching more then) or refuses: the dashboard's p +# stops them first without the lock. +v_agents running +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run post-process) || _v_rc=$? +assert_eq "post-process while a numbered agent runs: refused" "1" "$_v_rc" +assert_eq "with the next step" "true" \ + "$(v_has "stop them (./launch.sh stop --numbered) or wait for them" \ + "$_v_out")" +assert_eq "and nothing made" "0" "$(v_writes)" +: > "$V_DIR/containers" + +# A stop Docker cannot do fails the command, so that the dashboard's p +# does not go on to post-process while an agent may still run; one that +# stopped or went meanwhile is fine. So is the label check: a container +# Docker cannot describe, though it is there, fails the stop. +for _v_case in stop:1:"cannot stop ${_v_img}-1" \ + stop-exits:0:"${_v_img}-1 not running" \ + stop-gone:0:"${_v_img}-1 not running" \ + inspect:1:"cannot inspect container id-${_v_img}-1"; do + _v_mode="${_v_case%%:*}" + _v_want="${_v_case#*:}" + _v_msg="${_v_want#*:}" + _v_want="${_v_want%%:*}" + v_agents running + echo "$_v_mode" > "$V_DIR/fail" + _v_rc=0 + _v_out=$(v_run stop) || _v_rc=$? + rm -f "$V_DIR/fail" + assert_eq "stop, $_v_mode: status" "$_v_want" "$_v_rc" + assert_eq "stop, $_v_mode: says what happened" "true" \ + "$(v_has "$_v_msg" "$_v_out")" +done + +# A token that is not the engagement's refuses. +for _v_cmd in stop wait post-process; do + if [ "$_v_cmd" = stop ]; then v_agents running; else v_agents exited; fi + : > "$V_DIR/log" + _v_rc=0 + _v_out=$(SWARM_ENGAGEMENT_TOKEN=0123456789abcdef v_run "$_v_cmd") \ + || _v_rc=$? + assert_eq "another token: $_v_cmd refused" "1" "$_v_rc" + assert_eq "another token: $_v_cmd says the engagement is gone" "true" \ + "$(v_has "the engagement this command was started for is gone" \ + "$_v_out")" + assert_eq "another token: $_v_cmd changes nothing" "0" "$(v_writes)" +done + +# A token names an engagement: once it is gone (cleaned up, or no state +# file at all), a stop bound to it refuses instead of finding nothing +# to stop. +cp "$V_STATE" "$_v_root/state.saved" +for _v_st in cleaned missing; do + case "$_v_st" in + cleaned) + jq -c 'del(.engagement_id, .engagement_started_at, + .base_commit, .docker_daemon, .engagement_token)' \ + "$_v_root/state.saved" > "$V_STATE" ;; + missing) rm -f "$V_STATE" ;; + esac + : > "$V_DIR/log" + _v_rc=0 + _v_out=$(SWARM_ENGAGEMENT_TOKEN="$_v_tok" v_run stop) || _v_rc=$? + assert_eq "a token-bound stop, engagement $_v_st: refused" "1" "$_v_rc" + assert_eq "a token-bound stop, engagement $_v_st: says it is gone" \ + "true" "$(v_has "the engagement this command was started for is gone" "$_v_out")" + assert_eq "a token-bound stop, engagement $_v_st: stops nothing" "0" \ + "$(v_writes)" +done +cp "$_v_root/state.saved" "$V_STATE" +chmod 600 "$V_STATE" + +# wait reads the token once; an engagement started again while it polls +# makes it stop before post-process and harvest. +v_agents exited +cp "$V_STATE" "$_v_root/state.saved" +touch "$V_DIR/newtoken" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run wait) || _v_rc=$? +assert_eq "wait: an engagement replaced while it polls stops it" "1" \ + "$_v_rc" +assert_eq "saying so" "true" \ + "$(v_has "the engagement changed while waiting" "$_v_out")" +assert_eq "no post-process starts" "0" \ + "$(grep -c '|run ' "$V_DIR/log" || true)" +assert_eq "and no harvest" "false" "$(v_has "Harvesting results" "$_v_out")" +cp "$_v_root/state.saved" "$V_STATE" +# Without the switch, wait goes on to post-process with the token. +echo run > "$V_DIR/fail" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run wait) || _v_rc=$? +rm -f "$V_DIR/fail" +assert_eq "wait: all agents done, post-process starts" "1" \ + "$(grep -c "|run -d .* --name ${_v_img}-post " "$V_DIR/log" || true)" +assert_eq "wait: post-process ended at the failed docker run" "125" \ + "$_v_rc" + +# wait goes on only once no numbered agent runs: one that runs for two +# more listings keeps it waiting for them (then one more listing, under +# the lock). +v_agents exited +mv "$V_DIR/containers" "$V_DIR/at.3" +v_agents running +rm -f "$V_DIR/wait-listed" +echo run > "$V_DIR/fail" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run wait) || _v_rc=$? +rm -f "$V_DIR/fail" +# Four listings by wait, one by post-process under its lock. +assert_eq "wait: a running agent keeps it waiting" "5" \ + "$(cat "$V_DIR/wait-listed" 2>/dev/null)" +assert_eq "wait: then post-process starts" "1" \ + "$(grep -c "|run -d .* --name ${_v_img}-post " "$V_DIR/log" || true)" +# A start records the engagement before it launches the agents, and +# holds the lock until they run: agents that run by the time wait has +# the lock are waited for too. +: > "$V_DIR/containers" +v_agents running +mv "$V_DIR/containers" "$V_DIR/at.2" +v_agents exited +cp "$V_DIR/containers" "$V_DIR/at.4" +: > "$V_DIR/containers" +rm -f "$V_DIR/wait-listed" +echo run > "$V_DIR/fail" +: > "$V_DIR/log" +_v_rc=0 +_v_out=$(v_run wait) || _v_rc=$? +rm -f "$V_DIR/fail" +assert_eq "wait: agents launched while it took the lock are waited for" \ + "6" "$(cat "$V_DIR/wait-listed" 2>/dev/null)" +assert_eq "wait: post-process only after them" "1" \ + "$(grep -c "|run -d .* --name ${_v_img}-post " "$V_DIR/log" || true)" +# The count itself: numbered agents only, any state that may still run. +awk '/^wait_count\(\)[[:space:]]*\{/ { p = 1 } p { print } + p && /^\}[[:space:]]*$/ { exit }' "$_LAUNCH_SH" > "$_v_root/wait_count.sh" +assert_eq "wait counts the numbered agents that run or may" "3 2" \ + "$(IMAGE_NAME="$_v_img" + # shellcheck source=/dev/null + source "$_v_root/wait_count.sh" + wait_count "${_v_img}-1 running +${_v_img}-2 exited +${_v_img}-3 created +${_v_img}-4 paused +${_v_img}-10 dead +${_v_img}-post running +${_v_img}-interactive-x-1 running +other-agent-1 running")" +# No read loop over a here-string (Bash 3.2 makes a temporary file for +# it that can fail to be made, and a failed read ends the loop early). +assert_eq "wait reads no listing through a here-string" "0" \ + "$(awk '/^cmd_wait\(\)[[:space:]]*\{/ { p = 1 } p { print } + p && /^\}[[:space:]]*$/ { exit }' "$_LAUNCH_SH" \ + | grep -c '<<<' || true)" + +# --- post-process with this checkout's -post container --------------- +for _v_state in running exited; do + : > "$V_DIR/containers" + v_container "${_v_img}-post" "$_v_state" "$_v_tok" + : > "$V_DIR/log" + _v_before=$(v_snap) + _v_rc=0 + _v_out=$(v_run post-process) || _v_rc=$? + assert_eq "a $_v_state -post container: post-process refused" "1" \ + "$_v_rc" + case "$_v_state" in + running) _v_next="stop it first (./launch.sh stop)" ;; + *) _v_next="retire it first: ./launch.sh retire --post-process" ;; + esac + assert_eq "a $_v_state -post container: the next step" "true" \ + "$(v_has "$_v_next" "$_v_out")" + assert_eq "a $_v_state -post container: nothing removed or made" "0" \ + "$(v_writes)" + assert_eq "a $_v_state -post container: runtime as it was" \ + "$_v_before" "$(v_snap)" +done +: > "$V_DIR/containers" + +# A token names an engagement to join: with the bare repo gone, +# post-process and interactive do not make a new one. +mv "$V_RT/upstream.git" "$_v_root/upstream.saved" +for _v_cmd in post-process "interactive --agent-index 1"; do + : > "$V_DIR/log" + _v_rc=0 + # shellcheck disable=SC2086 + _v_out=$(SWARM_ENGAGEMENT_TOKEN="$_v_tok" v_run $_v_cmd) || _v_rc=$? + assert_eq "token, no bare repo: $_v_cmd refused" "1" "$_v_rc" + assert_eq "token, no bare repo: $_v_cmd says why" "true" \ + "$(v_has "has no bare repo any more" "$_v_out")" + assert_eq "token, no bare repo: $_v_cmd makes nothing" "0" \ + "$(v_writes)" +done +mv "$_v_root/upstream.saved" "$V_RT/upstream.git" + +# --- a new engagement only on a clean runtime ------------------------- +# The leftovers: the bare repo (with and without agent work the checkout +# lacks), upstream.git.new, mirrors/, a remnant in the runtime +# directory, a hidden one, a container of this checkout in any state, a +# volume, an open interactive session. Each makes start refuse and +# change nothing; post-process and interactive, which would create the +# engagement when there is no bare repo, refuse the same. +sleep 60 & +_v_live=$! +# A runtime with a recorded engagement and, of what start made, only +# leftover KIND; "work" keeps the bare repo, with a commit on agent-work +# that the checkout lacks. +v_leftover() { + v_new + v_run start >/dev/null || true + if [ "$1" = work ]; then + git -C "$V_RT/upstream.git" -c user.name=t -c user.email=t@t \ + -c commit.gpgsign=false commit-tree -m work -p agent-work \ + "$(git -C "$V_RT/upstream.git" rev-parse 'agent-work^{tree}')" \ + > "$_v_root/work-commit" + git -C "$V_RT/upstream.git" update-ref refs/heads/agent-work \ + "$(cat "$_v_root/work-commit")" + else + rm -rf "$V_RT/upstream.git" "$V_RT/mirrors" + rm -f "$V_RT"/interactive-*.pending + fi + case "$1" in + new) mkdir "$V_RT/upstream.git.new" ;; + mirrors) mkdir "$V_RT/mirrors" ;; + remnant) mkdir "$V_RT/x.removing" ;; + hidden) mkdir "$V_RT/.x.removing" ;; + container) v_container "${_v_img}-1" exited "" ;; + volume) echo "${_v_img}-1-ws-0123abcd" > "$V_DIR/volumes" ;; + session) + swarm_pending_write "$V_RT" "${_v_img}-interactive-x-1" \ + "$_v_live" daemon-a ;; + esac + : > "$V_DIR/log" +} +for _v_kind in work new mirrors remnant hidden container volume session; do + v_leftover "$_v_kind" + _v_before=$(v_snap) + _v_cmds="start post-process interactive" + [ "$_v_kind" = work ] && _v_cmds="start" + for _v_cmd in $_v_cmds; do + _v_rc=0 + if [ "$_v_cmd" = interactive ]; then + _v_out=$(v_run interactive --agent-index 1) || _v_rc=$? + else + _v_out=$(v_run "$_v_cmd") || _v_rc=$? + fi + assert_eq "$_v_kind left: $_v_cmd refused" "1" "$_v_rc" + assert_eq "$_v_kind left: $_v_cmd names the clean runtime rule" \ + "true" "$(v_has "a new engagement starts only on a clean runtime" "$_v_out")" + assert_eq "$_v_kind left: $_v_cmd gives the way out" "true" \ + "$(v_has "then ./launch.sh cleanup" "$_v_out")" + assert_eq "$_v_kind left: $_v_cmd changes nothing" "$_v_before" \ + "$(v_snap)" + assert_eq "$_v_kind left: $_v_cmd makes nothing in Docker" "0" \ + "$(v_writes)" + if [ "$_v_kind" = work ]; then + assert_eq "agent work the checkout lacks is named" "true" \ + "$(v_has "agent-work $(cut -c1-12 "$_v_root/work-commit"): run ./harvest.sh" "$_v_out")" + fi + done +done +kill "$_v_live" 2>/dev/null || true +wait "$_v_live" 2>/dev/null || true +# A bare repo whose work the checkout has: refused, without that list. +v_new +v_run start >/dev/null || true +: > "$V_DIR/log" +_v_before=$(v_snap) +_v_rc=0 +_v_out=$(v_run start) || _v_rc=$? +assert_eq "a second start on the same runtime is refused" "1" "$_v_rc" +assert_eq "naming the bare repo" "true" \ + "$(v_has " $V_RT/upstream.git" "$_v_out")" +assert_eq "with nothing to harvest listed" "false" \ + "$(v_has "does not contain this agent work" "$_v_out")" +assert_eq "and changes nothing" "$_v_before" "$(v_snap)" +assert_eq "nor makes anything in Docker" "0" "$(v_writes)" + +# Listings that fail refuse, after the lock and before anything is +# made. +for _v_case in ps3:"cannot list Docker containers" \ + volume-ls:"cannot list Docker volumes"; do + v_new + echo "${_v_case%%:*}" > "$V_DIR/fail" + _v_rc=0 + _v_out=$(v_run start) || _v_rc=$? + assert_eq "${_v_case%%:*} fails: start refused" "1" "$_v_rc" + assert_eq "${_v_case%%:*} fails: saying so" "true" \ + "$(v_has "${_v_case#*:}" "$_v_out")" + assert_eq "${_v_case%%:*} fails: in the gate (the runtime is claimed)" \ + "true" "$(jq -e '.repo_root != null' "$V_STATE" >/dev/null 2>&1 \ + && echo true || echo false)" + assert_eq "${_v_case%%:*} fails: no engagement made" "" \ + "$(jq -r '.engagement_id // empty' "$V_STATE" 2>/dev/null)" + assert_eq "${_v_case%%:*} fails: nothing made in Docker" "0" \ + "$(v_writes)" +done + +# --- creation stopped half way ------------------------------------------ +# The clone fails: only the .new it made goes; the engagement stays +# recorded, and the runtime is clean for the next start. +_v_git="$TMPDIR/v55-git" +mkdir -p "$_v_git" +cat > "$_v_git/git" < "\$a/objects/x" + exit 128 + fi ;; + esac +done +exec "$(command -v git)" "\$@" +EOF +chmod +x "$_v_git/git" +v_new +_v_rc=0 +_v_out=$(PATH="$_v_git:$PATH" v_run start) || _v_rc=$? +assert_eq "a clone that fails stops start" "1" "$_v_rc" +assert_eq "its upstream.git.new is removed" "false" \ + "$([ -e "$V_RT/upstream.git.new" ] && echo true || echo false)" +assert_eq "no bare repo is left" "false" \ + "$([ -e "$V_RT/upstream.git" ] && echo true || echo false)" +assert_eq "the engagement was recorded first" "true" \ + "$([ -n "$(jq -r '.engagement_id // empty' "$V_STATE" 2>/dev/null)" ] \ + && echo true || echo false)" +assert_eq "no container was started" "0" \ + "$(grep -c '|run ' "$V_DIR/log" || true)" +_v_rc=0 +v_run start >/dev/null || _v_rc=$? +assert_eq "the next start finds the runtime clean" "0" "$_v_rc" + +# A run killed during the clone leaves upstream.git.new: the gate +# refuses it (cleanup preserves and removes it, see test_cleanup.sh). +v_leftover new +_v_out=$(v_run start || true) +assert_eq "a .new left by a killed run is named" "true" \ + "$(v_has " $V_RT/upstream.git.new" "$_v_out")" + +# A failed image build leaves the runtime clean too. +v_new +echo build > "$V_DIR/fail" +_v_rc=0 +_v_out=$(v_run start) || _v_rc=$? +rm -f "$V_DIR/fail" +assert_eq "a failed image build stops start" "1" "$_v_rc" +assert_eq "before an engagement is recorded" "" \ + "$(jq -r '.engagement_id // empty' "$V_STATE" 2>/dev/null)" +assert_eq "and before a bare repo is made" "false" \ + "$([ -e "$V_RT/upstream.git" ] && echo true || echo false)" +_v_rc=0 +v_run start >/dev/null || _v_rc=$? +assert_eq "the next start works" "0" "$_v_rc" + +# --- a runtime inside the checkout ---------------------------------- +v_new +V_RT="$_st_repo/.swarm-v55" +V_STATE="$V_RT/state.json" +_v_rc=0 +_v_out=$(v_run start) || _v_rc=$? +assert_eq "a runtime inside the checkout: start refused" "1" "$_v_rc" +assert_eq "saying so" "true" \ + "$(v_has "is inside the checkout or its git dir" "$_v_out")" +assert_eq "and nothing is made there" "false" \ + "$([ -e "$V_RT" ] && echo true || echo false)" +assert_eq "nor in Docker" "0" "$(v_writes)" + +# --- an engagement of an earlier release ------------------------------ +# Its state file records no daemon and no token, and its containers +# (one started with --rm, one with an anonymous /workspace volume) keep +# their files themselves. Every command but status and logs refuses +# it, says how to finish it, and changes nothing. +v_new +v_run start >/dev/null || true +jq -c 'del(.docker_daemon, .engagement_token)' "$V_STATE" \ + > "$V_STATE.x" && mv "$V_STATE.x" "$V_STATE" +find "$V_DIR/vols" -type f -exec rm -f {} + +v_container "${_v_img}-1" running "" +v_container "${_v_img}-2" running "" +v_container "${_v_img}-post" exited "" +v_container "${_v_img}-interactive-x-20260101000000-0" running "" +: > "$V_DIR/log" +_v_before=$(v_snap) +for _v_cmd in start post-process "interactive --agent-index 1" stop wait \ + retire cleanup; do + _v_rc=0 + # shellcheck disable=SC2086 + _v_out=$(v_run $_v_cmd) || _v_rc=$? + assert_eq "earlier release: $_v_cmd refused" "1" "$_v_rc" + assert_eq "earlier release: $_v_cmd says how to finish it" "true" \ + "$(v_has "Finish it with the release that made it" "$_v_out")" + assert_eq "earlier release: $_v_cmd changes nothing" "$_v_before" \ + "$(v_snap)" + assert_eq "earlier release: $_v_cmd stops, removes or makes nothing" \ + "0" "$(v_writes)" +done +_v_rc=0 +_v_out=$(v_run status) || _v_rc=$? +assert_eq "earlier release: status still answers" "0" "$_v_rc" +assert_eq "with the agents' states" "true" \ + "$(v_has "running" "$_v_out")" +_v_rc=0 +v_run logs 1 >/dev/null || _v_rc=$? +assert_eq "earlier release: logs still answer" "0" "$_v_rc" +assert_eq "earlier release: the runtime is as it was" "$_v_before" \ + "$(v_snap)" + +# Leftovers with no engagement recorded come from an earlier release. +v_new +v_run start >/dev/null || true +jq -c 'del(.engagement_id, .engagement_started_at, .base_commit, + .docker_daemon, .engagement_token)' "$V_STATE" \ + > "$V_STATE.x" && mv "$V_STATE.x" "$V_STATE" +: > "$V_DIR/log" +_v_before=$(v_snap) +_v_rc=0 +_v_out=$(v_run start) || _v_rc=$? +assert_eq "leftovers without an engagement: start refused" "1" "$_v_rc" +assert_eq "as an earlier release's" "true" \ + "$(v_has "Finish it with the release that made it" "$_v_out")" +assert_eq "naming what is left" "true" \ + "$(v_has " $V_RT/upstream.git" "$_v_out")" +assert_eq "and changes nothing" "$_v_before" "$(v_snap)" + +# --- launch.sh removes nothing by force --------------------------------- +assert_eq "launch.sh has no docker rm -f" "0" \ + "$(grep -c 'docker rm -f' "$_LAUNCH_SH" || true)" +assert_eq "no alpine helper" "0" "$(grep -c 'alpine' "$_LAUNCH_SH" || true)" +assert_eq "no rm_docker_dir" "0" \ + "$(grep -c 'rm_docker_dir' "$_LAUNCH_SH" || true)" + +# A session record is never written over another. +_v_pd="$_v_root/pending" +mkdir -p "$_v_pd" +swarm_pending_write "$_v_pd" "${_v_img}-interactive-y-1" "$$" daemon-a +_v_rec=$(cat "$_v_pd/interactive-${_v_img}-interactive-y-1.pending") +_v_rc=0 +swarm_pending_write "$_v_pd" "${_v_img}-interactive-y-1" 1 daemon-b \ + 2>/dev/null || _v_rc=$? +assert_eq "a second record of the same name is refused" "1" "$_v_rc" +assert_eq "and the first stays as it was" "$_v_rec" \ + "$(cat "$_v_pd/interactive-${_v_img}-interactive-y-1.pending")" + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" diff --git a/tests/test_preserve.sh b/tests/test_preserve.sh index cbf0abd..37f3e46 100755 --- a/tests/test_preserve.sh +++ b/tests/test_preserve.sh @@ -13,6 +13,8 @@ TMPDIR=$(mktemp -d) REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" trap 'rm -rf "$TMPDIR"' EXIT +# shellcheck source=../lib/project.sh +source "$REPO_ROOT/lib/project.sh" # shellcheck source=../lib/preserve.sh source "$REPO_ROOT/lib/preserve.sh" @@ -121,6 +123,30 @@ assert_eq "ref to a missing object: status 2" "2" \ rm -f "$BARE/refs/heads/agent-parked/missing" assert_eq "clean again after removing them" "0|" "$(check)" +echo "" +echo "=== 5c. A read that fails part of the way is no end of the list ===" +# An I/O error ends a read loop as the end of the list would; the refs +# after it, uncontained ones among them, would never be looked at. A +# read override fails the first read of the list. +z=$(agent_push refs/heads/agent-parked/9-z) +rc=0 +( + # Shadows the builtin for the check below; it passes the caller's + # options on. + # shellcheck disable=SC2329,SC2162 + read() { + if [ "${!#}" = ref ]; then return 1; fi + builtin read "$@" + } + swarm_preservation_check "$BARE" "$WORK" +) >/dev/null 2>"$TMPDIR/read.err" || rc=$? +assert_eq "a failed read before an uncontained ref: status 2" "2" "$rc" +assert_eq "saying the refs could not be read in full" "1" \ + "$(grep -c 'cannot read the refs of' "$TMPDIR/read.err" || true)" +assert_eq "without the failure the ref is listed" "1|refs/heads/agent-parked/9-z ${z}" \ + "$(check)" +git -C "$BARE" update-ref -d refs/heads/agent-parked/9-z "$z" + echo "" echo "=== 6. Not a bare repository ===" rc=0 @@ -1065,6 +1091,8 @@ if [ -r /proc/self/mountinfo ] && unshare -rm true 2>/dev/null; then rc=0 unshare -rm "$BASH" -c ' mount --bind "$1" "$2" || exit 9 + # shellcheck source=../lib/project.sh + source "$3/lib/project.sh" # shellcheck source=../lib/preserve.sh source "$3/lib/preserve.sh" swarm_checkout_storage_ok "$4" "$5" "$2"' sh \ diff --git a/tests/test_retire.sh b/tests/test_retire.sh index 975f9b8..3f5f766 100755 --- a/tests/test_retire.sh +++ b/tests/test_retire.sh @@ -136,6 +136,10 @@ case "$1" in cat "$F/inspect/$1" exit 0 ;; volume) + if [ "$2" = ls ]; then + ls "$F/volumes" + exit 0 + fi [ "$2" = inspect ] || exit 1 [ -f "$F/volumes/$5" ] || exit 1 cat "$F/volumes/$5" @@ -1310,17 +1314,27 @@ add proj-agent-1 echo d2 > "$F/daemon" assert_eq "another Docker daemon is refused" "1" "$(retire)" rm -f "$F/daemon" +# An engagement with no recorded daemon was started by an earlier +# release: refused, untouched, with how to finish it. swarm_state_update "$RT/state.json" 'del(.docker_daemon)' -assert_eq "no recorded daemon is refused" "1" "$(retire)" -assert_eq "the message says how to name it" "1" \ - "$(grep -c 'SWARM_DOCKER_DAEMON=' "$TMPDIR/out" || true)" -rc=0 -(cd "$REPO" && PATH="$TMPDIR/bin:$PATH" FAKE_DOCKER="$F" FAKE_CK="$CK" \ - SWARM_RUNTIME_DIR="$RT" SWARM_PROJECT="$P" SWARM_SKIP_DEP_CHECK=1 \ - SWARM_DOCKER_DAEMON=d1 "$BASH" "$LAUNCH" retire > /dev/null 2>&1) \ - || rc=$? -assert_eq "SWARM_DOCKER_DAEMON names it" "0" "$rc" -swarm_state_update "$RT/state.json" '.docker_daemon = "d1"' +assert_eq "an earlier release's engagement is refused" "1" "$(retire)" +assert_eq "the message says how to finish it" "1" \ + "$(grep -c 'release that made it' "$TMPDIR/out" || true)" +assert_eq "nothing removed then" "0" "$(removed)" +# Containers with no engagement recorded at all are an earlier +# release's too; with nothing left, there is nothing to retire. +swarm_state_update "$RT/state.json" 'del(.engagement_id, .docker_daemon)' +assert_eq "a container without a recorded engagement is refused" "1" \ + "$(retire)" +assert_eq "as an earlier release's" "1" \ + "$(grep -c 'release that made it' "$TMPDIR/out" || true)" +assert_eq "nothing removed then" "0" "$(removed)" +reset_fake +assert_eq "no engagement and nothing left: retire succeeds" "0" "$(retire)" +assert_eq "with nothing to retire" "1" \ + "$(grep -c 'nothing to retire' "$TMPDIR/out" || true)" +swarm_state_update "$RT/state.json" \ + '.engagement_id = "e1" | .docker_daemon = "d1"' reset_fake add proj-agent-1 rc=0 diff --git a/tests/test_runtime.sh b/tests/test_runtime.sh index 9f63915..f2f207f 100755 --- a/tests/test_runtime.sh +++ b/tests/test_runtime.sh @@ -331,43 +331,16 @@ assert_eq "legacy bare repo is refused" "1" \ _legacy_msg=$(swarm_refuse_legacy_state "$_legacy_project" 2>&1 || true) assert_contains "message shows how to fetch the old refs" \ "git fetch ${LEGACY_BARE}" "$_legacy_msg" -assert_contains "message stops old containers with the stop grace" \ - " docker stop -t ${SWARM_STOP_TIMEOUT:-60} ..." \ - "$_legacy_msg" +assert_contains "message sends the user to the release that made it" \ + "with the release that made it" "$_legacy_msg" +assert_eq "message prints no stop or removal command" "0" \ + "$(printf '%s\n' "$_legacy_msg" \ + | grep -cE 'docker (stop|run|rm)|rm -rf' || true)" assert_eq "legacy state is left in place" "true" \ "$([ -d "$LEGACY_BARE" ] && echo true || echo false)" - -# Run a printed removal line. The docker stub runs on this host the -# command that the alpine container would run. -run_removal() { - bash -c 'docker() { - while [ "$#" -gt 0 ] && [ "$1" != alpine ]; do shift; done - shift - "$@" - } - eval "$1"' _ "$1" -} - -# The printed removal command lists existing paths, with no glob that -# zsh would reject when nothing matches. It goes through Docker, since -# agents wrote files there as another user. -_rm_prefix=" docker run --rm -v /tmp:/tmp alpine rm -rf" -_legacy_rm=$(printf '%s\n' "$_legacy_msg" | grep '^ docker run ' || true) -assert_eq "removal command names the old bare repo" \ - "${_rm_prefix} ${LEGACY_BARE}" "$_legacy_rm" -run_removal "$_legacy_rm" -assert_eq "printed command removes the old state" "0" \ +rm -rf "$LEGACY_BARE" +assert_eq "without it nothing is refused" "0" \ "$(status_of swarm_refuse_legacy_state "$_legacy_project")" - -LEGACY_MIRROR="/tmp/${_legacy_project}-mirror-lib.git" -mkdir -p "$LEGACY_BARE" "$LEGACY_MIRROR" -_legacy_rm=$(swarm_refuse_legacy_state "$_legacy_project" 2>&1 \ - | grep '^ docker run ' || true) -assert_eq "removal command names existing mirrors too" \ - "${_rm_prefix} ${LEGACY_BARE} ${LEGACY_MIRROR}" "$_legacy_rm" -run_removal "$_legacy_rm" -assert_eq "old mirror is removed" "false" \ - "$([ -e "$LEGACY_MIRROR" ] && echo true || echo false)" mkdir -p "$LEGACY_BARE" # Anyone can create a path in /tmp. Only a directory the user owns @@ -382,21 +355,6 @@ assert_eq "file at the legacy path is ignored" "0" \ "$(status_of swarm_refuse_legacy_state "$_legacy_project")" rm -f "$LEGACY_BARE" -# The removal command for a runtime bare repo mounts only the runtime -# directory, as one argument even with a space in it, and removes only -# the bare repo there. -_rm_dir="$TMPDIR/run time" -_rm_args=$(bash -c 'docker() { printf "<%s>\n" "$@"; }; eval "$1"' _ \ - "$(swarm_bare_rm_command "$_rm_dir")") -assert_eq "bare removal mounts the runtime directory only" " -<--rm> -<-v> -<${_rm_dir}:/runtime> - - -<-rf> -" "$_rm_args" - # ============================================================ echo "" echo "=== 5. State file ===" @@ -553,16 +511,20 @@ assert_contains "another checkout's containers: use SWARM_PROJECT" \ assert_eq "another checkout's containers: no old-release steps" "0" \ "$(printf '%s\n' "$_ctr_msg" | grep -c 'older swarm-core' || true)" -# Containers of an older release get the upgrade steps instead: stop -# with the grace, so their agents can still push to /tmp. +# Containers of an older release get the upgrade steps instead, with +# no stop or removal command (stopping one started with --rm removes +# its files). _ctr_msg=$( # shellcheck disable=SC2329 docker() { ctr_listing proj-agent-1 "" "" proj-agent-post "" ""; } swarm_containers_verify proj "$_own_a" 2>&1 || true ) -assert_contains "old containers: stop them with the grace" \ - " docker stop -t ${SWARM_STOP_TIMEOUT:-60} proj-agent-1 proj-agent-post" \ +assert_contains "old containers: named" "proj-agent-1 proj-agent-post" \ "$_ctr_msg" +assert_contains "old containers: finish them with that release" \ + "finish it with that release" "$_ctr_msg" +assert_eq "old containers: no docker stop advice" "0" \ + "$(printf '%s\n' "$_ctr_msg" | grep -c 'docker stop' || true)" assert_contains "old containers: points to the upgrade steps" \ "Upgrading from" "$_ctr_msg" assert_eq "old containers: no SWARM_PROJECT advice" "0" \ From 60a22fdce4afbbf5a563d844b39aa3255a140e82 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Thu, 5 Nov 2026 07:29:20 +0100 Subject: [PATCH 11/27] Keep a failing status when Bash 3.2 stops a run on an unset variable --- CHANGELOG.md | 7 ++++++ launch.sh | 21 +++++++++++++++++- lib/cleanup.sh | 2 +- lib/retire.sh | 16 +++++++++++++- tests/test.sh | 8 +++++++ tests/test_ci_workflows.sh | 45 ++++++++++++++++++++++++++++++++++++++ tests/test_cleanup.sh | 14 ++++++++++++ tests/test_launch.sh | 27 +++++++++++++++++++++++ tests/test_retire.sh | 17 ++++++++++++++ 9 files changed, 154 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 695348a..e4044ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## Unreleased +- **Fix: a run that Bash 3.2 stops on an unset variable no longer ends + with status 0.** macOS's `/bin/bash` runs an EXIT trap with status 0 + when `set -u` stops a script, so `launch.sh`, retire and cleanup, + which have EXIT traps, would report a stop of that kind (a bug) as + success. Such a run now ends with 1. `tests/test.sh --unit` also + fails a test file that never printed its summary, which the same + quirk would have let pass; files that skip on purpose print one. - **Breaking: workspace volumes, one Docker daemon per engagement, new engagements only on a clean runtime directory.** Each engine container gets a `/workspace` volume of its own (`-ws-`, labeled diff --git a/launch.sh b/launch.sh index 4c230a7..bd92a50 100755 --- a/launch.sh +++ b/launch.sh @@ -526,12 +526,28 @@ esac # INT or TERM end the script (retire and cleanup, above, make none and # set their own traps). SWARM_TMP_FILES=() +# Set when the run gets to an end it means: its last line, or an exit +# with status 0 it makes on purpose. +SWARM_EXIT_OK=false swarm_tmp_cleanup() { if [ "${#SWARM_TMP_FILES[@]}" -gt 0 ]; then rm -f "${SWARM_TMP_FILES[@]}" fi } -trap swarm_tmp_cleanup EXIT +# The EXIT trap: remove the temporary files, and never let a run that +# stopped short end with status 0. Bash 3.2 (macOS's /bin/bash) runs an +# EXIT trap with $? at 0 when set -u stops the script on an unset +# variable, so that stop would pass for success. Any other status +# stays, a signal's too. +swarm_exit_trap() { + local rc=$? + swarm_tmp_cleanup + if [ "$rc" -eq 0 ] && ! $SWARM_EXIT_OK; then + echo "ERROR: launch.sh stopped before the end of its run." >&2 + exit 1 + fi +} +trap swarm_exit_trap EXIT # Make a temporary file swarm-NAME.XXXXXX in TMPDIR, removed when this # run exits, and set the variable VAR to its absolute path (the run @@ -750,6 +766,7 @@ Usage: $0 interactive [--agent NAME | --agent-index N] [--shell] Start one human-guided container from an agents[] profile. HELP + SWARM_EXIT_OK=true exit 0 ;; *) if [ -n "$selector" ]; then @@ -1597,3 +1614,5 @@ case "${1:-start}" in exit 1 ;; esac +# The run got to its end (see swarm_exit_trap). +SWARM_EXIT_OK=true diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 3e47159..fb6c883 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -905,7 +905,7 @@ cmd_cleanup() { RETIRE_SCRATCH="" # shellcheck disable=SC2034 # read by retire_cleanup (lib/retire.sh) RETIRE_SNAP_REF="" - trap retire_cleanup EXIT + trap retire_stopped EXIT trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 129' HUP trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 130' INT trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 143' TERM diff --git a/lib/retire.sh b/lib/retire.sh index 58768ee..8decfde 100644 --- a/lib/retire.sh +++ b/lib/retire.sh @@ -106,6 +106,20 @@ $refs EOF } +# The EXIT trap of retire and cleanup. It fires only when the run +# stops short (its normal end removes it first): clean up, and never end +# with status 0 then. Bash 3.2 runs an EXIT trap with $? at 0 when set +# -u stops the script on an unset variable, so that stop would pass for +# success; any other status stays. +retire_stopped() { + local rc=$? + retire_cleanup + if [ "$rc" -eq 0 ]; then + echo "ERROR: the run stopped before its end." >&2 + exit 1 + fi +} + retire_cleanup() { # Every background job is a docker command retire started; a signal # can come before its pid is known, so wait for them all. @@ -720,7 +734,7 @@ cmd_retire() { # The traps come first, so that nothing retire makes is ever # outside them. Once a signal starts the cleanup, further ones wait # for it. - trap retire_cleanup EXIT + trap retire_stopped EXIT trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 129' HUP trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 130' INT trap 'trap "" HUP INT TERM; retire_cleanup; trap - EXIT; exit 143' TERM diff --git a/tests/test.sh b/tests/test.sh index 0118d34..e3a849d 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -359,6 +359,14 @@ cmd_unit() { rc=0 output=$("$f" 2>&1) || rc=$? count=$(printf '%s' "$output" | grep -oE '[0-9]+ passed' | tail -1 || true) + # A file passes only when it got to its summary: Bash 3.2 (the + # files' /bin/bash on macOS) exits 0 when set -u stops a script + # that has an EXIT trap. + if [ "$rc" -eq 0 ] \ + && ! grep -qE '^ *[0-9]+ passed, 0 failed' <<< "$output"; then + rc=1 + output="${output}"$'\n'"(no summary line: the file stopped early)" + fi if [ "$rc" -eq 0 ]; then printf " PASS %-24s (%s)\n" "$name" "${count:-?}" pass=$((pass + 1)) diff --git a/tests/test_ci_workflows.sh b/tests/test_ci_workflows.sh index fbe0114..0769e63 100755 --- a/tests/test_ci_workflows.sh +++ b/tests/test_ci_workflows.sh @@ -75,6 +75,51 @@ echo "=== 4. Unit test dependencies ===" assert_eq "macOS unit job installs flock" "1" \ "$(grep -cE 'brew install .*flock' "$CI_YML")" +echo "" +echo "=== 5. The unit runner needs each file's summary ===" + +# A test file that stops short (set -u, with an EXIT trap) ends with +# status 0 under Bash 3.2, the files' /bin/bash on macOS; the runner +# fails it for the summary it never printed. A file that skips on +# purpose prints its summary, and passes. +_r=$(mktemp -d) +mkdir -p "$_r/tests" "$_r/lib" +cp "$TESTS_DIR/test.sh" "$TESTS_DIR/_test_env.sh" "$_r/tests/" +cp "$REPO_ROOT/lib/project.sh" "$_r/lib/" +git init -q "$_r" +printf '#!/bin/bash\necho " 1 passed, 0 failed"\n' > "$_r/tests/test_a.sh" +printf '#!/bin/bash\necho "SKIP: not here"\necho " 0 passed, 0 failed"\nexit 0\n' \ + > "$_r/tests/test_c.sh" +cat > "$_r/tests/test_b.sh" <<'EOF' +#!/bin/bash +set -euo pipefail +T=$(mktemp -d) +trap 'rm -rf "$T"' EXIT +echo " PASS: one" +echo "$swarm_test_unset_variable" +echo " 2 passed, 0 failed" +EOF +chmod +x "$_r/tests/"*.sh +_rc=0 +_out=$(cd "$_r" && ./tests/test.sh --unit 2>&1) || _rc=$? +rm -rf "$_r" +assert_eq "a file that stops short fails the run" "1" "$_rc" +assert_eq "that file is the one failed" "true" \ + "$(grep -qE 'FAIL +test_b\.sh' <<< "$_out" && echo true || echo false)" +assert_eq "the other passes" "true" \ + "$(grep -qE 'PASS +test_a\.sh' <<< "$_out" && echo true || echo false)" +assert_eq "so does one that skips with its summary" "true" \ + "$(grep -qE 'PASS +test_c\.sh' <<< "$_out" && echo true || echo false)" +# The test files that skip on purpose print their summary too. +for _f in "$TESTS_DIR"/test_*.sh; do + if grep -q '^ echo "SKIP:' "$_f"; then + assert_eq "$(basename "$_f") prints a summary when it skips" "true" \ + "$(awk '/^ echo "SKIP:/ { s = 1; next } + s && /^ exit 0/ { print (ok ? "true" : "false"); exit } + s && /passed, 0 failed/ { ok = 1 }' "$_f")" + fi +done + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" diff --git a/tests/test_cleanup.sh b/tests/test_cleanup.sh index c6e306b..b7676f6 100755 --- a/tests/test_cleanup.sh +++ b/tests/test_cleanup.sh @@ -24,6 +24,8 @@ trap 'chmod -R u+rwX "$TMPDIR" 2>/dev/null; rm -rf "$TMPDIR"' EXIT # or later); here it runs on this host's git. if ! git var GIT_CONFIG_SYSTEM >/dev/null 2>&1; then echo "SKIP: these tests need Git 2.42 or later (git var)" + # The runner wants a summary from every file. + echo " 0 passed, 0 failed" exit 0 fi @@ -1099,6 +1101,18 @@ for _flag in "" --allow-unharvested; do assert_eq "nothing is copied${_flag:+ ($_flag)}" "0" "$(recovery_refs)" done +# Stopped on an unset variable (a flock that names one, through +# BASH_ENV), cleanup does not end with status 0, as Bash 3.2's EXIT trap +# would have it, and deletes nothing. +cat > "$TMPDIR/unset-hook.sh" <<'EOF' +flock() { : "$swarm_test_unset_variable"; } +EOF +setup +_rc=$(BASH_ENV="$TMPDIR/unset-hook.sh" cleanup) +assert_eq "cleanup stopped on an unset variable: not status 0" "true" \ + "$([ "$_rc" -ne 0 ] && echo true || echo false)" +assert_eq "the bare repo stays" "yes" "$(present "$RT/upstream.git")" + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" diff --git a/tests/test_launch.sh b/tests/test_launch.sh index 7bc6365..3555ee3 100755 --- a/tests/test_launch.sh +++ b/tests/test_launch.sh @@ -4408,6 +4408,33 @@ assert_eq "a second record of the same name is refused" "1" "$_v_rc" assert_eq "and the first stays as it was" "$_v_rec" \ "$(cat "$_v_pd/interactive-${_v_img}-interactive-y-1.pending")" +# ============================================================ +echo "" +echo "=== 56. A run stopped on an unset variable does not end with 0 ===" + +# Bash 3.2 (macOS's /bin/bash, which runs these tests there) runs an +# EXIT trap with $? at 0 when set -u stops the script on an unset +# variable. A flock that names one, through BASH_ENV, stops launch.sh in +# swarm_lock, in the main shell and after its EXIT trap is set. +_u_hook="$TMPDIR/unset-hook.sh" +cat > "$_u_hook" <<'EOF' +flock() { : "$swarm_test_unset_variable"; } +EOF +v_new +_v_rc=0 +_v_out=$(BASH_ENV="$_u_hook" v_run start) || _v_rc=$? +assert_eq "start stopped on an unset variable: not status 0" "true" \ + "$([ "$_v_rc" -ne 0 ] && echo true || echo false)" +assert_eq "having made no engagement" "" \ + "$(jq -r '.engagement_id // empty' "$V_STATE" 2>/dev/null)" +# The ends a run means keep status 0: help, and a run's last line. +_v_rc=0 +v_run interactive --help >/dev/null || _v_rc=$? +assert_eq "interactive --help ends with 0" "0" "$_v_rc" +_v_rc=0 +v_run status >/dev/null || _v_rc=$? +assert_eq "status ends with 0" "0" "$_v_rc" + echo "" echo "===============================" echo " ${PASS} passed, ${FAIL} failed" diff --git a/tests/test_retire.sh b/tests/test_retire.sh index 3f5f766..eb5290b 100755 --- a/tests/test_retire.sh +++ b/tests/test_retire.sh @@ -21,6 +21,8 @@ trap 'rm -rf "$TMPDIR"' EXIT # or later); here the scanner runs on this host's git. if ! git var GIT_CONFIG_SYSTEM >/dev/null 2>&1; then echo "SKIP: these tests need Git 2.42 or later (git var)" + # The runner wants a summary from every file. + echo " 0 passed, 0 failed" exit 0 fi @@ -1329,6 +1331,21 @@ assert_eq "a container without a recorded engagement is refused" "1" \ assert_eq "as an earlier release's" "1" \ "$(grep -c 'release that made it' "$TMPDIR/out" || true)" assert_eq "nothing removed then" "0" "$(removed)" +# Stopped on an unset variable (a flock that names one, through +# BASH_ENV), retire does not end with status 0, as Bash 3.2's EXIT trap +# would have it, and removes nothing. +swarm_state_update "$RT/state.json" \ + '.engagement_id = "e1" | .docker_daemon = "d1"' +cat > "$TMPDIR/unset-hook.sh" <<'EOF' +flock() { : "$swarm_test_unset_variable"; } +EOF +reset_fake +add proj-agent-1 +rc=$(BASH_ENV="$TMPDIR/unset-hook.sh" retire) +assert_eq "retire stopped on an unset variable: not status 0" "true" \ + "$([ "$rc" -ne 0 ] && echo true || echo false)" +assert_eq "and removes nothing" "0" "$(removed)" +swarm_state_update "$RT/state.json" 'del(.engagement_id, .docker_daemon)' reset_fake assert_eq "no engagement and nothing left: retire succeeds" "0" "$(retire)" assert_eq "with nothing to retire" "1" \ From 3f4a6a6dc437d4b15af12b2272b3c0b38ac4b622 Mon Sep 17 00:00:00 2001 From: Simone <88100810+orsissimo@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:17:46 +0100 Subject: [PATCH 12/27] Harvest agent work built on the engagement's base after HEAD moved on --- CHANGELOG.md | 18 ++ USAGE.md | 26 ++- harvest.sh | 316 +++++++++++++++++++++++++++--- launch.sh | 5 +- tests/test_harvest.sh | 444 ++++++++++++++++++++++++++++++++++++++++-- tests/test_launch.sh | 10 + 6 files changed, 772 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4044ca..61a0846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ ## Unreleased +- **Fix: harvest merges agent work built on the engagement's base + after HEAD moved on.** It refused any branch that did not descend from + HEAD, so agent work pushed after an interactive branch was merged, or + after commits made in the checkout, could not be harvested. It now + checks the commit the engagement started from (`base_commit`, recorded + on the host): a branch built on it merges while HEAD's history still + has it, and still has what earlier harvests of the engagement merged + (recorded too). A history rewritten after the start, or one that + dropped harvested work or gave up a merge (each merge is recorded + before it starts, and committed whatever the Git config says), still + refuses; + `harvest.sh --again` merges that work again on purpose, after undoing + a harvest say. A merge that stops on a conflict says how to finish or + give it up, a harvest refuses while a merge is in progress, and one + within the same second as the previous one no longer fails on its + restore tag. Harvest no longer fetches tags from the bare repo, and + refuses when a tag or branch of the checkout has the name of a + harvested branch (`_agent-harvest/`). - **Fix: a run that Bash 3.2 stops on an unset variable no longer ends with status 0.** macOS's `/bin/bash` runs an EXIT trap with status 0 when `set -u` stops a script, so `launch.sh`, retire and cleanup, diff --git a/USAGE.md b/USAGE.md index fb46737..1ebad4e 100644 --- a/USAGE.md +++ b/USAGE.md @@ -564,8 +564,27 @@ post-processing, run `./launch.sh post-process`; use that command directly when you intentionally want to run only the post-process agent and then harvest. +`harvest.sh` merges a branch built on the commit the engagement started +from (`base_commit` in the state file) as long as HEAD's history still +has that commit, so it also merges agent work after you merged an +interactive branch or made commits of your own. It refuses when HEAD's +history no longer has it (rewritten after the engagement started, by a +force-push or a rebase): take what you need by hand, then end the +engagement with `./launch.sh cleanup --allow-unharvested`. It also +records what each harvest merged, and refuses when HEAD no longer has +that (a rebase dropped some of it): merging again would bring it back. +Each merge is recorded before it starts, and committed whatever merge +options the Git config sets (`--no-commit`, `--squash`). A merge that +stops on a conflict says how to finish it or give it up. Finished by +hand, it counts as merged; given up, or cut short with the harvest +(killed, say), the next harvest refuses as well, as does one run while +the merge is still in progress. After undoing a harvest on purpose +(`git reset --hard `, below) or giving up such a merge, +`./harvest.sh --again` merges that work again. + Before merging, `harvest.sh` tags the current branch tip with a -local `swarm-harvest--