Improve handshaking between processes

A .tkn file, containing the runner registration token is created inside
the microVM. The main orchestrator will wait for the first such token
file for a given runner loop before continuing iterating to the next
runner loop. This slows down startup time slightly, but provides
improved determinism.

Also, when an exit signal is caught, a file with the extension .brk is
created with a secret. This file and the content of the secret are
verified in the runner loop, and, when present and matching, the loop is
stopped instead of a new runner being created.

orchestrator.sh

@@ -29,6 +29,9 @@ abspath() {
 # links resolved.
 ORCHESTRATOR_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")")")" && pwd -P )
 
+# shellcheck source=lib/common.sh
+. "$ORCHESTRATOR_ROOTDIR/lib/common.sh"
+
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
 ORCHESTRATOR_VERBOSE=${ORCHESTRATOR_VERBOSE:-0}
@@ -100,15 +103,14 @@ RUNNER_UPDATE=${RUNNER_UPDATE:-"0"}
 # Number of times to repeat the runner loop
 RUNNER_REPEAT=${RUNNER_REPEAT:-"-1"}
 
-# shellcheck source=lib/common.sh
-. "$ORCHESTRATOR_ROOTDIR/lib/common.sh"
-
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host"
 
 
-while getopts "c:d:D:g:G:i:Il:L:m:M:n:p:r:s:t:T:u:Uvh-" opt; do
+while getopts "a:c:d:D:g:G:i:Il:L:m:M:n:p:r:s:t:T:u:Uvh-" opt; do
   case "$opt" in
+    a) # Number of seconds to sleep between microVM creation at start, when no isolation
+      ORCHESTRATOR_SLEEP="$OPTARG";;
     c) # Number of CPUs to allocate to the VM
       ORCHESTRATOR_CPUS="$OPTARG";;
     d) # DNS server to use in VM
@@ -166,6 +168,7 @@ KRUNVM_RUNNER_LOG=$ORCHESTRATOR_LOG
 KRUNVM_RUNNER_VERBOSE=$ORCHESTRATOR_VERBOSE
 
 cleanup() {
+  trap - INT TERM EXIT
   if [ -n "${ORCHESTRATOR_ENVIRONMENT:-}" ]; then
     verbose "Removing isolation environment $ORCHESTRATOR_ENVIRONMENT"
     rm -rf "$ORCHESTRATOR_ENVIRONMENT"
@@ -187,7 +190,7 @@ runners=${1:-0}
 # Create isolation mount point
 if [ "$ORCHESTRATOR_ISOLATION" = 1 ]; then
   ORCHESTRATOR_ENVIRONMENT=$(mktemp -d)
-  trap cleanup INT TERM QUIT
+  trap cleanup INT TERM EXIT
 fi
 
 # Create the VM used for orchestration. Add --volume options for all necessary
@@ -249,8 +252,18 @@ EOF
         -E "${ORCHESTRATOR_ENVIRONMENT:-}" \
         -- "$i" &
       set -- "$@" "$!"
+
+      # Wait for runner to be ready or have progresses somewhat before starting
+      # the next one.
       if [ "$i" -lt "$runners" ]; then
-        if [ -n "$ORCHESTRATOR_SLEEP" ] && [ "$ORCHESTRATOR_SLEEP" -gt 0 ]; then
+        # Wait for the runner token to be ready before starting the next runner,
+        # or sleep for some time.
+        if [ -n "${ORCHESTRATOR_ENVIRONMENT:-}" ]; then
+          wait_path -f "${ORCHESTRATOR_ENVIRONMENT}/${i}-*.tkn" -1 5
+          token=$(find_pattern "${ORCHESTRATOR_ENVIRONMENT}/${i}-*.tkn")
+          rm -f "$token"
+          verbose "Removed token file $token"
+        elif [ -n "$ORCHESTRATOR_SLEEP" ] && [ "$ORCHESTRATOR_SLEEP" -gt 0 ]; then
           debug "Sleeping for $ORCHESTRATOR_SLEEP seconds"
           sleep "$ORCHESTRATOR_SLEEP"
         fi

runner.sh

@@ -29,6 +29,9 @@ abspath() {
 # links resolved.
 RUNNER_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")")")" && pwd -P )
 
+# shellcheck source=lib/common.sh
+. "$RUNNER_ROOTDIR/lib/common.sh"
+
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
 RUNNER_VERBOSE=${RUNNER_VERBOSE:-0}
@@ -82,14 +85,14 @@ RUNNER_EPHEMERAL=${RUNNER_EPHEMERAL:-"1"}
 # Number of times to repeat the runner loop
 RUNNER_REPEAT=${RUNNER_REPEAT:-"-1"}
 
-# shellcheck source=lib/common.sh
-. "$RUNNER_ROOTDIR/lib/common.sh"
+# Secret to be used to request for loop end. Good default is a random string.
+RUNNER_SECRET=${RUNNER_SECRET:-"$(random_string)"}
 
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Create runners forever using krunvm"
 
 
-while getopts "D:E:g:G:l:L:M:n:p:r:s:T:u:Uvh-" opt; do
+while getopts "D:E:g:G:l:L:M:n:p:r:s:S:T:u:Uvh-" opt; do
   case "$opt" in
     D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing).
       RUNNER_DIR=$OPTARG;;
@@ -113,6 +116,8 @@ while getopts "D:E:g:G:l:L:M:n:p:r:s:T:u:Uvh-" opt; do
       RUNNER_REPEAT="$OPTARG";;
     s) # Scope of the runner, one of repo, org or enterprise
       RUNNER_SCOPE="$OPTARG";;
+    S) # Secret to be used to request for loop end
+      RUNNER_SECRET="$OPTARG";;
     T) # Authorization token at the GitHub API to acquire runner token with
       RUNNER_PAT="$OPTARG";;
     u) # User to run the runner as
@@ -165,7 +170,11 @@ while true; do
 $(set | grep '^RUNNER_' | grep -vE '(ROOTDIR|ENVIRONMENT|NAME|MOUNT)')
 EOF
 
+    # Pass the location of the env. file to the runner script
     set -- -E "/_environment/${RUNNER_ID}.env"
+
+    # Also pass the location of a file that will contain the token.
+    set -- -k "/_environment/${RUNNER_ID}.tkn" "$@"
   else
     set -- \
         -e \
@@ -176,6 +185,7 @@ EOF
         -L "$RUNNER_LABELS" \
         -p "$RUNNER_PRINCIPAL" \
         -s "$RUNNER_SCOPE" \
+        -S "$RUNNER_SECRET" \
         -T "$RUNNER_PAT" \
         -u "$RUNNER_USER"
     for _ in $(seq 1 "$RUNNER_VERBOSE"); do
@@ -191,4 +201,16 @@ EOF
       break
     fi
   fi
+
+  if [ -n "$RUNNER_ENVIRONMENT" ]; then
+    if [ -f "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.brk" ]; then
+      break=$(cat "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.brk")
+      if [ "$break" = "$RUNNER_SECRET" ]; then
+        verbose "Break file found, stopping runner loop"
+        break
+      else
+        warning "Break file found at ${RUNNER_ENVIRONMENT}/${RUNNER_ID}.brk, but it does not match the secret"
+      fi
+    fi
+  fi
 done

runner/runner.sh

@@ -30,6 +30,9 @@ abspath() {
 # links resolved.
 RUNNER_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")")")" && pwd -P )
 
+# shellcheck source=../lib/common.sh
+. "$RUNNER_ROOTDIR/../lib/common.sh"
+
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
 RUNNER_VERBOSE=${RUNNER_VERBOSE:-0}
@@ -86,16 +89,14 @@ RUNNER_ENVFILE=${RUNNER_ENVFILE:-""}
 # Identifier of the runner (used in logs)
 RUNNER_ID=${RUNNER_ID:-""}
 
-RUNNER_TOOL_CACHE=${RUNNER_TOOL_CACHE:-"${AGENT_TOOLSDIRECTORY:-"/opt/hostedtoolcache"}"}
-
-# shellcheck source=../lib/common.sh
-. "$RUNNER_ROOTDIR/../lib/common.sh"
+# Location of a file where to store the token
+RUNNER_TOKENFILE=${RUNNER_TOKENFILE:-""}
 
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Configure and run the installed GitHub runner"
 
 
-while getopts "eE:g:G:i:l:L:n:p:s:t:T:u:Uvh-" opt; do
+while getopts "eE:g:G:i:k:l:L:n:p:s:S:t:T:u:Uvh-" opt; do
   case "$opt" in
     e) # Ephemeral runner
       RUNNER_EPHEMERAL=1;;
@@ -107,6 +108,8 @@ while getopts "eE:g:G:i:l:L:n:p:s:t:T:u:Uvh-" opt; do
       RUNNER_GROUP="$OPTARG";;
     i) # Identifier of the runner (used in logs and to name the runner)
       RUNNER_ID="$OPTARG";;
+    k) # Location of a file where to store the token
+      RUNNER_TOKENFILE="$OPTARG";;
     l) # Where to send logs
       RUNNER_LOG="$OPTARG";;
     L) # Comma separated list of labels to attach to the runner
@@ -211,24 +214,48 @@ runner_configure() {
   # Configure the runner from the installation copy
   verbose "Configuring runner ${RUNNER_NAME}..."
   runner_control config.sh "$@"
+
+  # Store runner registration token in a file, if set
+  if [ -n "${RUNNER_TOKENFILE:-}" ]; then
+    verbose "Storing runner token in $RUNNER_TOKENFILE"
+    printf %s\\n "$RUNNER_TOKEN" > "$RUNNER_TOKENFILE"
+  fi
 }
 
 
 # Unregister the runner from GitHub. This will use the runner installation copy
 runner_unregister() {
+  trap - INT TERM EXIT
+
   verbose "Caught termination signal, unregistering runner"
   if [ -n "${RUNNER_PAT:-}" ]; then
-    verbose "Requesting (back) runner token with PAT"
-    RUNNER_TOKEN=$( TOKEN_VERBOSE=$RUNNER_VERBOSE \
-                    "$RUNNER_ROOTDIR/token.sh" \
-                      -g "$RUNNER_GITHUB" \
-                      -l "$RUNNER_LOG" \
-                      -p "$RUNNER_PRINCIPAL" \
-                      -s "$RUNNER_SCOPE" \
-                      -T "$RUNNER_PAT" )
+    if [ -n "${RUNNER_TOKENFILE:-}" ] && [ -f "$RUNNER_TOKENFILE" ]; then
+      verbose "Reading runner token from $RUNNER_TOKENFILE"
+      RUNNER_TOKEN=$(cat "$RUNNER_TOKENFILE")
+    fi
+
+    if [ -z "${RUNNER_TOKEN:-}" ]; then
+      verbose "Requesting (back) runner token with PAT"
+      RUNNER_TOKEN=$( TOKEN_VERBOSE=$RUNNER_VERBOSE \
+                      "$RUNNER_ROOTDIR/token.sh" \
+                        -g "$RUNNER_GITHUB" \
+                        -l "$RUNNER_LOG" \
+                        -p "$RUNNER_PRINCIPAL" \
+                        -s "$RUNNER_SCOPE" \
+                        -T "$RUNNER_PAT" )
+    fi
   fi
   verbose "Removing runner at GitHub"
   runner_control config.sh remove --token "$RUNNER_TOKEN"
+
+  if [ -n "${RUNNER_TOKENFILE:-}" ] && [ -f "$RUNNER_TOKENFILE" ]; then
+    rm -f "$RUNNER_TOKENFILE"
+    verbose "Removed runner token file at $RUNNER_TOKENFILE"
+  fi
+
+  if [ -n "${RUNNER_TOKENFILE:-}" ] && [ -n "${RUNNER_SECRET:-}" ]; then
+    printf %s\\n "$RUNNER_SECRET" > "${RUNNER_TOKENFILE%.*}.brk"
+  fi
 }
 
 
@@ -337,8 +364,8 @@ if [ -z "$RUNNER_TAR" ]; then
 fi
 
 # Construct the runner URL, i.e. where the runner will be registered
-debug "Constructing runner URL"
 RUNNER_SCOPE=$(to_lower "$RUNNER_SCOPE")
+debug "Constructing $RUNNER_SCOPE runner URL"
 case "$RUNNER_SCOPE" in
   rep*)
     RUNNER_URL="https://${RUNNER_GITHUB%/}/${RUNNER_PRINCIPAL}"
@@ -367,7 +394,7 @@ if [ "$#" = 0 ]; then
 fi
 
 # Capture termination signals
-trap runner_unregister INT TERM QUIT
+trap runner_unregister INT TERM EXIT
 
 # Start the docker daemon. Prefer podman if available (it will be the only one
 # available, unless the dockerd is installed in the future)
@@ -388,8 +415,7 @@ case "$RUNNER_USER" in
     if id "$RUNNER_USER" >/dev/null 2>&1; then
       if [ "$(id -u)" = "0" ]; then
         verbose "Starting runner as $RUNNER_USER"
-        chown -R "$RUNNER_USER" "$RUNNER_INSTALL" "$RUNNER_WORKDIR"
-        chown "$RUNNER_USER" "$RUNNER_TOOL_CACHE"
+        chown -R "$RUNNER_USER" "$RUNNER_WORKDIR"
         runas "$@"
       elif [ "$(id -un)" = "$RUNNER_USER" ]; then
         "$@"