From fa8a5ed79340c2bb917278f0da33610da41f14cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Sun, 18 Feb 2024 23:47:32 +0100 Subject: [PATCH] Migrate VM creation to runner Change the logic to adapt to the semantics of the microVMs lifecycle. MicroVMs are created, started and deleted by the runner loops instead. The orchestrator is only responsible for creating the loops. As a result, many ORCHESTRATOR_ variables (and their options) have been moved to the runner.sh loop script, and renamed to RUNNER_. The orchestrator now accepts much fewer options. Instead, options should be passed after a -- at the command-line, and these are blindly passed to each runner loop. This means that the number of loops to create is not an argument anymore, but rather an option to the orchestrator (defaults to 1). This is a breaking change. Also the tgz binary distribution is now removed during installation, and, instead the runner-specific installation is (recursively) copied to the target directory. This is to avoid the need to have the tgz distribution in the runner's directory, but also because copying is actually quicker than moving files (hierarchies). --- orchestrator.sh | 253 +++++++++++++--------------------------------- runner.sh | 142 +++++++++++++++++++++----- runner/install.sh | 1 + runner/runner.sh | 40 +++----- 4 files changed, 201 insertions(+), 235 deletions(-) diff --git a/orchestrator.sh b/orchestrator.sh index 2f77b80..aac0706 100755 --- a/orchestrator.sh +++ b/orchestrator.sh @@ -39,32 +39,12 @@ ORCHESTRATOR_VERBOSE=${ORCHESTRATOR_VERBOSE:-0} # Where to send logs ORCHESTRATOR_LOG=${ORCHESTRATOR_LOG:-2} -# Name of the OCI image (fully-qualified) to use. You need to have access. -ORCHESTRATOR_IMAGE=${ORCHESTRATOR_IMAGE:-"ghcr.io/efrecon/runner-krunvm:main"} +# Number of runners to create +ORCHESTRATOR_RUNNERS=${ORCHESTRATOR_RUNNERS:-1} -# Memory to allocate to the VM (in MB). Regular runners use more than the -# default. -ORCHESTRATOR_MEMORY=${ORCHESTRATOR_MEMORY:-"1024"} - -# Number of vCPUs to allocate to the VM. Regular runners use more than the -# default. -ORCHESTRATOR_CPUS=${ORCHESTRATOR_CPUS:-"2"} - -# Name of the VM to create (krunvm create) -ORCHESTRATOR_NAME=${ORCHESTRATOR_NAME:-"runner"} - -# DNS to use on the VM. This is the same as the default in krunvm. -ORCHESTRATOR_DNS=${ORCHESTRATOR_DNS:-"1.1.1.1"} - -# Host->VM mount points, lines containing pairs of directory mappings separated -# by a colon. -ORCHESTRATOR_MOUNT=${ORCHESTRATOR_MOUNT:-""} - -# Name of top directory in VM where to host a copy of the root directory of this -# script. When this is set, the runner starter script from that directory will -# be used -- instead of the one already in the OCI image. This option is mainly -# usefull for development and testing. -ORCHESTRATOR_DIR=${ORCHESTRATOR_DIR:-""} +# Prefix to use for the VM name. The VM name will be $ORCHESTRATOR_PREFIX-xxx. +# All VMs prefixed with this name will be deleted on exit. +ORCHESTRATOR_PREFIX=${ORCHESTRATOR_PREFIX:-"GH-runner"} # Should the runner be isolated in its own environment. This will pass all # configuration down to the runner starter script as an environment variable to @@ -72,90 +52,31 @@ ORCHESTRATOR_DIR=${ORCHESTRATOR_DIR:-""} # used. ORCHESTRATOR_ISOLATION=${ORCHESTRATOR_ISOLATION:-"1"} -# Number of seconds to sleep between microVM creation at start +# Number of seconds to sleep between microVM creation at start, unless isolation +# has been turned on. ORCHESTRATOR_SLEEP=${ORCHESTRATOR_SLEEP:-"30"} -# GitHub host, e.g. github.com or github.example.com -RUNNER_GITHUB=${RUNNER_GITHUB:-"github.com"} - -# Group to attach the runner to -RUNNER_GROUP=${RUNNER_GROUP:-"Default"} - -# Comma separated list of labels to attach to the runner (good defaults will be used if empty) -RUNNER_LABELS=${RUNNER_LABELS:-""} - -# Name of the user to run the runner as, defaults to runner. User must exist. -RUNNER_USER=${RUNNER_USER:-"runner"} - -# Scope of the runner, one of: repo, org or enterprise -RUNNER_SCOPE=${RUNNER_SCOPE:-"repo"} - -# Name of organisation, enterprise or repo to attach the runner to, when -# relevant scope. -RUNNER_PRINCIPAL=${RUNNER_PRINCIPAL:-""} - -# PAT to acquire runner token with -RUNNER_PAT=${RUNNER_PAT:-""} - -# Should the runner auto-update -RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} - -# Number of times to repeat the runner loop -RUNNER_REPEAT=${RUNNER_REPEAT:-"-1"} - # shellcheck disable=SC2034 # Used in sourced scripts KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host" -while getopts "a:c:d:D:g:G:i:Il:L:m:M:n:p:r:s:t:T:u:Uvh-" opt; do +while getopts "s:Il:n:p:vh-" opt; do case "$opt" in - a) # Number of seconds to sleep between microVM creation at start, when no isolation + s) # Number of seconds to sleep between microVM creation at start, when no isolation ORCHESTRATOR_SLEEP="$OPTARG";; - c) # Number of CPUs to allocate to the VM - ORCHESTRATOR_CPUS="$OPTARG";; - d) # DNS server to use in VM - ORCHESTRATOR_DNS=$OPTARG;; - D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing). - ORCHESTRATOR_DIR=$OPTARG;; - g) # GitHub host, e.g. github.com or github.example.com - RUNNER_GITHUB="$OPTARG";; - G) # Group to attach the runners to - RUNNER_GROUP="$OPTARG";; - i) # Fully-qualified name of the OCI image to use - ORCHESTRATOR_IMAGE="$OPTARG";; I) # Turn off variables isolation (not recommended, security risk) ORCHESTRATOR_ISOLATION=0;; l) # Where to send logs ORCHESTRATOR_LOG="$OPTARG";; - L) # Comma separated list of labels to attach to the runner - RUNNER_LABELS="$OPTARG";; - m) # Memory to allocate to the VM - ORCHESTRATOR_MEMORY="$OPTARG";; - M) # Mount local host directories into the VM : - if [ -z "$ORCHESTRATOR_MOUNT" ]; then - ORCHESTRATOR_MOUNT="$OPTARG" - else - ORCHESTRATOR_MOUNT="$(printf %s\\n%s\\n "$ORCHESTRATOR_MOUNT" "$OPTARG")" - fi;; - n) # Name of the VM to create - ORCHESTRATOR_NAME="$OPTARG";; - p) # Principal to authorise the runner for, name of repo, org or enterprise - RUNNER_PRINCIPAL="$OPTARG";; - r) # Number of times to repeat the runner loop - RUNNER_REPEAT="$OPTARG";; - s) # Scope of the runner, one of repo, org or enterprise - RUNNER_SCOPE="$OPTARG";; - T) # Authorization token at the GitHub API to acquire runner token with - RUNNER_PAT="$OPTARG";; - u) # User to run the runner as - RUNNER_USER="$OPTARG";; - U) # Turn on auto-updating of the runner - RUNNER_UPDATE=1;; + n) # Number of runners to create + ORCHESTRATOR_RUNNERS="$OPTARG";; + p) # Prefix to use for the VM name + ORCHESTRATOR_PREFIX="$OPTARG";; v) # Increase verbosity, will otherwise log on errors/warnings only ORCHESTRATOR_VERBOSE=$((ORCHESTRATOR_VERBOSE+1));; h) # Print help and exit usage 0 "(ORCHESTRATOR|RUNNER)";; - -) # End of options. Single argument: number of runners to create + -) # End of options. All subsequent arguments are passed to the runner.sh script break;; ?) usage 1;; @@ -169,6 +90,16 @@ KRUNVM_RUNNER_VERBOSE=$ORCHESTRATOR_VERBOSE cleanup() { trap - INT TERM EXIT + + if run_krunvm list | grep -qE "^${ORCHESTRATOR_PREFIX}-"; then + while IFS= read -r vm; do + verbose "Removing microVM $vm" + run_krunvm delete "$vm" + done <VM mount points, lines containing pairs of directory mappings separated +# by a colon. +RUNNER_MOUNT=${ORCHESTRATOR_MOUNT:-""} + +# Name of top directory in VM where to host a copy of the root directory of this +# script. When this is set, the runner starter script from that directory will +# be used -- instead of the one already in the OCI image. This option is mainly +# usefull for development and testing. +RUNNER_DIR=${RUNNER_DIR:-""} + # GitHub host, e.g. github.com or github.example.com RUNNER_GITHUB=${RUNNER_GITHUB:-"github.com"} @@ -64,8 +88,8 @@ RUNNER_PAT=${RUNNER_PAT:-""} # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} -# Name of the microVM to run from -RUNNER_NAME=${RUNNER_NAME:-"runner"} +# Prefix to use for the VM names. The VM name will be $RUNNER_PREFIX-xxx +RUNNER_PREFIX=${RUNNER_PREFIX:-"GH-runner"} # Name of top directory in VM where to host a copy of the root directory of this # script. When this is set, the runner starter script from that directory will @@ -92,24 +116,32 @@ RUNNER_SECRET=${RUNNER_SECRET:-"$(random_string)"} KRUNVM_RUNNER_DESCR="Create runners forever using krunvm" -while getopts "D:E:g:G:l:L:M:n:p:r:s:S:T:u:Uvh-" opt; do +while getopts "c:d:D:g:G:i:l:L:m:M:p:r:s:S:T:u:Uvh-" opt; do case "$opt" in + c) # Number of CPUs to allocate to the VM + RUNNER_CPUS="$OPTARG";; + d) # DNS server to use in VM + RUNNER_DNS=$OPTARG;; D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing). RUNNER_DIR=$OPTARG;; - E) # Location (at host) where to place environment files for each run. - RUNNER_ENVIRONMENT="$OPTARG";; g) # GitHub host, e.g. github.com or github.example.com RUNNER_GITHUB="$OPTARG";; G) # Group to attach the runner to RUNNER_GROUP="$OPTARG";; + i) # Name of the OCI image (fully-qualified) to use. You need to have access. + RUNNER_IMAGE="$OPTARG";; l) # Where to send logs RUNNER_LOG="$OPTARG";; L) # Comma separated list of labels to attach to the runner RUNNER_LABELS="$OPTARG";; - M) # Mount passed to the microVM - RUNNER_MOUNT="$OPTARG";; - n) # Name of the microVM to run from - RUNNER_NAME="$OPTARG";; + m) # Memory to allocate to the VM + RUNNER_MEMORY="$OPTARG";; + M) # Mount local host directories into the VM : + if [ -z "$RUNNER_MOUNT" ]; then + RUNNER_MOUNT="$OPTARG" + else + RUNNER_MOUNT="$(printf %s\\n%s\\n "$RUNNER_MOUNT" "$OPTARG")" + fi;; p) # Principal to authorise the runner for, name of repo, org or enterprise RUNNER_PRINCIPAL="$OPTARG";; r) # Number of times to repeat the runner loop @@ -139,48 +171,86 @@ shift $((OPTIND-1)) # Pass logging configuration and level to imported scripts KRUNVM_RUNNER_LOG=$RUNNER_LOG KRUNVM_RUNNER_VERBOSE=$RUNNER_VERBOSE -loop=${1:-} +loop=${1:-"0"} if [ -n "${loop:-}" ]; then KRUNVM_RUNNER_BIN=$(basename "$0") KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$loop" fi +# Name of the root directory in the VM where to map environment and +# synchronisation files. +RUNNER_VM_ENVDIR="_environment" + +if [ -z "$RUNNER_PAT" ]; then + error "You need to specify a PAT to acquire the runner token with" +fi +check_positive_number "$RUNNER_CPUS" "Number of vCPUs" +check_positive_number "$RUNNER_MEMORY" "Memory (in MB)" + # Decide which runner.sh implementation (this is the "entrypoint" of the # microVM) to use: the one from the mount point, or the built-in one. if [ -z "$RUNNER_DIR" ]; then - runner=/opt/gh-runner-krunvm/bin/runner.sh + RUNNER_ENTRYPOINT=/opt/gh-runner-krunvm/bin/runner.sh else check_command "${RUNNER_ROOTDIR}/runner/runner.sh" - runner=${RUNNER_DIR%/}/runner/runner.sh + RUNNER_ENTRYPOINT=${RUNNER_DIR%/}/runner/runner.sh fi -iteration=0 -while true; do - id=$(random_string) - RUNNER_ID=${loop}-${id} - verbose "Starting microVM $RUNNER_NAME to run ephemeral GitHub runner $RUNNER_ID" +# Create the VM used for orchestration. Add --volume options for all necessary +# mappings, i.e. inheritance of "live" code, environment isolation and all +# requested mount points. +vm_create() { + verbose "Creating microVM '${RUNNER_PREFIX}-$1', $RUNNER_CPUS vCPUs, ${RUNNER_MEMORY}M memory" + # Note: reset arguments! + set -- \ + --cpus "$RUNNER_CPUS" \ + --mem "$RUNNER_MEMORY" \ + --dns "$RUNNER_DNS" \ + --name "${RUNNER_PREFIX}-$1" + if [ -n "${RUNNER_DIR:-}" ]; then + set -- "$@" --volume "${RUNNER_ROOTDIR}:${RUNNER_DIR}" + fi + if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then + set -- "$@" --volume "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}" + fi + if [ -n "$RUNNER_MOUNT" ]; then + while IFS= read -r mount || [ -n "$mount" ]; do + if [ -n "$mount" ]; then + set -- "$@" --volume "$mount" + fi + done <> "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.env" + printf '%s\n' "$varset" >> "${RUNNER_ENVIRONMENT}/${_id}.env" done < "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" verbose "Installing runner to $INSTALL_DIR" tar -C "${INSTALL_DIR}/runner-${INSTALL_VERSION}" -zxf "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" +rm -f "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" # Install the dependencies (this is distro specific and aware) "${INSTALL_DIR}/runner-${INSTALL_VERSION}/bin/installdependencies.sh" diff --git a/runner/runner.sh b/runner/runner.sh index a0ed6c0..37c4d07 100755 --- a/runner/runner.sh +++ b/runner/runner.sh @@ -79,9 +79,6 @@ RUNNER_EPHEMERAL=${RUNNER_EPHEMERAL:-"0"} # Root installation of the runner RUNNER_INSTALL=${RUNNER_INSTALL:-"/opt/gh-runner-krunvm/share/runner"} -# Permit several installations -RUNNER_MULTI=${RUNNER_MULTI:-"1"} - # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} @@ -156,23 +153,14 @@ KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$RUNNER_ID" # minimal verification of the installation through checking that there is a # config.sh script executable within the copy. runner_install() { - if [ "$RUNNER_MULTI" = 1 ]; then - # Make a directory where to install a copy of the runner. - if ! [ -d "${RUNNER_WORKDIR%/}/runner" ]; then - mkdir -p "${RUNNER_WORKDIR%/}/runner" - verbose "Created runner directory ${RUNNER_WORKDIR%/}/runner" - fi - verbose "Installing runner in ${RUNNER_WORKDIR%/}/runner" - tar -C "${RUNNER_WORKDIR%/}/runner" -zxf "$RUNNER_TAR" - else - if ! [ -d "${RUNNER_WORKDIR%/}" ]; then - mkdir -p "${RUNNER_WORKDIR%/}" - verbose "Created runner directory ${RUNNER_WORKDIR%/}" - fi - verbose "Moving runner installation to ${RUNNER_WORKDIR%/}/runner" - mv -f "$RUNNER_INSTDIR" "${RUNNER_WORKDIR%/}/runner" 2>/dev/null + if ! [ -d "${RUNNER_WORKDIR%/}" ]; then + mkdir -p "${RUNNER_WORKDIR%/}" + verbose "Created runner directory ${RUNNER_WORKDIR%/}" fi - check_command "${RUNNER_WORKDIR%/}/runner/config.sh" + RUNNER_BINROOT="${RUNNER_WORKDIR%/}/runner" + verbose "Copying runner installation to $RUNNER_BINROOT" + cp -rf "$RUNNER_INSTDIR" "$RUNNER_BINROOT" 2>/dev/null + check_command "${RUNNER_BINROOT}/config.sh" } @@ -284,7 +272,7 @@ runner_unregister() { # temporary changes directory before calling the script. runner_control() { cwd=$(pwd) - cd "${RUNNER_WORKDIR%/}/runner" + cd "$RUNNER_BINROOT" script=./${1}; shift check_command "$script" debug "Running $script $*" @@ -369,8 +357,8 @@ fi debug "Setting up missing defaults" distro=$(get_env "/etc/os-release" "ID") RUNNER_DISTRO=${RUNNER_DISTRO:-"${distro:-"unknown}"}"} -RUNNER_NAME_PREFIX=${RUNNER_NAME_PREFIX:-"${RUNNER_DISTRO}-krunvm"} -RUNNER_NAME=${RUNNER_NAME:-"${RUNNER_NAME_PREFIX}-$RUNNER_ID"} +RUNNER_PREFIX=${RUNNER_PREFIX:-"${RUNNER_DISTRO}-krunvm"} +RUNNER_NAME=${RUNNER_NAME:-"${RUNNER_PREFIX}-$RUNNER_ID"} RUNNER_WORKDIR=${RUNNER_WORKDIR:-"/_work/${RUNNER_NAME}"} if [ -n "${distro:-}" ]; then @@ -379,13 +367,13 @@ else RUNNER_LABELS=${RUNNER_LABELS:-"krunvm"} fi -RUNNER_TAR=$(find_pattern "${RUNNER_INSTALL}/*.tgz" f | sort -r | head -n 1) -if [ -z "$RUNNER_TAR" ]; then - error "No runner tar file found under $RUNNER_INSTALL" -fi +# Find the (versioned) directory containing the full installation of the runner +# binary distribution (unpacked by the installer) RUNNER_INSTDIR=$(find_pattern "${RUNNER_INSTALL}/runner-*" d | sort -r | head -n 1) if [ -z "$RUNNER_INSTDIR" ]; then error "No runner installation directory found under $RUNNER_INSTALL" +else + debug "Found unpacked binary distribution at $RUNNER_INSTDIR" fi # Construct the runner URL, i.e. where the runner will be registered