diff --git a/orchestrator.sh b/orchestrator.sh index 2f77b80..aac0706 100755 --- a/orchestrator.sh +++ b/orchestrator.sh @@ -39,32 +39,12 @@ ORCHESTRATOR_VERBOSE=${ORCHESTRATOR_VERBOSE:-0} # Where to send logs ORCHESTRATOR_LOG=${ORCHESTRATOR_LOG:-2} -# Name of the OCI image (fully-qualified) to use. You need to have access. -ORCHESTRATOR_IMAGE=${ORCHESTRATOR_IMAGE:-"ghcr.io/efrecon/runner-krunvm:main"} +# Number of runners to create +ORCHESTRATOR_RUNNERS=${ORCHESTRATOR_RUNNERS:-1} -# Memory to allocate to the VM (in MB). Regular runners use more than the -# default. -ORCHESTRATOR_MEMORY=${ORCHESTRATOR_MEMORY:-"1024"} - -# Number of vCPUs to allocate to the VM. Regular runners use more than the -# default. -ORCHESTRATOR_CPUS=${ORCHESTRATOR_CPUS:-"2"} - -# Name of the VM to create (krunvm create) -ORCHESTRATOR_NAME=${ORCHESTRATOR_NAME:-"runner"} - -# DNS to use on the VM. This is the same as the default in krunvm. -ORCHESTRATOR_DNS=${ORCHESTRATOR_DNS:-"1.1.1.1"} - -# Host->VM mount points, lines containing pairs of directory mappings separated -# by a colon. -ORCHESTRATOR_MOUNT=${ORCHESTRATOR_MOUNT:-""} - -# Name of top directory in VM where to host a copy of the root directory of this -# script. When this is set, the runner starter script from that directory will -# be used -- instead of the one already in the OCI image. This option is mainly -# usefull for development and testing. -ORCHESTRATOR_DIR=${ORCHESTRATOR_DIR:-""} +# Prefix to use for the VM name. The VM name will be $ORCHESTRATOR_PREFIX-xxx. +# All VMs prefixed with this name will be deleted on exit. +ORCHESTRATOR_PREFIX=${ORCHESTRATOR_PREFIX:-"GH-runner"} # Should the runner be isolated in its own environment. This will pass all # configuration down to the runner starter script as an environment variable to @@ -72,90 +52,31 @@ ORCHESTRATOR_DIR=${ORCHESTRATOR_DIR:-""} # used. ORCHESTRATOR_ISOLATION=${ORCHESTRATOR_ISOLATION:-"1"} -# Number of seconds to sleep between microVM creation at start +# Number of seconds to sleep between microVM creation at start, unless isolation +# has been turned on. ORCHESTRATOR_SLEEP=${ORCHESTRATOR_SLEEP:-"30"} -# GitHub host, e.g. github.com or github.example.com -RUNNER_GITHUB=${RUNNER_GITHUB:-"github.com"} - -# Group to attach the runner to -RUNNER_GROUP=${RUNNER_GROUP:-"Default"} - -# Comma separated list of labels to attach to the runner (good defaults will be used if empty) -RUNNER_LABELS=${RUNNER_LABELS:-""} - -# Name of the user to run the runner as, defaults to runner. User must exist. -RUNNER_USER=${RUNNER_USER:-"runner"} - -# Scope of the runner, one of: repo, org or enterprise -RUNNER_SCOPE=${RUNNER_SCOPE:-"repo"} - -# Name of organisation, enterprise or repo to attach the runner to, when -# relevant scope. -RUNNER_PRINCIPAL=${RUNNER_PRINCIPAL:-""} - -# PAT to acquire runner token with -RUNNER_PAT=${RUNNER_PAT:-""} - -# Should the runner auto-update -RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} - -# Number of times to repeat the runner loop -RUNNER_REPEAT=${RUNNER_REPEAT:-"-1"} - # shellcheck disable=SC2034 # Used in sourced scripts KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host" -while getopts "a:c:d:D:g:G:i:Il:L:m:M:n:p:r:s:t:T:u:Uvh-" opt; do +while getopts "s:Il:n:p:vh-" opt; do case "$opt" in - a) # Number of seconds to sleep between microVM creation at start, when no isolation + s) # Number of seconds to sleep between microVM creation at start, when no isolation ORCHESTRATOR_SLEEP="$OPTARG";; - c) # Number of CPUs to allocate to the VM - ORCHESTRATOR_CPUS="$OPTARG";; - d) # DNS server to use in VM - ORCHESTRATOR_DNS=$OPTARG;; - D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing). - ORCHESTRATOR_DIR=$OPTARG;; - g) # GitHub host, e.g. github.com or github.example.com - RUNNER_GITHUB="$OPTARG";; - G) # Group to attach the runners to - RUNNER_GROUP="$OPTARG";; - i) # Fully-qualified name of the OCI image to use - ORCHESTRATOR_IMAGE="$OPTARG";; I) # Turn off variables isolation (not recommended, security risk) ORCHESTRATOR_ISOLATION=0;; l) # Where to send logs ORCHESTRATOR_LOG="$OPTARG";; - L) # Comma separated list of labels to attach to the runner - RUNNER_LABELS="$OPTARG";; - m) # Memory to allocate to the VM - ORCHESTRATOR_MEMORY="$OPTARG";; - M) # Mount local host directories into the VM : - if [ -z "$ORCHESTRATOR_MOUNT" ]; then - ORCHESTRATOR_MOUNT="$OPTARG" - else - ORCHESTRATOR_MOUNT="$(printf %s\\n%s\\n "$ORCHESTRATOR_MOUNT" "$OPTARG")" - fi;; - n) # Name of the VM to create - ORCHESTRATOR_NAME="$OPTARG";; - p) # Principal to authorise the runner for, name of repo, org or enterprise - RUNNER_PRINCIPAL="$OPTARG";; - r) # Number of times to repeat the runner loop - RUNNER_REPEAT="$OPTARG";; - s) # Scope of the runner, one of repo, org or enterprise - RUNNER_SCOPE="$OPTARG";; - T) # Authorization token at the GitHub API to acquire runner token with - RUNNER_PAT="$OPTARG";; - u) # User to run the runner as - RUNNER_USER="$OPTARG";; - U) # Turn on auto-updating of the runner - RUNNER_UPDATE=1;; + n) # Number of runners to create + ORCHESTRATOR_RUNNERS="$OPTARG";; + p) # Prefix to use for the VM name + ORCHESTRATOR_PREFIX="$OPTARG";; v) # Increase verbosity, will otherwise log on errors/warnings only ORCHESTRATOR_VERBOSE=$((ORCHESTRATOR_VERBOSE+1));; h) # Print help and exit usage 0 "(ORCHESTRATOR|RUNNER)";; - -) # End of options. Single argument: number of runners to create + -) # End of options. All subsequent arguments are passed to the runner.sh script break;; ?) usage 1;; @@ -169,6 +90,16 @@ KRUNVM_RUNNER_VERBOSE=$ORCHESTRATOR_VERBOSE cleanup() { trap - INT TERM EXIT + + if run_krunvm list | grep -qE "^${ORCHESTRATOR_PREFIX}-"; then + while IFS= read -r vm; do + verbose "Removing microVM $vm" + run_krunvm delete "$vm" + done <VM mount points, lines containing pairs of directory mappings separated +# by a colon. +RUNNER_MOUNT=${ORCHESTRATOR_MOUNT:-""} + +# Name of top directory in VM where to host a copy of the root directory of this +# script. When this is set, the runner starter script from that directory will +# be used -- instead of the one already in the OCI image. This option is mainly +# usefull for development and testing. +RUNNER_DIR=${RUNNER_DIR:-""} + # GitHub host, e.g. github.com or github.example.com RUNNER_GITHUB=${RUNNER_GITHUB:-"github.com"} @@ -64,8 +88,8 @@ RUNNER_PAT=${RUNNER_PAT:-""} # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} -# Name of the microVM to run from -RUNNER_NAME=${RUNNER_NAME:-"runner"} +# Prefix to use for the VM names. The VM name will be $RUNNER_PREFIX-xxx +RUNNER_PREFIX=${RUNNER_PREFIX:-"GH-runner"} # Name of top directory in VM where to host a copy of the root directory of this # script. When this is set, the runner starter script from that directory will @@ -92,24 +116,32 @@ RUNNER_SECRET=${RUNNER_SECRET:-"$(random_string)"} KRUNVM_RUNNER_DESCR="Create runners forever using krunvm" -while getopts "D:E:g:G:l:L:M:n:p:r:s:S:T:u:Uvh-" opt; do +while getopts "c:d:D:g:G:i:l:L:m:M:p:r:s:S:T:u:Uvh-" opt; do case "$opt" in + c) # Number of CPUs to allocate to the VM + RUNNER_CPUS="$OPTARG";; + d) # DNS server to use in VM + RUNNER_DNS=$OPTARG;; D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing). RUNNER_DIR=$OPTARG;; - E) # Location (at host) where to place environment files for each run. - RUNNER_ENVIRONMENT="$OPTARG";; g) # GitHub host, e.g. github.com or github.example.com RUNNER_GITHUB="$OPTARG";; G) # Group to attach the runner to RUNNER_GROUP="$OPTARG";; + i) # Name of the OCI image (fully-qualified) to use. You need to have access. + RUNNER_IMAGE="$OPTARG";; l) # Where to send logs RUNNER_LOG="$OPTARG";; L) # Comma separated list of labels to attach to the runner RUNNER_LABELS="$OPTARG";; - M) # Mount passed to the microVM - RUNNER_MOUNT="$OPTARG";; - n) # Name of the microVM to run from - RUNNER_NAME="$OPTARG";; + m) # Memory to allocate to the VM + RUNNER_MEMORY="$OPTARG";; + M) # Mount local host directories into the VM : + if [ -z "$RUNNER_MOUNT" ]; then + RUNNER_MOUNT="$OPTARG" + else + RUNNER_MOUNT="$(printf %s\\n%s\\n "$RUNNER_MOUNT" "$OPTARG")" + fi;; p) # Principal to authorise the runner for, name of repo, org or enterprise RUNNER_PRINCIPAL="$OPTARG";; r) # Number of times to repeat the runner loop @@ -139,48 +171,86 @@ shift $((OPTIND-1)) # Pass logging configuration and level to imported scripts KRUNVM_RUNNER_LOG=$RUNNER_LOG KRUNVM_RUNNER_VERBOSE=$RUNNER_VERBOSE -loop=${1:-} +loop=${1:-"0"} if [ -n "${loop:-}" ]; then KRUNVM_RUNNER_BIN=$(basename "$0") KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$loop" fi +# Name of the root directory in the VM where to map environment and +# synchronisation files. +RUNNER_VM_ENVDIR="_environment" + +if [ -z "$RUNNER_PAT" ]; then + error "You need to specify a PAT to acquire the runner token with" +fi +check_positive_number "$RUNNER_CPUS" "Number of vCPUs" +check_positive_number "$RUNNER_MEMORY" "Memory (in MB)" + # Decide which runner.sh implementation (this is the "entrypoint" of the # microVM) to use: the one from the mount point, or the built-in one. if [ -z "$RUNNER_DIR" ]; then - runner=/opt/gh-runner-krunvm/bin/runner.sh + RUNNER_ENTRYPOINT=/opt/gh-runner-krunvm/bin/runner.sh else check_command "${RUNNER_ROOTDIR}/runner/runner.sh" - runner=${RUNNER_DIR%/}/runner/runner.sh + RUNNER_ENTRYPOINT=${RUNNER_DIR%/}/runner/runner.sh fi -iteration=0 -while true; do - id=$(random_string) - RUNNER_ID=${loop}-${id} - verbose "Starting microVM $RUNNER_NAME to run ephemeral GitHub runner $RUNNER_ID" +# Create the VM used for orchestration. Add --volume options for all necessary +# mappings, i.e. inheritance of "live" code, environment isolation and all +# requested mount points. +vm_create() { + verbose "Creating microVM '${RUNNER_PREFIX}-$1', $RUNNER_CPUS vCPUs, ${RUNNER_MEMORY}M memory" + # Note: reset arguments! + set -- \ + --cpus "$RUNNER_CPUS" \ + --mem "$RUNNER_MEMORY" \ + --dns "$RUNNER_DNS" \ + --name "${RUNNER_PREFIX}-$1" + if [ -n "${RUNNER_DIR:-}" ]; then + set -- "$@" --volume "${RUNNER_ROOTDIR}:${RUNNER_DIR}" + fi + if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then + set -- "$@" --volume "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}" + fi + if [ -n "$RUNNER_MOUNT" ]; then + while IFS= read -r mount || [ -n "$mount" ]; do + if [ -n "$mount" ]; then + set -- "$@" --volume "$mount" + fi + done <> "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.env" + printf '%s\n' "$varset" >> "${RUNNER_ENVIRONMENT}/${_id}.env" done < "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" verbose "Installing runner to $INSTALL_DIR" tar -C "${INSTALL_DIR}/runner-${INSTALL_VERSION}" -zxf "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" +rm -f "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" # Install the dependencies (this is distro specific and aware) "${INSTALL_DIR}/runner-${INSTALL_VERSION}/bin/installdependencies.sh" diff --git a/runner/runner.sh b/runner/runner.sh index a0ed6c0..37c4d07 100755 --- a/runner/runner.sh +++ b/runner/runner.sh @@ -79,9 +79,6 @@ RUNNER_EPHEMERAL=${RUNNER_EPHEMERAL:-"0"} # Root installation of the runner RUNNER_INSTALL=${RUNNER_INSTALL:-"/opt/gh-runner-krunvm/share/runner"} -# Permit several installations -RUNNER_MULTI=${RUNNER_MULTI:-"1"} - # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} @@ -156,23 +153,14 @@ KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$RUNNER_ID" # minimal verification of the installation through checking that there is a # config.sh script executable within the copy. runner_install() { - if [ "$RUNNER_MULTI" = 1 ]; then - # Make a directory where to install a copy of the runner. - if ! [ -d "${RUNNER_WORKDIR%/}/runner" ]; then - mkdir -p "${RUNNER_WORKDIR%/}/runner" - verbose "Created runner directory ${RUNNER_WORKDIR%/}/runner" - fi - verbose "Installing runner in ${RUNNER_WORKDIR%/}/runner" - tar -C "${RUNNER_WORKDIR%/}/runner" -zxf "$RUNNER_TAR" - else - if ! [ -d "${RUNNER_WORKDIR%/}" ]; then - mkdir -p "${RUNNER_WORKDIR%/}" - verbose "Created runner directory ${RUNNER_WORKDIR%/}" - fi - verbose "Moving runner installation to ${RUNNER_WORKDIR%/}/runner" - mv -f "$RUNNER_INSTDIR" "${RUNNER_WORKDIR%/}/runner" 2>/dev/null + if ! [ -d "${RUNNER_WORKDIR%/}" ]; then + mkdir -p "${RUNNER_WORKDIR%/}" + verbose "Created runner directory ${RUNNER_WORKDIR%/}" fi - check_command "${RUNNER_WORKDIR%/}/runner/config.sh" + RUNNER_BINROOT="${RUNNER_WORKDIR%/}/runner" + verbose "Copying runner installation to $RUNNER_BINROOT" + cp -rf "$RUNNER_INSTDIR" "$RUNNER_BINROOT" 2>/dev/null + check_command "${RUNNER_BINROOT}/config.sh" } @@ -284,7 +272,7 @@ runner_unregister() { # temporary changes directory before calling the script. runner_control() { cwd=$(pwd) - cd "${RUNNER_WORKDIR%/}/runner" + cd "$RUNNER_BINROOT" script=./${1}; shift check_command "$script" debug "Running $script $*" @@ -369,8 +357,8 @@ fi debug "Setting up missing defaults" distro=$(get_env "/etc/os-release" "ID") RUNNER_DISTRO=${RUNNER_DISTRO:-"${distro:-"unknown}"}"} -RUNNER_NAME_PREFIX=${RUNNER_NAME_PREFIX:-"${RUNNER_DISTRO}-krunvm"} -RUNNER_NAME=${RUNNER_NAME:-"${RUNNER_NAME_PREFIX}-$RUNNER_ID"} +RUNNER_PREFIX=${RUNNER_PREFIX:-"${RUNNER_DISTRO}-krunvm"} +RUNNER_NAME=${RUNNER_NAME:-"${RUNNER_PREFIX}-$RUNNER_ID"} RUNNER_WORKDIR=${RUNNER_WORKDIR:-"/_work/${RUNNER_NAME}"} if [ -n "${distro:-}" ]; then @@ -379,13 +367,13 @@ else RUNNER_LABELS=${RUNNER_LABELS:-"krunvm"} fi -RUNNER_TAR=$(find_pattern "${RUNNER_INSTALL}/*.tgz" f | sort -r | head -n 1) -if [ -z "$RUNNER_TAR" ]; then - error "No runner tar file found under $RUNNER_INSTALL" -fi +# Find the (versioned) directory containing the full installation of the runner +# binary distribution (unpacked by the installer) RUNNER_INSTDIR=$(find_pattern "${RUNNER_INSTALL}/runner-*" d | sort -r | head -n 1) if [ -z "$RUNNER_INSTDIR" ]; then error "No runner installation directory found under $RUNNER_INSTALL" +else + debug "Found unpacked binary distribution at $RUNNER_INSTDIR" fi # Construct the runner URL, i.e. where the runner will be registered