From 553de6cced9b75120b8aa7fe1a57e97a345fa536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Fri, 16 Feb 2024 23:38:02 +0100 Subject: [PATCH 1/8] Install in version-specific directory --- runner/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/install.sh b/runner/install.sh index 32a9652..2d58cd7 100755 --- a/runner/install.sh +++ b/runner/install.sh @@ -104,13 +104,13 @@ esac # Download and install the runner verbose "Downloading version ${INSTALL_VERSION} of the $INSTALL_ARCH runner" -mkdir -p "$INSTALL_DIR" +mkdir -p "${INSTALL_DIR}/runner-${INSTALL_VERSION}" curl -sSL "https://github.com/${INSTALL_PROJECT}/releases/download/v${INSTALL_VERSION}/actions-runner-linux-${INSTALL_ARCH}-${INSTALL_VERSION}.tar.gz" > "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" verbose "Installing runner to $INSTALL_DIR" -tar -C "$INSTALL_DIR" -zxf "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" +tar -C "${INSTALL_DIR}/runner-${INSTALL_VERSION}" -zxf "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" # Install the dependencies (this is distro specific and aware) -"${INSTALL_DIR}/bin/installdependencies.sh" +"${INSTALL_DIR}/runner-${INSTALL_VERSION}/bin/installdependencies.sh" # Create the directories for the environment. Ensure ownership if a user was # set. From a0b3a08d47002b8d505b4a549bf4a8f31a297a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Sat, 17 Feb 2024 00:10:56 +0100 Subject: [PATCH 2/8] Move (not unpack) runner directory --- CONTRIBUTING.md | 18 ++++++++++++++++++ runner/runner.sh | 30 +++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3309c67..a8ecd8e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,3 +33,21 @@ automatically removed as soon as the microVM has booted is running the `runner.sh` script, workflows are not able to break the external loop: they are able to create files in the `/_environment` directory, but they cannot know the value of the secret to put into the file to force the exiting handshake. + +## Changes to the Installation Scripts + +The installation of both images is handled by the [`base.sh`](./base/base.sh) +and [`install.sh`](./runner/install.sh). When making changes to these scripts, +or to the [`docker.sh`](./base/docker.sh) docker CLI wrapper, you will need to +wait for the results of the [`dev.yml`](./.github/workflows/dev.yml) workflow to +finish and for the resulting image to be published at the GHCR before being able +to test. The images will be published for amd64 only and with a tag named after +the name of the branch. Check out the "Inspect image" step of the `merge` job to +collect the fully-qualified name of the image. Once done, provide that name to +the `-i` option of the [`orchestrator.sh`](./orchestrator.sh) script. + +Note that when changing the logic of the "entrypoints", i.e. the scripts run at +microVM initialisation, you do not need to wait for the image to be created. +Instead, pass `-D /local` to the [`orchestrator.sh`](./orchestrator.sh) script. +This will mount the [`runner`](./runner/) directory into the microVM at `/local` +and run the scripts that it contains from there instead. diff --git a/runner/runner.sh b/runner/runner.sh index 95fe4fe..a283b32 100755 --- a/runner/runner.sh +++ b/runner/runner.sh @@ -79,6 +79,9 @@ RUNNER_EPHEMERAL=${RUNNER_EPHEMERAL:-"0"} # Root installation of the runner RUNNER_INSTALL=${RUNNER_INSTALL:-"/opt/gh-runner-krunvm/share/runner"} +# Permit several installations +RUNNER_MULTI=${RUNNER_MULTI:-"0"} + # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} @@ -153,13 +156,22 @@ KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$RUNNER_ID" # minimal verification of the installation through checking that there is a # config.sh script executable within the copy. runner_install() { - # Make a directory where to install a copy of the runner. - if ! [ -d "${RUNNER_WORKDIR%/}/runner" ]; then - mkdir -p "${RUNNER_WORKDIR%/}/runner" - verbose "Created runner directory ${RUNNER_WORKDIR%/}/runner" + if [ "$RUNNER_MULTI" = 1 ]; then + # Make a directory where to install a copy of the runner. + if ! [ -d "${RUNNER_WORKDIR%/}/runner" ]; then + mkdir -p "${RUNNER_WORKDIR%/}/runner" + verbose "Created runner directory ${RUNNER_WORKDIR%/}/runner" + fi + verbose "Installing runner in ${RUNNER_WORKDIR%/}/runner" + tar -C "${RUNNER_WORKDIR%/}/runner" -zxf "$RUNNER_TAR" + else + if ! [ -d "${RUNNER_WORKDIR%/}" ]; then + mkdir -p "${RUNNER_WORKDIR%/}" + verbose "Created runner directory ${RUNNER_WORKDIR%/}" + fi + verbose "Moving runner installation to ${RUNNER_WORKDIR%/}/runner" + mv -f "$RUNNER_INSTDIR" "${RUNNER_WORKDIR%/}/runner" 2>/dev/null fi - verbose "Installing runner in ${RUNNER_WORKDIR%/}/runner" - tar -C "${RUNNER_WORKDIR%/}/runner" -zxf "$RUNNER_TAR" check_command "${RUNNER_WORKDIR%/}/runner/config.sh" } @@ -367,10 +379,14 @@ else RUNNER_LABELS=${RUNNER_LABELS:-"krunvm"} fi -RUNNER_TAR=$(find "$RUNNER_INSTALL" -type f -name "*.tgz" | sort -r | head -n 1) +RUNNER_TAR=$(find_pattern "${RUNNER_INSTALL}/*.tgz" f | sort -r | head -n 1) if [ -z "$RUNNER_TAR" ]; then error "No runner tar file found under $RUNNER_INSTALL" fi +RUNNER_INSTDIR=$(find_pattern "${RUNNER_INSTALL}/runner-*" d | sort -r | head -n 1) +if [ -z "$RUNNER_INSTDIR" ]; then + error "No runner installation directory found under $RUNNER_INSTALL" +fi # Construct the runner URL, i.e. where the runner will be registered RUNNER_SCOPE=$(to_lower "$RUNNER_SCOPE") From e5cf11215ebff64fb8dfa65e12fab45013e1527c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Sat, 17 Feb 2024 00:13:23 +0100 Subject: [PATCH 3/8] Turn off for the time being --- runner/runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.sh b/runner/runner.sh index a283b32..a0ed6c0 100755 --- a/runner/runner.sh +++ b/runner/runner.sh @@ -80,7 +80,7 @@ RUNNER_EPHEMERAL=${RUNNER_EPHEMERAL:-"0"} RUNNER_INSTALL=${RUNNER_INSTALL:-"/opt/gh-runner-krunvm/share/runner"} # Permit several installations -RUNNER_MULTI=${RUNNER_MULTI:-"0"} +RUNNER_MULTI=${RUNNER_MULTI:-"1"} # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} From 5060eae82e8c3e004bf42b889db367649ad037dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Sun, 18 Feb 2024 23:07:31 +0100 Subject: [PATCH 4/8] Add number validation functions --- lib/common.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/common.sh b/lib/common.sh index 7b36716..d5fb366 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -104,6 +104,27 @@ wait_path() { done } +check_number() { + if ! printf %d\\n "$1" >/dev/null 2>&1; then + if [ -n "${2:-}" ]; then + error "$2 is an invalid number: $1" + else + error "Invalid number: $1" + fi + fi +} + +check_positive_number() { + check_number "$1" "$2" + if [ "$1" -le 0 ]; then + if [ -n "${2:-}" ]; then + error "$2 must be a positive number: $1" + else + error "Invalid positive number: $1" + fi + fi +} + # PML: Poor Man's Logging _log() { From 7a27ce7bd01b95af77d909a15a395c8476b64f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Sun, 18 Feb 2024 23:08:09 +0100 Subject: [PATCH 5/8] Lessen size of random strings --- lib/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.sh b/lib/common.sh index d5fb366..feaa8f6 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -33,7 +33,7 @@ is_true() { # shellcheck disable=SC2120 # Function has good default. random_string() { - LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c "${1:-12}" + LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c "${1:-7}" } usage() { From fa8a5ed79340c2bb917278f0da33610da41f14cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Sun, 18 Feb 2024 23:47:32 +0100 Subject: [PATCH 6/8] Migrate VM creation to runner Change the logic to adapt to the semantics of the microVMs lifecycle. MicroVMs are created, started and deleted by the runner loops instead. The orchestrator is only responsible for creating the loops. As a result, many ORCHESTRATOR_ variables (and their options) have been moved to the runner.sh loop script, and renamed to RUNNER_. The orchestrator now accepts much fewer options. Instead, options should be passed after a -- at the command-line, and these are blindly passed to each runner loop. This means that the number of loops to create is not an argument anymore, but rather an option to the orchestrator (defaults to 1). This is a breaking change. Also the tgz binary distribution is now removed during installation, and, instead the runner-specific installation is (recursively) copied to the target directory. This is to avoid the need to have the tgz distribution in the runner's directory, but also because copying is actually quicker than moving files (hierarchies). --- orchestrator.sh | 253 +++++++++++++--------------------------------- runner.sh | 142 +++++++++++++++++++++----- runner/install.sh | 1 + runner/runner.sh | 40 +++----- 4 files changed, 201 insertions(+), 235 deletions(-) diff --git a/orchestrator.sh b/orchestrator.sh index 2f77b80..aac0706 100755 --- a/orchestrator.sh +++ b/orchestrator.sh @@ -39,32 +39,12 @@ ORCHESTRATOR_VERBOSE=${ORCHESTRATOR_VERBOSE:-0} # Where to send logs ORCHESTRATOR_LOG=${ORCHESTRATOR_LOG:-2} -# Name of the OCI image (fully-qualified) to use. You need to have access. -ORCHESTRATOR_IMAGE=${ORCHESTRATOR_IMAGE:-"ghcr.io/efrecon/runner-krunvm:main"} +# Number of runners to create +ORCHESTRATOR_RUNNERS=${ORCHESTRATOR_RUNNERS:-1} -# Memory to allocate to the VM (in MB). Regular runners use more than the -# default. -ORCHESTRATOR_MEMORY=${ORCHESTRATOR_MEMORY:-"1024"} - -# Number of vCPUs to allocate to the VM. Regular runners use more than the -# default. -ORCHESTRATOR_CPUS=${ORCHESTRATOR_CPUS:-"2"} - -# Name of the VM to create (krunvm create) -ORCHESTRATOR_NAME=${ORCHESTRATOR_NAME:-"runner"} - -# DNS to use on the VM. This is the same as the default in krunvm. -ORCHESTRATOR_DNS=${ORCHESTRATOR_DNS:-"1.1.1.1"} - -# Host->VM mount points, lines containing pairs of directory mappings separated -# by a colon. -ORCHESTRATOR_MOUNT=${ORCHESTRATOR_MOUNT:-""} - -# Name of top directory in VM where to host a copy of the root directory of this -# script. When this is set, the runner starter script from that directory will -# be used -- instead of the one already in the OCI image. This option is mainly -# usefull for development and testing. -ORCHESTRATOR_DIR=${ORCHESTRATOR_DIR:-""} +# Prefix to use for the VM name. The VM name will be $ORCHESTRATOR_PREFIX-xxx. +# All VMs prefixed with this name will be deleted on exit. +ORCHESTRATOR_PREFIX=${ORCHESTRATOR_PREFIX:-"GH-runner"} # Should the runner be isolated in its own environment. This will pass all # configuration down to the runner starter script as an environment variable to @@ -72,90 +52,31 @@ ORCHESTRATOR_DIR=${ORCHESTRATOR_DIR:-""} # used. ORCHESTRATOR_ISOLATION=${ORCHESTRATOR_ISOLATION:-"1"} -# Number of seconds to sleep between microVM creation at start +# Number of seconds to sleep between microVM creation at start, unless isolation +# has been turned on. ORCHESTRATOR_SLEEP=${ORCHESTRATOR_SLEEP:-"30"} -# GitHub host, e.g. github.com or github.example.com -RUNNER_GITHUB=${RUNNER_GITHUB:-"github.com"} - -# Group to attach the runner to -RUNNER_GROUP=${RUNNER_GROUP:-"Default"} - -# Comma separated list of labels to attach to the runner (good defaults will be used if empty) -RUNNER_LABELS=${RUNNER_LABELS:-""} - -# Name of the user to run the runner as, defaults to runner. User must exist. -RUNNER_USER=${RUNNER_USER:-"runner"} - -# Scope of the runner, one of: repo, org or enterprise -RUNNER_SCOPE=${RUNNER_SCOPE:-"repo"} - -# Name of organisation, enterprise or repo to attach the runner to, when -# relevant scope. -RUNNER_PRINCIPAL=${RUNNER_PRINCIPAL:-""} - -# PAT to acquire runner token with -RUNNER_PAT=${RUNNER_PAT:-""} - -# Should the runner auto-update -RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} - -# Number of times to repeat the runner loop -RUNNER_REPEAT=${RUNNER_REPEAT:-"-1"} - # shellcheck disable=SC2034 # Used in sourced scripts KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host" -while getopts "a:c:d:D:g:G:i:Il:L:m:M:n:p:r:s:t:T:u:Uvh-" opt; do +while getopts "s:Il:n:p:vh-" opt; do case "$opt" in - a) # Number of seconds to sleep between microVM creation at start, when no isolation + s) # Number of seconds to sleep between microVM creation at start, when no isolation ORCHESTRATOR_SLEEP="$OPTARG";; - c) # Number of CPUs to allocate to the VM - ORCHESTRATOR_CPUS="$OPTARG";; - d) # DNS server to use in VM - ORCHESTRATOR_DNS=$OPTARG;; - D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing). - ORCHESTRATOR_DIR=$OPTARG;; - g) # GitHub host, e.g. github.com or github.example.com - RUNNER_GITHUB="$OPTARG";; - G) # Group to attach the runners to - RUNNER_GROUP="$OPTARG";; - i) # Fully-qualified name of the OCI image to use - ORCHESTRATOR_IMAGE="$OPTARG";; I) # Turn off variables isolation (not recommended, security risk) ORCHESTRATOR_ISOLATION=0;; l) # Where to send logs ORCHESTRATOR_LOG="$OPTARG";; - L) # Comma separated list of labels to attach to the runner - RUNNER_LABELS="$OPTARG";; - m) # Memory to allocate to the VM - ORCHESTRATOR_MEMORY="$OPTARG";; - M) # Mount local host directories into the VM : - if [ -z "$ORCHESTRATOR_MOUNT" ]; then - ORCHESTRATOR_MOUNT="$OPTARG" - else - ORCHESTRATOR_MOUNT="$(printf %s\\n%s\\n "$ORCHESTRATOR_MOUNT" "$OPTARG")" - fi;; - n) # Name of the VM to create - ORCHESTRATOR_NAME="$OPTARG";; - p) # Principal to authorise the runner for, name of repo, org or enterprise - RUNNER_PRINCIPAL="$OPTARG";; - r) # Number of times to repeat the runner loop - RUNNER_REPEAT="$OPTARG";; - s) # Scope of the runner, one of repo, org or enterprise - RUNNER_SCOPE="$OPTARG";; - T) # Authorization token at the GitHub API to acquire runner token with - RUNNER_PAT="$OPTARG";; - u) # User to run the runner as - RUNNER_USER="$OPTARG";; - U) # Turn on auto-updating of the runner - RUNNER_UPDATE=1;; + n) # Number of runners to create + ORCHESTRATOR_RUNNERS="$OPTARG";; + p) # Prefix to use for the VM name + ORCHESTRATOR_PREFIX="$OPTARG";; v) # Increase verbosity, will otherwise log on errors/warnings only ORCHESTRATOR_VERBOSE=$((ORCHESTRATOR_VERBOSE+1));; h) # Print help and exit usage 0 "(ORCHESTRATOR|RUNNER)";; - -) # End of options. Single argument: number of runners to create + -) # End of options. All subsequent arguments are passed to the runner.sh script break;; ?) usage 1;; @@ -169,6 +90,16 @@ KRUNVM_RUNNER_VERBOSE=$ORCHESTRATOR_VERBOSE cleanup() { trap - INT TERM EXIT + + if run_krunvm list | grep -qE "^${ORCHESTRATOR_PREFIX}-"; then + while IFS= read -r vm; do + verbose "Removing microVM $vm" + run_krunvm delete "$vm" + done <VM mount points, lines containing pairs of directory mappings separated +# by a colon. +RUNNER_MOUNT=${ORCHESTRATOR_MOUNT:-""} + +# Name of top directory in VM where to host a copy of the root directory of this +# script. When this is set, the runner starter script from that directory will +# be used -- instead of the one already in the OCI image. This option is mainly +# usefull for development and testing. +RUNNER_DIR=${RUNNER_DIR:-""} + # GitHub host, e.g. github.com or github.example.com RUNNER_GITHUB=${RUNNER_GITHUB:-"github.com"} @@ -64,8 +88,8 @@ RUNNER_PAT=${RUNNER_PAT:-""} # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} -# Name of the microVM to run from -RUNNER_NAME=${RUNNER_NAME:-"runner"} +# Prefix to use for the VM names. The VM name will be $RUNNER_PREFIX-xxx +RUNNER_PREFIX=${RUNNER_PREFIX:-"GH-runner"} # Name of top directory in VM where to host a copy of the root directory of this # script. When this is set, the runner starter script from that directory will @@ -92,24 +116,32 @@ RUNNER_SECRET=${RUNNER_SECRET:-"$(random_string)"} KRUNVM_RUNNER_DESCR="Create runners forever using krunvm" -while getopts "D:E:g:G:l:L:M:n:p:r:s:S:T:u:Uvh-" opt; do +while getopts "c:d:D:g:G:i:l:L:m:M:p:r:s:S:T:u:Uvh-" opt; do case "$opt" in + c) # Number of CPUs to allocate to the VM + RUNNER_CPUS="$OPTARG";; + d) # DNS server to use in VM + RUNNER_DNS=$OPTARG;; D) # Local top VM directory where to host a copy of the root directory of this script (for dev and testing). RUNNER_DIR=$OPTARG;; - E) # Location (at host) where to place environment files for each run. - RUNNER_ENVIRONMENT="$OPTARG";; g) # GitHub host, e.g. github.com or github.example.com RUNNER_GITHUB="$OPTARG";; G) # Group to attach the runner to RUNNER_GROUP="$OPTARG";; + i) # Name of the OCI image (fully-qualified) to use. You need to have access. + RUNNER_IMAGE="$OPTARG";; l) # Where to send logs RUNNER_LOG="$OPTARG";; L) # Comma separated list of labels to attach to the runner RUNNER_LABELS="$OPTARG";; - M) # Mount passed to the microVM - RUNNER_MOUNT="$OPTARG";; - n) # Name of the microVM to run from - RUNNER_NAME="$OPTARG";; + m) # Memory to allocate to the VM + RUNNER_MEMORY="$OPTARG";; + M) # Mount local host directories into the VM : + if [ -z "$RUNNER_MOUNT" ]; then + RUNNER_MOUNT="$OPTARG" + else + RUNNER_MOUNT="$(printf %s\\n%s\\n "$RUNNER_MOUNT" "$OPTARG")" + fi;; p) # Principal to authorise the runner for, name of repo, org or enterprise RUNNER_PRINCIPAL="$OPTARG";; r) # Number of times to repeat the runner loop @@ -139,48 +171,86 @@ shift $((OPTIND-1)) # Pass logging configuration and level to imported scripts KRUNVM_RUNNER_LOG=$RUNNER_LOG KRUNVM_RUNNER_VERBOSE=$RUNNER_VERBOSE -loop=${1:-} +loop=${1:-"0"} if [ -n "${loop:-}" ]; then KRUNVM_RUNNER_BIN=$(basename "$0") KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$loop" fi +# Name of the root directory in the VM where to map environment and +# synchronisation files. +RUNNER_VM_ENVDIR="_environment" + +if [ -z "$RUNNER_PAT" ]; then + error "You need to specify a PAT to acquire the runner token with" +fi +check_positive_number "$RUNNER_CPUS" "Number of vCPUs" +check_positive_number "$RUNNER_MEMORY" "Memory (in MB)" + # Decide which runner.sh implementation (this is the "entrypoint" of the # microVM) to use: the one from the mount point, or the built-in one. if [ -z "$RUNNER_DIR" ]; then - runner=/opt/gh-runner-krunvm/bin/runner.sh + RUNNER_ENTRYPOINT=/opt/gh-runner-krunvm/bin/runner.sh else check_command "${RUNNER_ROOTDIR}/runner/runner.sh" - runner=${RUNNER_DIR%/}/runner/runner.sh + RUNNER_ENTRYPOINT=${RUNNER_DIR%/}/runner/runner.sh fi -iteration=0 -while true; do - id=$(random_string) - RUNNER_ID=${loop}-${id} - verbose "Starting microVM $RUNNER_NAME to run ephemeral GitHub runner $RUNNER_ID" +# Create the VM used for orchestration. Add --volume options for all necessary +# mappings, i.e. inheritance of "live" code, environment isolation and all +# requested mount points. +vm_create() { + verbose "Creating microVM '${RUNNER_PREFIX}-$1', $RUNNER_CPUS vCPUs, ${RUNNER_MEMORY}M memory" + # Note: reset arguments! + set -- \ + --cpus "$RUNNER_CPUS" \ + --mem "$RUNNER_MEMORY" \ + --dns "$RUNNER_DNS" \ + --name "${RUNNER_PREFIX}-$1" + if [ -n "${RUNNER_DIR:-}" ]; then + set -- "$@" --volume "${RUNNER_ROOTDIR}:${RUNNER_DIR}" + fi + if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then + set -- "$@" --volume "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}" + fi + if [ -n "$RUNNER_MOUNT" ]; then + while IFS= read -r mount || [ -n "$mount" ]; do + if [ -n "$mount" ]; then + set -- "$@" --volume "$mount" + fi + done <> "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.env" + printf '%s\n' "$varset" >> "${RUNNER_ENVIRONMENT}/${_id}.env" done < "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" verbose "Installing runner to $INSTALL_DIR" tar -C "${INSTALL_DIR}/runner-${INSTALL_VERSION}" -zxf "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" +rm -f "${INSTALL_DIR}/${INSTALL_VERSION}.tgz" # Install the dependencies (this is distro specific and aware) "${INSTALL_DIR}/runner-${INSTALL_VERSION}/bin/installdependencies.sh" diff --git a/runner/runner.sh b/runner/runner.sh index a0ed6c0..37c4d07 100755 --- a/runner/runner.sh +++ b/runner/runner.sh @@ -79,9 +79,6 @@ RUNNER_EPHEMERAL=${RUNNER_EPHEMERAL:-"0"} # Root installation of the runner RUNNER_INSTALL=${RUNNER_INSTALL:-"/opt/gh-runner-krunvm/share/runner"} -# Permit several installations -RUNNER_MULTI=${RUNNER_MULTI:-"1"} - # Should the runner auto-update RUNNER_UPDATE=${RUNNER_UPDATE:-"0"} @@ -156,23 +153,14 @@ KRUNVM_RUNNER_BIN="${KRUNVM_RUNNER_BIN%.sh}-$RUNNER_ID" # minimal verification of the installation through checking that there is a # config.sh script executable within the copy. runner_install() { - if [ "$RUNNER_MULTI" = 1 ]; then - # Make a directory where to install a copy of the runner. - if ! [ -d "${RUNNER_WORKDIR%/}/runner" ]; then - mkdir -p "${RUNNER_WORKDIR%/}/runner" - verbose "Created runner directory ${RUNNER_WORKDIR%/}/runner" - fi - verbose "Installing runner in ${RUNNER_WORKDIR%/}/runner" - tar -C "${RUNNER_WORKDIR%/}/runner" -zxf "$RUNNER_TAR" - else - if ! [ -d "${RUNNER_WORKDIR%/}" ]; then - mkdir -p "${RUNNER_WORKDIR%/}" - verbose "Created runner directory ${RUNNER_WORKDIR%/}" - fi - verbose "Moving runner installation to ${RUNNER_WORKDIR%/}/runner" - mv -f "$RUNNER_INSTDIR" "${RUNNER_WORKDIR%/}/runner" 2>/dev/null + if ! [ -d "${RUNNER_WORKDIR%/}" ]; then + mkdir -p "${RUNNER_WORKDIR%/}" + verbose "Created runner directory ${RUNNER_WORKDIR%/}" fi - check_command "${RUNNER_WORKDIR%/}/runner/config.sh" + RUNNER_BINROOT="${RUNNER_WORKDIR%/}/runner" + verbose "Copying runner installation to $RUNNER_BINROOT" + cp -rf "$RUNNER_INSTDIR" "$RUNNER_BINROOT" 2>/dev/null + check_command "${RUNNER_BINROOT}/config.sh" } @@ -284,7 +272,7 @@ runner_unregister() { # temporary changes directory before calling the script. runner_control() { cwd=$(pwd) - cd "${RUNNER_WORKDIR%/}/runner" + cd "$RUNNER_BINROOT" script=./${1}; shift check_command "$script" debug "Running $script $*" @@ -369,8 +357,8 @@ fi debug "Setting up missing defaults" distro=$(get_env "/etc/os-release" "ID") RUNNER_DISTRO=${RUNNER_DISTRO:-"${distro:-"unknown}"}"} -RUNNER_NAME_PREFIX=${RUNNER_NAME_PREFIX:-"${RUNNER_DISTRO}-krunvm"} -RUNNER_NAME=${RUNNER_NAME:-"${RUNNER_NAME_PREFIX}-$RUNNER_ID"} +RUNNER_PREFIX=${RUNNER_PREFIX:-"${RUNNER_DISTRO}-krunvm"} +RUNNER_NAME=${RUNNER_NAME:-"${RUNNER_PREFIX}-$RUNNER_ID"} RUNNER_WORKDIR=${RUNNER_WORKDIR:-"/_work/${RUNNER_NAME}"} if [ -n "${distro:-}" ]; then @@ -379,13 +367,13 @@ else RUNNER_LABELS=${RUNNER_LABELS:-"krunvm"} fi -RUNNER_TAR=$(find_pattern "${RUNNER_INSTALL}/*.tgz" f | sort -r | head -n 1) -if [ -z "$RUNNER_TAR" ]; then - error "No runner tar file found under $RUNNER_INSTALL" -fi +# Find the (versioned) directory containing the full installation of the runner +# binary distribution (unpacked by the installer) RUNNER_INSTDIR=$(find_pattern "${RUNNER_INSTALL}/runner-*" d | sort -r | head -n 1) if [ -z "$RUNNER_INSTDIR" ]; then error "No runner installation directory found under $RUNNER_INSTALL" +else + debug "Found unpacked binary distribution at $RUNNER_INSTDIR" fi # Construct the runner URL, i.e. where the runner will be registered From 86e8a9886767022cadad599e74689a4c4202c193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Mon, 19 Feb 2024 00:10:48 +0100 Subject: [PATCH 7/8] Adapt documentation --- CONTRIBUTING.md | 13 +++++++---- README.md | 58 +++++++++++++++++++++++++++---------------------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8ecd8e..9a647f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,11 @@ This document contains notes about the internals of the implementation. +> [!TIP] +> The [orchestrator](./orchestrator.sh) takes few options. Run it with a `--`, +> all options after that separator will be blindly passed to the +> [runner](./runner.sh), which is the script with most user-facing options. + ## Signalling Between Processes When environment isolation is turned on, i.e. when the variable @@ -44,10 +49,10 @@ finish and for the resulting image to be published at the GHCR before being able to test. The images will be published for amd64 only and with a tag named after the name of the branch. Check out the "Inspect image" step of the `merge` job to collect the fully-qualified name of the image. Once done, provide that name to -the `-i` option of the [`orchestrator.sh`](./orchestrator.sh) script. +the `-i` option of the [`runner.sh`](./runner.sh) script. Note that when changing the logic of the "entrypoints", i.e. the scripts run at microVM initialisation, you do not need to wait for the image to be created. -Instead, pass `-D /local` to the [`orchestrator.sh`](./orchestrator.sh) script. -This will mount the [`runner`](./runner/) directory into the microVM at `/local` -and run the scripts that it contains from there instead. +Instead, pass `-D /local` to the [`runner.sh`](./runner.sh) script. This will +mount the [`runner`](./runner/) directory into the microVM at `/local` and run +the scripts that it contains from there instead. diff --git a/README.md b/README.md index 5b5e7ba..7fc8028 100644 --- a/README.md +++ b/README.md @@ -17,36 +17,35 @@ multi-platform OCI [images][image] created for this project. ## Example Provided you are at the root directory of this project, the following would -create two runner loops that are bound to *this* repository (the -`efrecon/gh-runner-krunvm` principal). Runners can also be registered at the -`organization` or `enterprise` scope using the `-s` option. In the example +create two runner loops (the `-n` option) that are bound to *this* repository +(the `efrecon/gh-runner-krunvm` principal). Runners can also be registered at +the `organization` or `enterprise` scope using the `-s` option. In the example below, the value of the `-T` option should be a [PAT]. In each loop, as soon as one job has been picked up and executed, a new pristine runner will be created and registered. - ```bash -./orchestrator.sh -v -T ghp_XXXX -p efrecon/gh-runner-krunvm -- 2 +./orchestrator.sh -v -n 2 -- -T ghp_XXXX -p efrecon/gh-runner-krunvm ``` -The project tries to have good default options and behaviour -- run with `-h` -for a list of options. For example, nor the value of the token, nor the value of -the runner registration token will be visible to the workflows using your -runners. The default is however to create far-less capable runners than the -GitHub [runners], i.e. 1G or memory and 2 vCPUs. By default, runners have random -names and carry labels with the name of the base repository, e.g. `fedora` and -`krunvm`. The GitHub runner implementation will automatically add other labels -in addition to those. +The project tries to have good default options and behaviour. For example, nor +the value of the token, nor the value of the runner registration token will be +visible to the workflows using your runners. The default is however to create +far-less capable runners than the GitHub [runners], i.e. 1G or memory and 2 +vCPUs. By default, runners have random names and carry labels with the name of +the base repository, e.g. `fedora` and `krunvm`. The GitHub runner +implementation will automatically add other labels in addition to those. All scripts within the project accepts short options only and can either be controlled through options or environment variables. Running with the `-h` -option will provide help and a list of those variables. Variables starting with -`ORCHESTRATOR_` will affect the behaviour of the +option will provide help and a list of those variables. From the command-line, +you will only be running one script: the [orchestrator](./orchestrator.sh). +However, runner loops are created using the [runner](./runner.sh) script, by the +orchestrator. At the orchestrator CLI, options that appear after the `--` will +be blindly passed to the runner loop and script. Environment variables starting +with `ORCHESTRATOR_` will affect the behaviour of the [orchestrator](./orchestrator.sh), while variables starting with `RUNNER_` will -affect the behaviour of each runner. Usually, the only script that you will have -to use is the [orchestrator](./orchestrator.sh). However, it is possible to -create the microVM with the orchestrator and manually run loops using the -[runner](./runner.sh) script. +affect the behaviour of each runner (loop). [PAT]: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens @@ -75,7 +74,7 @@ create the microVM with the orchestrator and manually run loops using the ## Requirements This project is coded in pure POSIX shell and has only been tested on Linux. The -images are automatically [built] both for x86_64 and AArch64. However, [krunvm] +images are automatically [built] both for amd64 and arm64. However, [krunvm] also runs on MacOS. No "esoteric" options have been used when using the standard UNIX binary utilities. PRs are welcome to make the project work on MacOS, if it does not already. @@ -106,12 +105,13 @@ installed on the host. Installation is easiest on Fedora ## Architecture and Design -The [orchestrator](./orchestrator.sh) focuses on creating (but not starting) a -microVM based on the default OCI image (see below). It then creates as many -loops of ephemeral runners as requested. These loops are implemented as part of -the [runner.sh](./runner.sh) script: the script will start a microVM that will -start an (ephemeral) [runner][self]. As soon as a job has been executed on that -runner, the microVM will end and a new will be created. +The [orchestrator](./orchestrator.sh) creates as many loops of ephemeral runners +as requested. These loops are implemented as part of the +[runner.sh](./runner.sh) script: the script will create a microVM based on the +default image (see below), memory and vCPU requirement. It will then start that +microVM using `krunvm` and that will start an (ephemeral) [runner][self]. As +soon as a job has been executed on that runner, the microVM will end and a new +will be created. The OCI image is built in two parts: @@ -143,6 +143,12 @@ user. The `runner` user shares the same id as the one at GitHub and is also a member of the `docker` group. Similarily to GitHub runners, the user is capable of `sudo` without a password. +Runner tokens are written to the directory that is shared with the host. This is +used during initial synchronisation, to avoid starting up several runners at the +same time from the main orchestrator loop. The tokens are automatically removed +as soon as the runner is up, they are also protected so that the `runner` user +cannot read their content. + ## History This project was written to combat my anxeity combatting my daughter's newly From 57dd8a1b1324e79a39d4500248a16473defd5d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= Date: Mon, 19 Feb 2024 00:16:21 +0100 Subject: [PATCH 8/8] Fix typo --- runner.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner.sh b/runner.sh index ba7879a..d7e669a 100755 --- a/runner.sh +++ b/runner.sh @@ -51,11 +51,11 @@ RUNNER_MEMORY=${RUNNER_MEMORY:-"1024"} RUNNER_CPUS=${RUNNER_CPUS:-"2"} # DNS to use on the VM. This is the same as the default in krunvm. -RUNNER_DNS=${ORCHESTRATOR_DNS:-"1.1.1.1"} +RUNNER_DNS=${RUNNER_DNS:-"1.1.1.1"} # Host->VM mount points, lines containing pairs of directory mappings separated # by a colon. -RUNNER_MOUNT=${ORCHESTRATOR_MOUNT:-""} +RUNNER_MOUNT=${RUNNER_MOUNT:-""} # Name of top directory in VM where to host a copy of the root directory of this # script. When this is set, the runner starter script from that directory will