From 892ab3c78484a68538e16ddfbc4d394bbb868229 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 5 Apr 2024 14:15:57 +0200
Subject: [PATCH 01/13] Add dev note for image cleanup

---
 CONTRIBUTING.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1e32e11..88fc951 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -67,3 +67,12 @@ Instead, pass `-D /local` to the [`runner.sh`](./runner.sh) script. This will
 mount the [`runner`](./runner/) directory into the microVM at `/local` and run
 the scripts that it contains from there instead. Which "entrypoint" to use is
 driven by the `RUNNER_ENTRYPOINT` variable in [`runner.sh`](./runner.sh).
+
+## Cleanup
+
+During development, many images might be created. To clean them away, you can
+run the following:
+
+```bash
+buildah rmi $(buildah images --format '{{.ID}}')
+```

From 970043c1f05d9cab84a8e017f57955a7041d0907 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 5 Apr 2024 14:16:09 +0200
Subject: [PATCH 02/13] Add generic microvm API lib and use it

Add a generic API to create microVMs in an "agnostic" way, and migrate
to the new API in the orchestrator.sh and runner.sh scripts. While the
API has code for podman, this update only focuses on migrating the
existing krunvm-based code to the new API.

The new API is implemented in lib/microvm.sh, and provides a set of
docker/podman lookalike commands to manage microVMs. The function
microvm_runtime can be used to pinpoint or auto-detect the runtime to
use depending on commands available at the host. The API is **not** able
to handle several krunvm-based microVMs at the same time.
---
 lib/common.sh   |  28 +++--
 lib/microvm.sh  | 269 ++++++++++++++++++++++++++++++++++++++++++++++++
 orchestrator.sh |  28 +++--
 runner.sh       | 157 ++++++++++++++--------------
 4 files changed, 389 insertions(+), 93 deletions(-)
 create mode 100644 lib/microvm.sh

diff --git a/lib/common.sh b/lib/common.sh
index 9fbab23..c382d9f 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -51,9 +51,30 @@ usage() {
 }
 
 check_command() {
+  OPTIND=1
+  _hard=1
+  _warn=0
+  while getopts "sw-" _opt; do
+    case "$_opt" in
+      s) # Soft check, return an error code instead of exiting
+        _hard=0;;
+      w) # Print a warning when soft checking
+        _warn=1;;
+      -) # End of options, everything after is the command
+        break;;
+      ?)
+        error "$_opt is an unrecognised option";;
+    esac
+  done
+  shift $((OPTIND-1))
   trace "Checking $1 is an accessible command"
   if ! command -v "$1" >/dev/null 2>&1; then
-    error "Command not found: $1"
+    if is_true "$_hard"; then
+      error "Command not found: $1"
+    elif is_true "$_warn"; then
+      warn "Command not found: $1"
+    fi
+    return 1
   fi
 }
 
@@ -70,11 +91,6 @@ get_env() (
   fi
 )
 
-run_krunvm() {
-  debug "Running krunvm $*"
-  buildah unshare krunvm "$@"
-}
-
 tac() {
   awk '{ buffer[NR] = $0; } END { for(i=NR; i>0; i--) { print buffer[i] } }'
 }
diff --git a/lib/microvm.sh b/lib/microvm.sh
new file mode 100644
index 0000000..5262210
--- /dev/null
+++ b/lib/microvm.sh
@@ -0,0 +1,269 @@
+#!/bin/sh
+
+# This implements an API to manage microVMs using krunvm and podman.
+
+
+# Runtime to use for microVMs: podman+krun, krunvm. When empty, it will be
+# automatically selected.
+: "${KRUNVM_RUNNER_RUNTIME:=""}"
+
+# Run krunvm with the provided arguments, behind a buildah unshare.
+run_krunvm() {
+  debug "Running krunvm $*"
+  buildah unshare krunvm "$@"
+}
+
+
+# Automatically select a microVM runtime based on the available commands. Set
+# the KRUNVM_RUNNER_RUNTIME variable.
+_microvm_runtime_auto() {
+  if check_command -s -- krun; then
+    check_command podman
+    KRUNVM_RUNNER_RUNTIME="podman+krun"
+  elif check_command -s -- krunvm; then
+    check_command buildah
+    KRUNVM_RUNNER_RUNTIME="krunvm"
+  fi
+  info "Automatically selected $KRUNVM_RUNNER_RUNTIME to handle microVMs"
+}
+
+
+# Set the microVM runtime to use. When no argument is provided, it will try to
+# automatically detect it based on the available commands.
+# shellcheck disable=SC2120
+microvm_runtime() {
+  # Pick runtime provided as an argument, when available.
+  [ "$#" -gt 0 ] && KRUNVM_RUNNER_RUNTIME="$1"
+
+  # When no runtime is provided, try to auto-detect it.
+  [ -z "${KRUNVM_RUNNER_RUNTIME:-""}" ] && _microvm_runtime_auto
+
+  # Enforce podman+krun as soon as anything starting podman is provided.
+  [ "${KRUNVM_RUNNER_RUNTIME#podman}" != "$KRUNVM_RUNNER_RUNTIME" ] && KRUNVM_RUNNER_RUNTIME="podman+krun"
+
+  # Check if the runtime is valid.
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      check_command podman
+      check_command krun
+      ;;
+    krunvm)
+      check_command krunvm
+      check_command buildah
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+
+# List all microVMs.
+microvm_list() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      podman ps -a --format "{{.Names}}"
+      ;;
+    krunvm)
+      run_krunvm list
+      ;;
+  esac
+}
+
+
+_krunvm_create() {
+  KRUNVM_RUNNER_IMAGE=$1
+  verbose "Creating microVM '${KRUNVM_RUNNER_NAME}', $KRUNVM_RUNNER_CPUS vCPUs, ${KRUNVM_RUNNER_MEM}M memory"
+  # Note: reset arguments!
+  set -- \
+    --cpus "$KRUNVM_RUNNER_CPUS" \
+    --mem "$KRUNVM_RUNNER_MEM" \
+    --dns "$KRUNVM_RUNNER_DNS" \
+    --name "$KRUNVM_RUNNER_NAME"
+  if [ -n "$KRUNVM_RUNNER_VOLS" ]; then
+    while IFS= read -r mount || [ -n "$mount" ]; do
+      if [ -n "$mount" ]; then
+        set -- "$@" --volume "$mount"
+      fi
+    done <<EOF
+$(printf %s\\n "$KRUNVM_RUNNER_VOLS")
+EOF
+  fi
+  run_krunvm create "$KRUNVM_RUNNER_IMAGE" "$@"
+}
+
+
+microvm_run() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  OPTIND=1
+  KRUNVM_RUNNER_CPUS=0
+  KRUNVM_RUNNER_MEM=0
+  KRUNVM_RUNNER_DNS="1.1.1.1"
+  KRUNVM_RUNNER_NAME=""
+  KRUNVM_RUNNER_VOLS=""
+  KRUNVM_RUNNER_ENTRYPOINT=""
+  while getopts "c:d:e:m:n:v:-" _opt; do
+    case "$_opt" in
+      c) # Number of CPUs
+        KRUNVM_RUNNER_CPUS="$OPTARG";;
+      d) # DNS Server
+        KRUNVM_RUNNER_DNS="$OPTARG";;
+      e) # Entrypoint
+        KRUNVM_RUNNER_ENTRYPOINT="$OPTARG";;
+      m) # Memory in Mb
+        KRUNVM_RUNNER_MEM="$OPTARG";;
+      n) # Name of container/VM
+        KRUNVM_RUNNER_NAME="$OPTARG";;
+      v) # Volumes to mount
+        if [ -z "$KRUNVM_RUNNER_VOLS" ]; then
+          KRUNVM_RUNNER_VOLS="$OPTARG"
+        else
+          KRUNVM_RUNNER_VOLS="$(printf %s\\n%s\\n "$KRUNVM_RUNNER_VOLS" "$OPTARG")"
+        fi;;
+      -) # End of options, everything after is the image and arguments to run
+        break;;
+      ?)
+        error "$_opt is an unrecognised option";;
+    esac
+  done
+  shift $((OPTIND-1))
+  if [ "$#" -lt 1 ]; then
+    error "No image specified"
+  fi
+  [ -z "$KRUNVM_RUNNER_NAME" ] && error "No name specified for microVM"
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      set -- \
+        --runtime "krun" \
+        --rm \
+        --tty \
+        --name "$KRUNVM_RUNNER_NAME" \
+        --cpus "$KRUNVM_RUNNER_CPUS" \
+        --memory "${KRUNVM_RUNNER_MEM}m" \
+        --dns "$KRUNVM_RUNNER_DNS" \
+        --entrypoint "$KRUNVM_RUNNER_ENTRYPOINT" \
+        "$@"
+      if [ -n "$KRUNVM_RUNNER_VOLS" ]; then
+        while IFS= read -r mount || [ -n "$mount" ]; do
+          if [ -n "$mount" ]; then
+            set -- --volume "$mount" "$@"
+          fi
+        done <<EOF
+$(printf %s\\n "$KRUNVM_RUNNER_VOLS")
+EOF
+      fi
+      verbose "Starting container '${KRUNVM_RUNNER_NAME}' with entrypoint $KRUNVM_RUNNER_ENTRYPOINT"
+      podman run "$@"
+      ;;
+    krunvm)
+      _krunvm_create "$1"
+      shift
+
+      verbose "Starting microVM '${KRUNVM_RUNNER_NAME}' with entrypoint $KRUNVM_RUNNER_ENTRYPOINT"
+      optstate=$(set +o)
+      set -m; # Disable job control
+      run_krunvm start "$KRUNVM_RUNNER_NAME" "$RUNNER_ENTRYPOINT" -- "$@" </dev/null &
+      KRUNVM_RUNNER_PID=$!
+      eval "$optstate"; # Restore options
+      verbose "Started microVM '$KRUNVM_RUNNER_NAME' with PID $KRUNVM_RUNNER_PID"
+      wait "$KRUNVM_RUNNER_PID"
+      KRUNVM_RUNNER_PID=
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+
+microvm_wait() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No name specified"
+  fi
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      podman wait "$1";;
+    krunvm)
+      if [ -n "$KRUNVM_RUNNER_PID" ]; then
+        # shellcheck disable=SC2046 # We want to wait for all children
+        waitpid $(ps_tree "$KRUNVM_RUNNER_PID"|tac)
+        KRUNVM_RUNNER_PID=
+      fi
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+
+# NOTE: we won't be using this much, since we terminate through the .trm file in most cases.
+microvm_stop() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No name specified"
+  fi
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      # TODO: Specify howlog to wait between TERM and KILL?
+      podman stop "$1";;
+    krunvm)
+      if [ -n "$KRUNVM_RUNNER_PID" ]; then
+        kill_tree "$KRUNVM_RUNNER_PID"
+        # shellcheck disable=SC2046 # We want to wait for all children
+        microvm_wait "$1"
+      fi
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+microvm_delete() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No name specified"
+  fi
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      verbose "Removing container '$1'"
+      podman rm -f "$1";;
+    krunvm)
+      verbose "Removing microVM '$1'"
+      run_krunvm delete "$1";;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+microvm_pull() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No image name specified"
+  fi
+
+  verbose "Pulling image(s) '$*'"
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      podman pull "$@";;
+    krunvm)
+      buildah pull "$@";;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+
+}
\ No newline at end of file
diff --git a/orchestrator.sh b/orchestrator.sh
index 424b468..25e1ed8 100755
--- a/orchestrator.sh
+++ b/orchestrator.sh
@@ -31,6 +31,8 @@ ORCHESTRATOR_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")
 
 # shellcheck source=lib/common.sh
 . "$ORCHESTRATOR_ROOTDIR/lib/common.sh"
+# shellcheck source=lib/microvm.sh
+. "$ORCHESTRATOR_ROOTDIR/lib/microvm.sh"
 
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
@@ -56,11 +58,14 @@ ORCHESTRATOR_ISOLATION=${ORCHESTRATOR_ISOLATION:-"1"}
 # has been turned on.
 ORCHESTRATOR_SLEEP=${ORCHESTRATOR_SLEEP:-"30"}
 
+# Runtime to use when managing microVMs.
+ORCHESTRATOR_RUNTIME=${ORCHESTRATOR_RUNTIME:-""}
+
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host"
 
 
-while getopts "s:Il:n:p:vh-" opt; do
+while getopts "s:Il:n:p:R:vh-" opt; do
   case "$opt" in
     s) # Number of seconds to sleep between microVM creation at start, when no isolation
       ORCHESTRATOR_SLEEP="$OPTARG";;
@@ -72,6 +77,8 @@ while getopts "s:Il:n:p:vh-" opt; do
       ORCHESTRATOR_RUNNERS="$OPTARG";;
     p) # Prefix to use for the VM name
       ORCHESTRATOR_PREFIX="$OPTARG";;
+    R) # Runtime to use when managing microVMs
+      ORCHESTRATOR_RUNTIME="$OPTARG";;
     v) # Increase verbosity, will otherwise log on errors/warnings only
       ORCHESTRATOR_VERBOSE=$((ORCHESTRATOR_VERBOSE+1));;
     h) # Print help and exit
@@ -99,14 +106,14 @@ cleanup() {
   # shellcheck disable=SC2086 # We want to wait for all pids
   waitpid $ORCHESTRATOR_PIDS
 
-  if run_krunvm list | grep -qE "^${ORCHESTRATOR_PREFIX}-"; then
-    while IFS= read -r vm; do
+  while IFS= read -r vm; do
+    if [ -n "$vm" ]; then
       verbose "Removing microVM $vm"
-      run_krunvm delete "$vm"
-    done <<EOF
-$(run_krunvm list | grep -E "^${ORCHESTRATOR_PREFIX}-")
+      microvm_delete "$vm"
+    fi
+  done <<EOF
+$(microvm_list | grep -E "^${ORCHESTRATOR_PREFIX}-")
 EOF
-  fi
 
   if [ -n "${ORCHESTRATOR_ENVIRONMENT:-}" ]; then
     verbose "Removing isolation environment $ORCHESTRATOR_ENVIRONMENT"
@@ -114,8 +121,8 @@ EOF
   fi
 }
 
-check_command buildah
-check_command krunvm
+# Pass the runtime to the microvm script
+microvm_runtime "$ORCHESTRATOR_RUNTIME"
 
 check_positive_number "$ORCHESTRATOR_RUNNERS" "Number of runners"
 
@@ -134,7 +141,8 @@ trap cleanup EXIT
 RUNNER_PREFIX=$ORCHESTRATOR_PREFIX
 RUNNER_VERBOSE=$ORCHESTRATOR_VERBOSE
 RUNNER_LOG=$ORCHESTRATOR_LOG
-export RUNNER_PREFIX RUNNER_ENVIRONMENT RUNNER_VERBOSE RUNNER_LOG
+RUNNER_RUNTIME=$ORCHESTRATOR_RUNTIME
+export RUNNER_PREFIX RUNNER_ENVIRONMENT RUNNER_VERBOSE RUNNER_LOG RUNNER_RUNTIME
 
 # Create runner loops in the background. One per runner. Each loop will
 # indefinitely create ephemeral runners. Looping is implemented in runner.sh,
diff --git a/runner.sh b/runner.sh
index ce8ef6e..cb1e6ab 100755
--- a/runner.sh
+++ b/runner.sh
@@ -31,6 +31,8 @@ RUNNER_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")")")"
 
 # shellcheck source=lib/common.sh
 . "$RUNNER_ROOTDIR/lib/common.sh"
+# shellcheck source=lib/microvm.sh
+. "$RUNNER_ROOTDIR/lib/microvm.sh"
 
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
@@ -115,11 +117,15 @@ RUNNER_SECRET=${RUNNER_SECRET:-"$(random_string)"}
 # Number of seconds after which to terminate (empty for never, the good default)
 RUNNER_TERMINATE=${RUNNER_TERMINATE:-""}
 
+# Runtime to use when managing microVMs.
+RUNNER_RUNTIME=${RUNNER_RUNTIME:-""}
+
+
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Create runners forever using krunvm"
 
 
-while getopts "c:d:D:g:G:i:l:L:m:M:p:k:r:s:S:T:u:Uvh-" opt; do
+while getopts "c:d:D:g:G:i:l:L:m:M:p:k:r:R:s:S:T:u:Uvh-" opt; do
   case "$opt" in
     c) # Number of CPUs to allocate to the VM
       RUNNER_CPUS="$OPTARG";;
@@ -151,6 +157,8 @@ while getopts "c:d:D:g:G:i:l:L:m:M:p:k:r:s:S:T:u:Uvh-" opt; do
       RUNNER_TERMINATE="$OPTARG";;
     r) # Number of times to repeat the runner loop
       RUNNER_REPEAT="$OPTARG";;
+    R) # Runtime to use when managing microVMs
+      RUNNER_RUNTIME="$OPTARG";;
     s) # Scope of the runner, one of repo, org or enterprise
       RUNNER_SCOPE="$OPTARG";;
     S) # Secret to be used to request for loop end
@@ -186,9 +194,7 @@ fi
 # synchronisation files.
 RUNNER_VM_ENVDIR="_environment"
 
-if [ -z "$RUNNER_PAT" ]; then
-  error "You need to specify a PAT to acquire the runner token with"
-fi
+[ -z "$RUNNER_PAT" ] && error "You need to specify a PAT to acquire the runner token with"
 check_positive_number "$RUNNER_CPUS" "Number of vCPUs"
 check_positive_number "$RUNNER_MEMORY" "Memory (in MB)"
 
@@ -201,38 +207,19 @@ else
   RUNNER_ENTRYPOINT=${RUNNER_DIR%/}/runner/entrypoint.sh
 fi
 
-# Create the VM used for orchestration. Add --volume options for all necessary
-# mappings, i.e. inheritance of "live" code, environment isolation and all
-# requested mount points.
-vm_create() {
-  verbose "Creating microVM '${RUNNER_PREFIX}-$1', $RUNNER_CPUS vCPUs, ${RUNNER_MEMORY}M memory"
-  # Note: reset arguments!
-  set -- \
-    --cpus "$RUNNER_CPUS" \
-    --mem "$RUNNER_MEMORY" \
-    --dns "$RUNNER_DNS" \
-    --name "${RUNNER_PREFIX}-$1"
-  if [ -n "${RUNNER_DIR:-}" ]; then
-    set -- "$@" --volume "${RUNNER_ROOTDIR}:${RUNNER_DIR}"
-  fi
-  if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then
-    set -- "$@" --volume "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}"
-  fi
-  if [ -n "$RUNNER_MOUNT" ]; then
-    while IFS= read -r mount || [ -n "$mount" ]; do
-      if [ -n "$mount" ]; then
-        set -- "$@" --volume "$mount"
-      fi
-    done <<EOF
-$(printf %s\\n "$RUNNER_MOUNT")
-EOF
-  fi
-  run_krunvm create "$RUNNER_IMAGE" "$@"
-}
+# Pass the runtime to the microvm script
+microvm_runtime "$RUNNER_RUNTIME"
 
 
-vm_start() {
-  _id=$1
+# Call the microvm_run function with the necessary arguments to start a runner
+vm_run() {
+  _id=$1;  # Remember the ID for later
+
+  # First prepare the arguments that will be sent to the entrypoint.sh script.
+  # When an isolation environment can be constructed, create such a file and
+  # pass all relevant content through the file -- and point to the environment
+  # file that was created. Whenever this isn't possible, pass all relevant
+  # options one-by-one.
   if [ -n "$RUNNER_ENVIRONMENT" ]; then
     # Create an env file with most of the RUNNER_ variables. This works because
     # the `runner.sh` script that will be called uses the same set of variables.
@@ -268,19 +255,38 @@ EOF
       set -- -v "$@"
     done
   fi
-  verbose "Starting microVM '${RUNNER_PREFIX}-$_id' with entrypoint $RUNNER_ENTRYPOINT"
-  optstate=$(set +o)
-  set -m; # Disable job control
-  run_krunvm start "${RUNNER_PREFIX}-$_id" "$RUNNER_ENTRYPOINT" -- "$@" </dev/null &
-  RUNNER_PID=$!
-  eval "$optstate"; # Restore options
-  verbose "Started microVM '${RUNNER_PREFIX}-$_id' with PID $RUNNER_PID"
-  if [ -n "$RUNNER_TERMINATE" ]; then
-    verbose "Terminating runner in $RUNNER_TERMINATE seconds"
-    sleep "$RUNNER_TERMINATE" && cleanup &
+
+  # Add image to create from
+  set -- -- "$RUNNER_IMAGE" "$@"
+
+  # Create the VM used for orchestration. Add -v (volumes) options for all
+  # necessary mappings, i.e. inheritance of "live" code, environment isolation
+  # and all requested mount points.
+  set -- \
+    -c "$RUNNER_CPUS" \
+    -m "$RUNNER_MEMORY" \
+    -d "$RUNNER_DNS" \
+    -n "${RUNNER_PREFIX}-$_id" \
+    -e "$RUNNER_ENTRYPOINT" \
+    "$@"
+  if [ -n "${RUNNER_DIR:-}" ]; then
+    set -- -v "${RUNNER_ROOTDIR}:${RUNNER_DIR}" "$@"
   fi
-  wait "$RUNNER_PID"
-  RUNNER_PID=
+  if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then
+    set -- -v "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}" "$@"
+  fi
+  if [ -n "$RUNNER_MOUNT" ]; then
+    while IFS= read -r mount || [ -n "$mount" ]; do
+      if [ -n "$mount" ]; then
+        set -- -v "$mount" "$@"
+      fi
+    done <<EOF
+$(printf %s\\n "$RUNNER_MOUNT")
+EOF
+  fi
+
+  trace "Running microVM with: $*"
+  microvm_run "$@"
 }
 
 
@@ -288,48 +294,35 @@ vm_delete() {
   # This is just a safety measure, the runner script should already have deleted
   # the environment
   if [ -n "$RUNNER_ENVIRONMENT" ] && [ -f "${RUNNER_ENVIRONMENT}/${1}.env" ]; then
-    warning "Removing isolation environment ${RUNNER_ENVIRONMENT}/${1}.env"
+    warn "Removing isolation environment ${RUNNER_ENVIRONMENT}/${1}.env"
     rm -f "${RUNNER_ENVIRONMENT}/${1}.env"
   fi
-  if run_krunvm list | grep -qE "^${RUNNER_PREFIX}-$1"; then
+  if microvm_list | grep -qE "^${RUNNER_PREFIX}-$1"; then
     verbose "Removing microVM '${RUNNER_PREFIX}-$1'"
-    run_krunvm delete "${RUNNER_PREFIX}-$1"
+    microvm_delete "${RUNNER_PREFIX}-$1"
   fi
 }
 
 vm_terminate() {
-  if [ -n "$RUNNER_ENVIRONMENT" ]; then
-    if [ -f "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.tkn" ]; then
-      if [ -n "${RUNNER_SECRET:-}" ]; then
-        verbose "Requesting termination via ${RUNNER_ENVIRONMENT}/${RUNNER_ID}.trm"
-        printf %s\\n "$RUNNER_SECRET" > "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.trm"
-      elif [ -n "$RUNNER_PID" ]; then
-        kill_tree "$RUNNER_PID"
-      fi
-      if [ "$RUNNER_PID" ]; then
-        # shellcheck disable=SC2046 # We want to wait for all children
-        waitpid $(ps_tree "$RUNNER_PID"|tac)
-      else
-        warning "No PID to wait for"
-      fi
-    elif [ -n "$RUNNER_PID" ]; then
-      kill_tree "$RUNNER_PID"
-      # shellcheck disable=SC2046 # We want to wait for all children
-      waitpid $(ps_tree "$RUNNER_PID"|tac)
-    fi
-  elif [ -n "$RUNNER_PID" ]; then
-    kill_tree "$RUNNER_PID"
-    # shellcheck disable=SC2046 # We want to wait for all children
-    waitpid $(ps_tree "$RUNNER_PID"|tac)
+  # Request for termination through .trm file, whenever possible. Otherwise,
+  # just stop the VM.
+  if [ -n "$RUNNER_ENVIRONMENT" ] \
+      && [ -f "${RUNNER_ENVIRONMENT}/${1}.tkn" ] \
+      && [ -n "${RUNNER_SECRET:-}" ]; then
+    verbose "Requesting termination via ${RUNNER_ENVIRONMENT}/${1}.trm"
+    printf %s\\n "$RUNNER_SECRET" > "${RUNNER_ENVIRONMENT}/${1}.trm"
+    microvm_wait "${RUNNER_PREFIX}-$1"
+  else
+    microvm_stop "${RUNNER_PREFIX}-$1"
   fi
 }
 
+
 cleanup() {
   trap '' EXIT
-  if [ -n "${RUNNER_PID:-}" ]; then
-    vm_terminate
-  fi
+
   if [ -n "${RUNNER_ID:-}" ]; then
+    vm_terminate "$RUNNER_ID"
     vm_delete "$RUNNER_ID"
   fi
 }
@@ -339,9 +332,19 @@ trap cleanup EXIT
 
 iteration=0
 while true; do
+  # Prefetch, since this might take time and we want to be ready to count away
+  # download time from the termination setting.
+  microvm_pull "$RUNNER_IMAGE"
+
+  # Terminate in xx seconds. This is mostly used for demo purposes, but might
+  # help keeping the machines "warm" and actualised (as per the pull above).
+  if [ -n "$RUNNER_TERMINATE" ]; then
+    verbose "Terminating runner in $RUNNER_TERMINATE seconds"
+    sleep "$RUNNER_TERMINATE" && cleanup &
+  fi
+
   RUNNER_ID="${loop}-$(random_string)"
-  vm_create "${RUNNER_ID}"
-  vm_start "${RUNNER_ID}"
+  vm_run "${RUNNER_ID}"
   vm_delete "${RUNNER_ID}"
   RUNNER_ID=
 

From 6b8ecce2657d2edf19ae52f3544f7835ae7afb76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 5 Apr 2024 14:38:48 +0200
Subject: [PATCH 03/13] Add support for multiple krunvm VMs

Add support for multiple krunvm-based VMs in the API. This uses a
temporary directory that will store the mappings between the name of the
VM and the PID of the underlying (top) process. The directory should be
removed through microvm_cleanup upon process exit.
---
 lib/microvm.sh  | 18 ++++++++++++++++--
 orchestrator.sh |  2 ++
 runner.sh       |  2 ++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index 5262210..34a9470 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -7,6 +7,9 @@
 # automatically selected.
 : "${KRUNVM_RUNNER_RUNTIME:=""}"
 
+# Directory used for VM->PID mapping when using krunvm
+: "${KRUNVM_RUNNER_STORAGE:=""}"
+
 # Run krunvm with the provided arguments, behind a buildah unshare.
 run_krunvm() {
   debug "Running krunvm $*"
@@ -50,6 +53,9 @@ microvm_runtime() {
     krunvm)
       check_command krunvm
       check_command buildah
+      if [ -z "$KRUNVM_RUNNER_STORAGE" ]; then
+        KRUNVM_RUNNER_STORAGE="$(mktemp -d)"
+      fi
       ;;
     *)
       error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
@@ -168,6 +174,7 @@ EOF
       run_krunvm start "$KRUNVM_RUNNER_NAME" "$RUNNER_ENTRYPOINT" -- "$@" </dev/null &
       KRUNVM_RUNNER_PID=$!
       eval "$optstate"; # Restore options
+      printf %d\\n "$KRUNVM_RUNNER_PID" > "${KRUNVM_RUNNER_STORAGE}/${KRUNVM_RUNNER_NAME}.pid"
       verbose "Started microVM '$KRUNVM_RUNNER_NAME' with PID $KRUNVM_RUNNER_PID"
       wait "$KRUNVM_RUNNER_PID"
       KRUNVM_RUNNER_PID=
@@ -190,6 +197,7 @@ microvm_wait() {
     podman*)
       podman wait "$1";;
     krunvm)
+      KRUNVM_RUNNER_PID=$(cat "${KRUNVM_RUNNER_STORAGE}/$1.pid")
       if [ -n "$KRUNVM_RUNNER_PID" ]; then
         # shellcheck disable=SC2046 # We want to wait for all children
         waitpid $(ps_tree "$KRUNVM_RUNNER_PID"|tac)
@@ -216,10 +224,12 @@ microvm_stop() {
       # TODO: Specify howlog to wait between TERM and KILL?
       podman stop "$1";;
     krunvm)
+      KRUNVM_RUNNER_PID=$(cat "${KRUNVM_RUNNER_STORAGE}/$1.pid")
       if [ -n "$KRUNVM_RUNNER_PID" ]; then
         kill_tree "$KRUNVM_RUNNER_PID"
         # shellcheck disable=SC2046 # We want to wait for all children
         microvm_wait "$1"
+        rm -f "${KRUNVM_RUNNER_STORAGE}/$1.pid" || true
       fi
       ;;
     *)
@@ -241,7 +251,8 @@ microvm_delete() {
       podman rm -f "$1";;
     krunvm)
       verbose "Removing microVM '$1'"
-      run_krunvm delete "$1";;
+      run_krunvm delete "$1"
+      ;;
     *)
       error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
       ;;
@@ -265,5 +276,8 @@ microvm_pull() {
       error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
       ;;
   esac
+}
 
-}
\ No newline at end of file
+microvm_cleanup() {
+  [ -n "$KRUNVM_RUNNER_STORAGE" ] && rm -rf "$KRUNVM_RUNNER_STORAGE"
+}
diff --git a/orchestrator.sh b/orchestrator.sh
index 25e1ed8..a5ffa23 100755
--- a/orchestrator.sh
+++ b/orchestrator.sh
@@ -119,6 +119,8 @@ EOF
     verbose "Removing isolation environment $ORCHESTRATOR_ENVIRONMENT"
     rm -rf "$ORCHESTRATOR_ENVIRONMENT"
   fi
+
+  microvm_cleanup
 }
 
 # Pass the runtime to the microvm script
diff --git a/runner.sh b/runner.sh
index cb1e6ab..7f8e282 100755
--- a/runner.sh
+++ b/runner.sh
@@ -325,6 +325,8 @@ cleanup() {
     vm_terminate "$RUNNER_ID"
     vm_delete "$RUNNER_ID"
   fi
+
+  microvm_cleanup
 }
 
 trap cleanup EXIT

From 7ce876f784b552e4970ef91b542f09e6ca9cd12c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 5 Apr 2024 23:27:47 +0200
Subject: [PATCH 04/13] Enforce mount options for podman

---
 lib/microvm.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index 34a9470..30f23fe 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -155,13 +155,14 @@ microvm_run() {
       if [ -n "$KRUNVM_RUNNER_VOLS" ]; then
         while IFS= read -r mount || [ -n "$mount" ]; do
           if [ -n "$mount" ]; then
-            set -- --volume "$mount" "$@"
+            set -- --volume "${mount}:Z,rw" "$@"
           fi
         done <<EOF
 $(printf %s\\n "$KRUNVM_RUNNER_VOLS")
 EOF
       fi
       verbose "Starting container '${KRUNVM_RUNNER_NAME}' with entrypoint $KRUNVM_RUNNER_ENTRYPOINT"
+      trace "Running: podman run $*"
       podman run "$@"
       ;;
     krunvm)

From 5c6c4f9d26e7eb44cafaa5181622e1424f0d5844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Sat, 6 Apr 2024 19:04:51 +0200
Subject: [PATCH 05/13] Add support for other podman runtimes

While the podman runtime is still enforced, the script now supports
several other runtimes. The KRUNVM_RUNNER_RUNTIME variable should
contain the word podman, followed by the plus sign, followed by the
runtime to use, e.g. podman+krun will pass the value of krun to the
--runtime option of podman create/run.
---
 lib/microvm.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index 30f23fe..15b8b77 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -16,6 +16,12 @@ run_krunvm() {
   buildah unshare krunvm "$@"
 }
 
+_podman_runtime() {
+  _runtime=${KRUNVM_RUNNER_RUNTIME#podman+}
+  [ -z "$_runtime" ] && _runtime="krun"
+  printf %s\\n "$_runtime"
+}
+
 
 # Automatically select a microVM runtime based on the available commands. Set
 # the KRUNVM_RUNNER_RUNTIME variable.
@@ -48,7 +54,7 @@ microvm_runtime() {
   case "$KRUNVM_RUNNER_RUNTIME" in
     podman*)
       check_command podman
-      check_command krun
+      check_command "$(_podman_runtime)"
       ;;
     krunvm)
       check_command krunvm
@@ -143,7 +149,7 @@ microvm_run() {
   case "$KRUNVM_RUNNER_RUNTIME" in
     podman*)
       set -- \
-        --runtime "krun" \
+        --runtime "$(_podman_runtime)" \
         --rm \
         --tty \
         --name "$KRUNVM_RUNNER_NAME" \

From f267bf374a940ef74b5a33d40de44d1c1e8ef4cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Sun, 7 Apr 2024 20:36:17 +0200
Subject: [PATCH 06/13] Add mount option only if not present

---
 lib/microvm.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index 15b8b77..4f32c72 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -161,7 +161,11 @@ microvm_run() {
       if [ -n "$KRUNVM_RUNNER_VOLS" ]; then
         while IFS= read -r mount || [ -n "$mount" ]; do
           if [ -n "$mount" ]; then
-            set -- --volume "${mount}:Z,rw" "$@"
+            if [ -z "$(printf %s\\n "$mount"|cut -d ':' -f 3)" ]; then
+              set -- --volume "${mount}:Z,rw" "$@"
+            else
+              set -- --volume "${mount}" "$@"
+            fi
           fi
         done <<EOF
 $(printf %s\\n "$KRUNVM_RUNNER_VOLS")

From 2fabb40e40e65a706c04f5e1ef56370050c2b3d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 27 Sep 2024 16:21:40 +0200
Subject: [PATCH 07/13] Add documentation

---
 CONTRIBUTING.md |  2 +-
 README.md       | 24 +++++++++++++++---------
 orchestrator.sh |  4 ++--
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 88fc951..73826fc 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -71,7 +71,7 @@ driven by the `RUNNER_ENTRYPOINT` variable in [`runner.sh`](./runner.sh).
 ## Cleanup
 
 During development, many images might be created. To clean them away, you can
-run the following:
+run the following command when using the `krunvm` runtime, or `podman image rm`:
 
 ```bash
 buildah rmi $(buildah images --format '{{.ID}}')
diff --git a/README.md b/README.md
index 08e949a..2002b0a 100644
--- a/README.md
+++ b/README.md
@@ -1,20 +1,24 @@
 # krunvm-based GitHub Runner(s)
 
 This project creates [self-hosted][self] (ephemeral) GitHub [runners] based on
-[krunvm]. [krunvm] creates [microVM]s, so the project enables fully isolated
+[libkrun]. [libkrun] creates [microVM]s, so the project enables fully isolated
 [runners] inside your infrastruture. MicroVMs boot fast, providing an experience
-close to running containers. [krunvm] creates and starts VMs based on the
+close to running containers. [libkrun] creates and starts VMs based on the
 multi-platform OCI images created for this project -- [ubuntu] (default) or
-[fedora].
+[fedora]. The project will create [microVM]s using either [krunvm] or
+[krun][crun] and [podman].
 
 ![Demo](./demo/demo.gif)
 
   [self]: https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners
   [runners]: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
+  [libkrun]: https://github.com/containers/libkrun
   [krunvm]: https://github.com/containers/krunvm
   [microVM]: https://github.com/infracloudio/awesome-microvm
   [ubuntu]: https://github.com/efrecon/gh-runner-krunvm/pkgs/container/runner-krunvm-ubuntu
   [fedora]: https://github.com/efrecon/gh-runner-krunvm/pkgs/container/runner-krunvm-fedora
+  [crun]: https://github.com/containers/crun
+  [podman]: https://github.com/containers/podman
 
 ## Example
 
@@ -68,6 +72,7 @@ starting with `RUNNER_` will affect the behaviour of each [runner] (loop).
   critical software tools.
 + Good compatibility with the regular GitHub [runners]: same user ID, member of
   the `docker` group, password-less `sudo`, etc.
++ Supports both [krunvm] or the [krun][crun] runtime under [podman].
 + In theory, the main [ubuntu] and [fedora] images should be able to be used in
   more traditional container-based solutions -- perhaps [sysbox]? Reports and/or
   changes are welcome.
@@ -89,10 +94,11 @@ installed on the host. Installation is easiest on Fedora
 
 + `curl`
 + `jq`
-+ `buildah`
-+ `krunvm` (and its [requirements])
++ A compatible runtime, i.e. either:
+  + `krun` and [`podman`][podman].
+  + `krunvm`, its [requirements] and `buildah`
 
-Note: You do not need `podman`.
+Note: When opting for `krunvm`, you do not need `podman`.
 
   [built]: ./.github/workflows/ci.yml
   [requirements]: https://github.com/containers/krunvm#installation
@@ -129,9 +135,9 @@ permissions.
 The [orchestrator] creates as many loops of ephemeral runners as requested.
 These loops are implemented as part of the [runner.sh][runner] script: the
 script will create a microVM based on the default image (see below), memory and
-vCPU requirement. It will then start that microVM using `krunvm` and that will
-start an (ephemeral) GitHub [runner][self]. As soon as a job has been executed
-on that runner, the microVM will end and a new will be created.
+vCPU requirement. It will then start that microVM using `krunvm` or `podman` and
+the VM will start an (ephemeral) GitHub [runner][self]. As soon as a job has
+been executed on that runner, the microVM will end and a new will be created.
 
 The OCI image is built in two parts:
 
diff --git a/orchestrator.sh b/orchestrator.sh
index a5ffa23..e2029c4 100755
--- a/orchestrator.sh
+++ b/orchestrator.sh
@@ -62,7 +62,7 @@ ORCHESTRATOR_SLEEP=${ORCHESTRATOR_SLEEP:-"30"}
 ORCHESTRATOR_RUNTIME=${ORCHESTRATOR_RUNTIME:-""}
 
 # shellcheck disable=SC2034 # Used in sourced scripts
-KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host"
+KRUNVM_RUNNER_DESCR="Run libkrun-based GitHub runners on a single host"
 
 
 while getopts "s:Il:n:p:R:vh-" opt; do
@@ -77,7 +77,7 @@ while getopts "s:Il:n:p:R:vh-" opt; do
       ORCHESTRATOR_RUNNERS="$OPTARG";;
     p) # Prefix to use for the VM name
       ORCHESTRATOR_PREFIX="$OPTARG";;
-    R) # Runtime to use when managing microVMs
+    R) # Runtime to use when managing microVMs, podman+krun or krunvm. Empty==first available
       ORCHESTRATOR_RUNTIME="$OPTARG";;
     v) # Increase verbosity, will otherwise log on errors/warnings only
       ORCHESTRATOR_VERBOSE=$((ORCHESTRATOR_VERBOSE+1));;

From 01cf390d84e7b742d1061f34f444e91988954eef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 27 Sep 2024 16:28:55 +0200
Subject: [PATCH 08/13] Add ref to installation on old Fedoras

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2002b0a..cad2900 100644
--- a/README.md
+++ b/README.md
@@ -90,7 +90,8 @@ UNIX binary utilities. PRs are welcome to make the project work on MacOS, if it
 does not already.
 
 Apart from the standard UNIX binary utilities, you will need the following
-installed on the host. Installation is easiest on Fedora
+installed on the host. Installation is easiest on Fedora (see original [issue]
+for installation on older versions).
 
 + `curl`
 + `jq`
@@ -102,6 +103,7 @@ Note: When opting for `krunvm`, you do not need `podman`.
 
   [built]: ./.github/workflows/ci.yml
   [requirements]: https://github.com/containers/krunvm#installation
+  [issue]: https://github.com/efrecon/gh-runner-krunvm/issues/22
 
 ## GitHub Token
 

From 0fa9ec6d8540a3736b4a24fedec47dfd4efc8a1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 27 Sep 2024 16:52:59 +0200
Subject: [PATCH 09/13] Enhance cleanup section

---
 CONTRIBUTING.md | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 73826fc..3937c35 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -71,8 +71,21 @@ driven by the `RUNNER_ENTRYPOINT` variable in [`runner.sh`](./runner.sh).
 ## Cleanup
 
 During development, many images might be created. To clean them away, you can
-run the following command when using the `krunvm` runtime, or `podman image rm`:
+run one of the following commands.
+
+When using the `krunvm` runtime:
 
 ```bash
 buildah rmi $(buildah images --format '{{.ID}}')
 ```
+
+When using `podman+krun`:
+
+```bash
+podman image rm $(podman images -q)
+```
+
+> [!WARNING]
+> These commands will remove all unused images on your system. Make sure you
+> don't need any of these images for other projects before running the cleanup.
+> You may need to rebuild images for this project after cleanup.

From 31f2b8bafc0464eb8e99beeff3f7dcbe5f2e49dd Mon Sep 17 00:00:00 2001
From: Emmanuel Frecon <9443924+efrecon@users.noreply.github.com>
Date: Fri, 27 Sep 2024 16:56:30 +0200
Subject: [PATCH 10/13] Check command is passed

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 lib/common.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/common.sh b/lib/common.sh
index c382d9f..110c7b6 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -67,6 +67,9 @@ check_command() {
     esac
   done
   shift $((OPTIND-1))
+  if [ -z "$1" ]; then
+    error "No command specified for checking"
+  fi
   trace "Checking $1 is an accessible command"
   if ! command -v "$1" >/dev/null 2>&1; then
     if is_true "$_hard"; then

From 813e06d67263bbe8dbd29851fee0228cde19c53e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 27 Sep 2024 17:02:49 +0200
Subject: [PATCH 11/13] Fix proper variable

---
 lib/microvm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index 4f32c72..9a564aa 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -182,7 +182,7 @@ EOF
       verbose "Starting microVM '${KRUNVM_RUNNER_NAME}' with entrypoint $KRUNVM_RUNNER_ENTRYPOINT"
       optstate=$(set +o)
       set -m; # Disable job control
-      run_krunvm start "$KRUNVM_RUNNER_NAME" "$RUNNER_ENTRYPOINT" -- "$@" </dev/null &
+      run_krunvm start "$KRUNVM_RUNNER_NAME" "$KRUNVM_RUNNER_ENTRYPOINT" -- "$@" </dev/null &
       KRUNVM_RUNNER_PID=$!
       eval "$optstate"; # Restore options
       printf %d\\n "$KRUNVM_RUNNER_PID" > "${KRUNVM_RUNNER_STORAGE}/${KRUNVM_RUNNER_NAME}.pid"

From 0ef3dfcec35c3957b895c292196f349f06662f1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 27 Sep 2024 17:04:18 +0200
Subject: [PATCH 12/13] Add error when no runtime found

---
 lib/microvm.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index 9a564aa..d8aa634 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -32,6 +32,8 @@ _microvm_runtime_auto() {
   elif check_command -s -- krunvm; then
     check_command buildah
     KRUNVM_RUNNER_RUNTIME="krunvm"
+  else
+    error "No suitable runtime found. Please install 'krun' or 'krunvm'"
   fi
   info "Automatically selected $KRUNVM_RUNNER_RUNTIME to handle microVMs"
 }

From abb127a37f75fd4ced4c3c01e2f2dedd19815ac5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 27 Sep 2024 22:22:21 +0200
Subject: [PATCH 13/13] Fix spelling in comment

---
 lib/microvm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/microvm.sh b/lib/microvm.sh
index d8aa634..4700de3 100644
--- a/lib/microvm.sh
+++ b/lib/microvm.sh
@@ -234,7 +234,7 @@ microvm_stop() {
 
   case "$KRUNVM_RUNNER_RUNTIME" in
     podman*)
-      # TODO: Specify howlog to wait between TERM and KILL?
+      # TODO: Specify how long to wait between TERM and KILL?
       podman stop "$1";;
     krunvm)
       KRUNVM_RUNNER_PID=$(cat "${KRUNVM_RUNNER_STORAGE}/$1.pid")