From 970043c1f05d9cab84a8e017f57955a7041d0907 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Emmanuel=20Fr=C3=A9con?= <efrecon@gmail.com>
Date: Fri, 5 Apr 2024 14:16:09 +0200
Subject: [PATCH] Add generic microvm API lib and use it

Add a generic API to create microVMs in an "agnostic" way, and migrate
to the new API in the orchestrator.sh and runner.sh scripts. While the
API has code for podman, this update only focuses on migrating the
existing krunvm-based code to the new API.

The new API is implemented in lib/microvm.sh, and provides a set of
docker/podman lookalike commands to manage microVMs. The function
microvm_runtime can be used to pinpoint or auto-detect the runtime to
use depending on commands available at the host. The API is **not** able
to handle several krunvm-based microVMs at the same time.
---
 lib/common.sh   |  28 +++--
 lib/microvm.sh  | 269 ++++++++++++++++++++++++++++++++++++++++++++++++
 orchestrator.sh |  28 +++--
 runner.sh       | 157 ++++++++++++++--------------
 4 files changed, 389 insertions(+), 93 deletions(-)
 create mode 100644 lib/microvm.sh

diff --git a/lib/common.sh b/lib/common.sh
index 9fbab23..c382d9f 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -51,9 +51,30 @@ usage() {
 }
 
 check_command() {
+  OPTIND=1
+  _hard=1
+  _warn=0
+  while getopts "sw-" _opt; do
+    case "$_opt" in
+      s) # Soft check, return an error code instead of exiting
+        _hard=0;;
+      w) # Print a warning when soft checking
+        _warn=1;;
+      -) # End of options, everything after is the command
+        break;;
+      ?)
+        error "$_opt is an unrecognised option";;
+    esac
+  done
+  shift $((OPTIND-1))
   trace "Checking $1 is an accessible command"
   if ! command -v "$1" >/dev/null 2>&1; then
-    error "Command not found: $1"
+    if is_true "$_hard"; then
+      error "Command not found: $1"
+    elif is_true "$_warn"; then
+      warn "Command not found: $1"
+    fi
+    return 1
   fi
 }
 
@@ -70,11 +91,6 @@ get_env() (
   fi
 )
 
-run_krunvm() {
-  debug "Running krunvm $*"
-  buildah unshare krunvm "$@"
-}
-
 tac() {
   awk '{ buffer[NR] = $0; } END { for(i=NR; i>0; i--) { print buffer[i] } }'
 }
diff --git a/lib/microvm.sh b/lib/microvm.sh
new file mode 100644
index 0000000..5262210
--- /dev/null
+++ b/lib/microvm.sh
@@ -0,0 +1,269 @@
+#!/bin/sh
+
+# This implements an API to manage microVMs using krunvm and podman.
+
+
+# Runtime to use for microVMs: podman+krun, krunvm. When empty, it will be
+# automatically selected.
+: "${KRUNVM_RUNNER_RUNTIME:=""}"
+
+# Run krunvm with the provided arguments, behind a buildah unshare.
+run_krunvm() {
+  debug "Running krunvm $*"
+  buildah unshare krunvm "$@"
+}
+
+
+# Automatically select a microVM runtime based on the available commands. Set
+# the KRUNVM_RUNNER_RUNTIME variable.
+_microvm_runtime_auto() {
+  if check_command -s -- krun; then
+    check_command podman
+    KRUNVM_RUNNER_RUNTIME="podman+krun"
+  elif check_command -s -- krunvm; then
+    check_command buildah
+    KRUNVM_RUNNER_RUNTIME="krunvm"
+  fi
+  info "Automatically selected $KRUNVM_RUNNER_RUNTIME to handle microVMs"
+}
+
+
+# Set the microVM runtime to use. When no argument is provided, it will try to
+# automatically detect it based on the available commands.
+# shellcheck disable=SC2120
+microvm_runtime() {
+  # Pick runtime provided as an argument, when available.
+  [ "$#" -gt 0 ] && KRUNVM_RUNNER_RUNTIME="$1"
+
+  # When no runtime is provided, try to auto-detect it.
+  [ -z "${KRUNVM_RUNNER_RUNTIME:-""}" ] && _microvm_runtime_auto
+
+  # Enforce podman+krun as soon as anything starting podman is provided.
+  [ "${KRUNVM_RUNNER_RUNTIME#podman}" != "$KRUNVM_RUNNER_RUNTIME" ] && KRUNVM_RUNNER_RUNTIME="podman+krun"
+
+  # Check if the runtime is valid.
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      check_command podman
+      check_command krun
+      ;;
+    krunvm)
+      check_command krunvm
+      check_command buildah
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+
+# List all microVMs.
+microvm_list() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      podman ps -a --format "{{.Names}}"
+      ;;
+    krunvm)
+      run_krunvm list
+      ;;
+  esac
+}
+
+
+_krunvm_create() {
+  KRUNVM_RUNNER_IMAGE=$1
+  verbose "Creating microVM '${KRUNVM_RUNNER_NAME}', $KRUNVM_RUNNER_CPUS vCPUs, ${KRUNVM_RUNNER_MEM}M memory"
+  # Note: reset arguments!
+  set -- \
+    --cpus "$KRUNVM_RUNNER_CPUS" \
+    --mem "$KRUNVM_RUNNER_MEM" \
+    --dns "$KRUNVM_RUNNER_DNS" \
+    --name "$KRUNVM_RUNNER_NAME"
+  if [ -n "$KRUNVM_RUNNER_VOLS" ]; then
+    while IFS= read -r mount || [ -n "$mount" ]; do
+      if [ -n "$mount" ]; then
+        set -- "$@" --volume "$mount"
+      fi
+    done <<EOF
+$(printf %s\\n "$KRUNVM_RUNNER_VOLS")
+EOF
+  fi
+  run_krunvm create "$KRUNVM_RUNNER_IMAGE" "$@"
+}
+
+
+microvm_run() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  OPTIND=1
+  KRUNVM_RUNNER_CPUS=0
+  KRUNVM_RUNNER_MEM=0
+  KRUNVM_RUNNER_DNS="1.1.1.1"
+  KRUNVM_RUNNER_NAME=""
+  KRUNVM_RUNNER_VOLS=""
+  KRUNVM_RUNNER_ENTRYPOINT=""
+  while getopts "c:d:e:m:n:v:-" _opt; do
+    case "$_opt" in
+      c) # Number of CPUs
+        KRUNVM_RUNNER_CPUS="$OPTARG";;
+      d) # DNS Server
+        KRUNVM_RUNNER_DNS="$OPTARG";;
+      e) # Entrypoint
+        KRUNVM_RUNNER_ENTRYPOINT="$OPTARG";;
+      m) # Memory in Mb
+        KRUNVM_RUNNER_MEM="$OPTARG";;
+      n) # Name of container/VM
+        KRUNVM_RUNNER_NAME="$OPTARG";;
+      v) # Volumes to mount
+        if [ -z "$KRUNVM_RUNNER_VOLS" ]; then
+          KRUNVM_RUNNER_VOLS="$OPTARG"
+        else
+          KRUNVM_RUNNER_VOLS="$(printf %s\\n%s\\n "$KRUNVM_RUNNER_VOLS" "$OPTARG")"
+        fi;;
+      -) # End of options, everything after is the image and arguments to run
+        break;;
+      ?)
+        error "$_opt is an unrecognised option";;
+    esac
+  done
+  shift $((OPTIND-1))
+  if [ "$#" -lt 1 ]; then
+    error "No image specified"
+  fi
+  [ -z "$KRUNVM_RUNNER_NAME" ] && error "No name specified for microVM"
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      set -- \
+        --runtime "krun" \
+        --rm \
+        --tty \
+        --name "$KRUNVM_RUNNER_NAME" \
+        --cpus "$KRUNVM_RUNNER_CPUS" \
+        --memory "${KRUNVM_RUNNER_MEM}m" \
+        --dns "$KRUNVM_RUNNER_DNS" \
+        --entrypoint "$KRUNVM_RUNNER_ENTRYPOINT" \
+        "$@"
+      if [ -n "$KRUNVM_RUNNER_VOLS" ]; then
+        while IFS= read -r mount || [ -n "$mount" ]; do
+          if [ -n "$mount" ]; then
+            set -- --volume "$mount" "$@"
+          fi
+        done <<EOF
+$(printf %s\\n "$KRUNVM_RUNNER_VOLS")
+EOF
+      fi
+      verbose "Starting container '${KRUNVM_RUNNER_NAME}' with entrypoint $KRUNVM_RUNNER_ENTRYPOINT"
+      podman run "$@"
+      ;;
+    krunvm)
+      _krunvm_create "$1"
+      shift
+
+      verbose "Starting microVM '${KRUNVM_RUNNER_NAME}' with entrypoint $KRUNVM_RUNNER_ENTRYPOINT"
+      optstate=$(set +o)
+      set -m; # Disable job control
+      run_krunvm start "$KRUNVM_RUNNER_NAME" "$RUNNER_ENTRYPOINT" -- "$@" </dev/null &
+      KRUNVM_RUNNER_PID=$!
+      eval "$optstate"; # Restore options
+      verbose "Started microVM '$KRUNVM_RUNNER_NAME' with PID $KRUNVM_RUNNER_PID"
+      wait "$KRUNVM_RUNNER_PID"
+      KRUNVM_RUNNER_PID=
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+
+microvm_wait() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No name specified"
+  fi
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      podman wait "$1";;
+    krunvm)
+      if [ -n "$KRUNVM_RUNNER_PID" ]; then
+        # shellcheck disable=SC2046 # We want to wait for all children
+        waitpid $(ps_tree "$KRUNVM_RUNNER_PID"|tac)
+        KRUNVM_RUNNER_PID=
+      fi
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+
+# NOTE: we won't be using this much, since we terminate through the .trm file in most cases.
+microvm_stop() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No name specified"
+  fi
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      # TODO: Specify howlog to wait between TERM and KILL?
+      podman stop "$1";;
+    krunvm)
+      if [ -n "$KRUNVM_RUNNER_PID" ]; then
+        kill_tree "$KRUNVM_RUNNER_PID"
+        # shellcheck disable=SC2046 # We want to wait for all children
+        microvm_wait "$1"
+      fi
+      ;;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+microvm_delete() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No name specified"
+  fi
+
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      verbose "Removing container '$1'"
+      podman rm -f "$1";;
+    krunvm)
+      verbose "Removing microVM '$1'"
+      run_krunvm delete "$1";;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+}
+
+microvm_pull() {
+  [ -z "$KRUNVM_RUNNER_RUNTIME" ] && microvm_runtime
+
+  if [ "$#" -lt 1 ]; then
+    error "No image name specified"
+  fi
+
+  verbose "Pulling image(s) '$*'"
+  case "$KRUNVM_RUNNER_RUNTIME" in
+    podman*)
+      podman pull "$@";;
+    krunvm)
+      buildah pull "$@";;
+    *)
+      error "Unknown microVM runtime: $KRUNVM_RUNNER_RUNTIME"
+      ;;
+  esac
+
+}
\ No newline at end of file
diff --git a/orchestrator.sh b/orchestrator.sh
index 424b468..25e1ed8 100755
--- a/orchestrator.sh
+++ b/orchestrator.sh
@@ -31,6 +31,8 @@ ORCHESTRATOR_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")
 
 # shellcheck source=lib/common.sh
 . "$ORCHESTRATOR_ROOTDIR/lib/common.sh"
+# shellcheck source=lib/microvm.sh
+. "$ORCHESTRATOR_ROOTDIR/lib/microvm.sh"
 
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
@@ -56,11 +58,14 @@ ORCHESTRATOR_ISOLATION=${ORCHESTRATOR_ISOLATION:-"1"}
 # has been turned on.
 ORCHESTRATOR_SLEEP=${ORCHESTRATOR_SLEEP:-"30"}
 
+# Runtime to use when managing microVMs.
+ORCHESTRATOR_RUNTIME=${ORCHESTRATOR_RUNTIME:-""}
+
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Run krunvm-based GitHub runners on a single host"
 
 
-while getopts "s:Il:n:p:vh-" opt; do
+while getopts "s:Il:n:p:R:vh-" opt; do
   case "$opt" in
     s) # Number of seconds to sleep between microVM creation at start, when no isolation
       ORCHESTRATOR_SLEEP="$OPTARG";;
@@ -72,6 +77,8 @@ while getopts "s:Il:n:p:vh-" opt; do
       ORCHESTRATOR_RUNNERS="$OPTARG";;
     p) # Prefix to use for the VM name
       ORCHESTRATOR_PREFIX="$OPTARG";;
+    R) # Runtime to use when managing microVMs
+      ORCHESTRATOR_RUNTIME="$OPTARG";;
     v) # Increase verbosity, will otherwise log on errors/warnings only
       ORCHESTRATOR_VERBOSE=$((ORCHESTRATOR_VERBOSE+1));;
     h) # Print help and exit
@@ -99,14 +106,14 @@ cleanup() {
   # shellcheck disable=SC2086 # We want to wait for all pids
   waitpid $ORCHESTRATOR_PIDS
 
-  if run_krunvm list | grep -qE "^${ORCHESTRATOR_PREFIX}-"; then
-    while IFS= read -r vm; do
+  while IFS= read -r vm; do
+    if [ -n "$vm" ]; then
       verbose "Removing microVM $vm"
-      run_krunvm delete "$vm"
-    done <<EOF
-$(run_krunvm list | grep -E "^${ORCHESTRATOR_PREFIX}-")
+      microvm_delete "$vm"
+    fi
+  done <<EOF
+$(microvm_list | grep -E "^${ORCHESTRATOR_PREFIX}-")
 EOF
-  fi
 
   if [ -n "${ORCHESTRATOR_ENVIRONMENT:-}" ]; then
     verbose "Removing isolation environment $ORCHESTRATOR_ENVIRONMENT"
@@ -114,8 +121,8 @@ EOF
   fi
 }
 
-check_command buildah
-check_command krunvm
+# Pass the runtime to the microvm script
+microvm_runtime "$ORCHESTRATOR_RUNTIME"
 
 check_positive_number "$ORCHESTRATOR_RUNNERS" "Number of runners"
 
@@ -134,7 +141,8 @@ trap cleanup EXIT
 RUNNER_PREFIX=$ORCHESTRATOR_PREFIX
 RUNNER_VERBOSE=$ORCHESTRATOR_VERBOSE
 RUNNER_LOG=$ORCHESTRATOR_LOG
-export RUNNER_PREFIX RUNNER_ENVIRONMENT RUNNER_VERBOSE RUNNER_LOG
+RUNNER_RUNTIME=$ORCHESTRATOR_RUNTIME
+export RUNNER_PREFIX RUNNER_ENVIRONMENT RUNNER_VERBOSE RUNNER_LOG RUNNER_RUNTIME
 
 # Create runner loops in the background. One per runner. Each loop will
 # indefinitely create ephemeral runners. Looping is implemented in runner.sh,
diff --git a/runner.sh b/runner.sh
index ce8ef6e..cb1e6ab 100755
--- a/runner.sh
+++ b/runner.sh
@@ -31,6 +31,8 @@ RUNNER_ROOTDIR=$( cd -P -- "$(dirname -- "$(command -v -- "$(abspath "$0")")")"
 
 # shellcheck source=lib/common.sh
 . "$RUNNER_ROOTDIR/lib/common.sh"
+# shellcheck source=lib/microvm.sh
+. "$RUNNER_ROOTDIR/lib/microvm.sh"
 
 # Level of verbosity, the higher the more verbose. All messages are sent to the
 # stderr.
@@ -115,11 +117,15 @@ RUNNER_SECRET=${RUNNER_SECRET:-"$(random_string)"}
 # Number of seconds after which to terminate (empty for never, the good default)
 RUNNER_TERMINATE=${RUNNER_TERMINATE:-""}
 
+# Runtime to use when managing microVMs.
+RUNNER_RUNTIME=${RUNNER_RUNTIME:-""}
+
+
 # shellcheck disable=SC2034 # Used in sourced scripts
 KRUNVM_RUNNER_DESCR="Create runners forever using krunvm"
 
 
-while getopts "c:d:D:g:G:i:l:L:m:M:p:k:r:s:S:T:u:Uvh-" opt; do
+while getopts "c:d:D:g:G:i:l:L:m:M:p:k:r:R:s:S:T:u:Uvh-" opt; do
   case "$opt" in
     c) # Number of CPUs to allocate to the VM
       RUNNER_CPUS="$OPTARG";;
@@ -151,6 +157,8 @@ while getopts "c:d:D:g:G:i:l:L:m:M:p:k:r:s:S:T:u:Uvh-" opt; do
       RUNNER_TERMINATE="$OPTARG";;
     r) # Number of times to repeat the runner loop
       RUNNER_REPEAT="$OPTARG";;
+    R) # Runtime to use when managing microVMs
+      RUNNER_RUNTIME="$OPTARG";;
     s) # Scope of the runner, one of repo, org or enterprise
       RUNNER_SCOPE="$OPTARG";;
     S) # Secret to be used to request for loop end
@@ -186,9 +194,7 @@ fi
 # synchronisation files.
 RUNNER_VM_ENVDIR="_environment"
 
-if [ -z "$RUNNER_PAT" ]; then
-  error "You need to specify a PAT to acquire the runner token with"
-fi
+[ -z "$RUNNER_PAT" ] && error "You need to specify a PAT to acquire the runner token with"
 check_positive_number "$RUNNER_CPUS" "Number of vCPUs"
 check_positive_number "$RUNNER_MEMORY" "Memory (in MB)"
 
@@ -201,38 +207,19 @@ else
   RUNNER_ENTRYPOINT=${RUNNER_DIR%/}/runner/entrypoint.sh
 fi
 
-# Create the VM used for orchestration. Add --volume options for all necessary
-# mappings, i.e. inheritance of "live" code, environment isolation and all
-# requested mount points.
-vm_create() {
-  verbose "Creating microVM '${RUNNER_PREFIX}-$1', $RUNNER_CPUS vCPUs, ${RUNNER_MEMORY}M memory"
-  # Note: reset arguments!
-  set -- \
-    --cpus "$RUNNER_CPUS" \
-    --mem "$RUNNER_MEMORY" \
-    --dns "$RUNNER_DNS" \
-    --name "${RUNNER_PREFIX}-$1"
-  if [ -n "${RUNNER_DIR:-}" ]; then
-    set -- "$@" --volume "${RUNNER_ROOTDIR}:${RUNNER_DIR}"
-  fi
-  if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then
-    set -- "$@" --volume "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}"
-  fi
-  if [ -n "$RUNNER_MOUNT" ]; then
-    while IFS= read -r mount || [ -n "$mount" ]; do
-      if [ -n "$mount" ]; then
-        set -- "$@" --volume "$mount"
-      fi
-    done <<EOF
-$(printf %s\\n "$RUNNER_MOUNT")
-EOF
-  fi
-  run_krunvm create "$RUNNER_IMAGE" "$@"
-}
+# Pass the runtime to the microvm script
+microvm_runtime "$RUNNER_RUNTIME"
 
 
-vm_start() {
-  _id=$1
+# Call the microvm_run function with the necessary arguments to start a runner
+vm_run() {
+  _id=$1;  # Remember the ID for later
+
+  # First prepare the arguments that will be sent to the entrypoint.sh script.
+  # When an isolation environment can be constructed, create such a file and
+  # pass all relevant content through the file -- and point to the environment
+  # file that was created. Whenever this isn't possible, pass all relevant
+  # options one-by-one.
   if [ -n "$RUNNER_ENVIRONMENT" ]; then
     # Create an env file with most of the RUNNER_ variables. This works because
     # the `runner.sh` script that will be called uses the same set of variables.
@@ -268,19 +255,38 @@ EOF
       set -- -v "$@"
     done
   fi
-  verbose "Starting microVM '${RUNNER_PREFIX}-$_id' with entrypoint $RUNNER_ENTRYPOINT"
-  optstate=$(set +o)
-  set -m; # Disable job control
-  run_krunvm start "${RUNNER_PREFIX}-$_id" "$RUNNER_ENTRYPOINT" -- "$@" </dev/null &
-  RUNNER_PID=$!
-  eval "$optstate"; # Restore options
-  verbose "Started microVM '${RUNNER_PREFIX}-$_id' with PID $RUNNER_PID"
-  if [ -n "$RUNNER_TERMINATE" ]; then
-    verbose "Terminating runner in $RUNNER_TERMINATE seconds"
-    sleep "$RUNNER_TERMINATE" && cleanup &
+
+  # Add image to create from
+  set -- -- "$RUNNER_IMAGE" "$@"
+
+  # Create the VM used for orchestration. Add -v (volumes) options for all
+  # necessary mappings, i.e. inheritance of "live" code, environment isolation
+  # and all requested mount points.
+  set -- \
+    -c "$RUNNER_CPUS" \
+    -m "$RUNNER_MEMORY" \
+    -d "$RUNNER_DNS" \
+    -n "${RUNNER_PREFIX}-$_id" \
+    -e "$RUNNER_ENTRYPOINT" \
+    "$@"
+  if [ -n "${RUNNER_DIR:-}" ]; then
+    set -- -v "${RUNNER_ROOTDIR}:${RUNNER_DIR}" "$@"
   fi
-  wait "$RUNNER_PID"
-  RUNNER_PID=
+  if [ -n "${RUNNER_ENVIRONMENT:-}" ]; then
+    set -- -v "${RUNNER_ENVIRONMENT}:/${RUNNER_VM_ENVDIR}" "$@"
+  fi
+  if [ -n "$RUNNER_MOUNT" ]; then
+    while IFS= read -r mount || [ -n "$mount" ]; do
+      if [ -n "$mount" ]; then
+        set -- -v "$mount" "$@"
+      fi
+    done <<EOF
+$(printf %s\\n "$RUNNER_MOUNT")
+EOF
+  fi
+
+  trace "Running microVM with: $*"
+  microvm_run "$@"
 }
 
 
@@ -288,48 +294,35 @@ vm_delete() {
   # This is just a safety measure, the runner script should already have deleted
   # the environment
   if [ -n "$RUNNER_ENVIRONMENT" ] && [ -f "${RUNNER_ENVIRONMENT}/${1}.env" ]; then
-    warning "Removing isolation environment ${RUNNER_ENVIRONMENT}/${1}.env"
+    warn "Removing isolation environment ${RUNNER_ENVIRONMENT}/${1}.env"
     rm -f "${RUNNER_ENVIRONMENT}/${1}.env"
   fi
-  if run_krunvm list | grep -qE "^${RUNNER_PREFIX}-$1"; then
+  if microvm_list | grep -qE "^${RUNNER_PREFIX}-$1"; then
     verbose "Removing microVM '${RUNNER_PREFIX}-$1'"
-    run_krunvm delete "${RUNNER_PREFIX}-$1"
+    microvm_delete "${RUNNER_PREFIX}-$1"
   fi
 }
 
 vm_terminate() {
-  if [ -n "$RUNNER_ENVIRONMENT" ]; then
-    if [ -f "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.tkn" ]; then
-      if [ -n "${RUNNER_SECRET:-}" ]; then
-        verbose "Requesting termination via ${RUNNER_ENVIRONMENT}/${RUNNER_ID}.trm"
-        printf %s\\n "$RUNNER_SECRET" > "${RUNNER_ENVIRONMENT}/${RUNNER_ID}.trm"
-      elif [ -n "$RUNNER_PID" ]; then
-        kill_tree "$RUNNER_PID"
-      fi
-      if [ "$RUNNER_PID" ]; then
-        # shellcheck disable=SC2046 # We want to wait for all children
-        waitpid $(ps_tree "$RUNNER_PID"|tac)
-      else
-        warning "No PID to wait for"
-      fi
-    elif [ -n "$RUNNER_PID" ]; then
-      kill_tree "$RUNNER_PID"
-      # shellcheck disable=SC2046 # We want to wait for all children
-      waitpid $(ps_tree "$RUNNER_PID"|tac)
-    fi
-  elif [ -n "$RUNNER_PID" ]; then
-    kill_tree "$RUNNER_PID"
-    # shellcheck disable=SC2046 # We want to wait for all children
-    waitpid $(ps_tree "$RUNNER_PID"|tac)
+  # Request for termination through .trm file, whenever possible. Otherwise,
+  # just stop the VM.
+  if [ -n "$RUNNER_ENVIRONMENT" ] \
+      && [ -f "${RUNNER_ENVIRONMENT}/${1}.tkn" ] \
+      && [ -n "${RUNNER_SECRET:-}" ]; then
+    verbose "Requesting termination via ${RUNNER_ENVIRONMENT}/${1}.trm"
+    printf %s\\n "$RUNNER_SECRET" > "${RUNNER_ENVIRONMENT}/${1}.trm"
+    microvm_wait "${RUNNER_PREFIX}-$1"
+  else
+    microvm_stop "${RUNNER_PREFIX}-$1"
   fi
 }
 
+
 cleanup() {
   trap '' EXIT
-  if [ -n "${RUNNER_PID:-}" ]; then
-    vm_terminate
-  fi
+
   if [ -n "${RUNNER_ID:-}" ]; then
+    vm_terminate "$RUNNER_ID"
     vm_delete "$RUNNER_ID"
   fi
 }
@@ -339,9 +332,19 @@ trap cleanup EXIT
 
 iteration=0
 while true; do
+  # Prefetch, since this might take time and we want to be ready to count away
+  # download time from the termination setting.
+  microvm_pull "$RUNNER_IMAGE"
+
+  # Terminate in xx seconds. This is mostly used for demo purposes, but might
+  # help keeping the machines "warm" and actualised (as per the pull above).
+  if [ -n "$RUNNER_TERMINATE" ]; then
+    verbose "Terminating runner in $RUNNER_TERMINATE seconds"
+    sleep "$RUNNER_TERMINATE" && cleanup &
+  fi
+
   RUNNER_ID="${loop}-$(random_string)"
-  vm_create "${RUNNER_ID}"
-  vm_start "${RUNNER_ID}"
+  vm_run "${RUNNER_ID}"
   vm_delete "${RUNNER_ID}"
   RUNNER_ID=