feat(helm): add list of container images

.github/workflows/k8s-ci.yml

@@ -19,10 +19,17 @@ jobs:
           nix-shell ./scripts/k8s/shell.nix --run "echo"
       - name: BootStrap k8s cluster
         run: |
-          nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label"
+          nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label --dump-kind-images"
+
       - name: Install Helm Chart
         run: |
           nix-shell ./scripts/k8s/shell.nix --run "./scripts/helm/install.sh --dep-update --wait"
+
+      - name: Container Image List
+        run: |
+          # Verifies if the list of images is accurate
+          nix-shell ./scripts/k8s/shell.nix --run "./scripts/helm/verify-kind-images.sh"
+
       - name: The job has failed
         if: ${{ failure() }}
         run: |

chart/dependencies-images.txt

@@ -0,0 +1,3 @@
+# localpv-provisioner
+# todo: we need a way to fetch this from the dependencies
+docker.io/openebs/linux-utils:4.1.0

chart/images.txt

@@ -0,0 +1,37 @@
+docker.io/grafana/promtail:2.8.3
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/mayastor-agent-ha-node:develop
+docker.io/openebs/mayastor-csi-node:develop
+registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/mayastor-metrics-exporter-io-engine:develop
+docker.io/openebs/mayastor-io-engine:develop
+openebs/provisioner-localpv:4.1.0
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/mayastor-agent-core:develop
+docker.io/openebs/mayastor-agent-ha-cluster:develop
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/mayastor-api-rest:develop
+docker.io/openebs/alpine-sh:4.1.0
+registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+registry.k8s.io/sig-storage/csi-attacher:v4.3.0
+registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3
+registry.k8s.io/sig-storage/snapshot-controller:v6.3.3
+registry.k8s.io/sig-storage/csi-resizer:v1.9.3
+docker.io/openebs/mayastor-csi-controller:develop
+docker.io/openebs/mayastor-obs-callhome:develop
+docker.io/openebs/mayastor-obs-callhome-stats:develop
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/alpine-sh:4.1.0
+docker.io/openebs/mayastor-operator-diskpool:develop
+docker.io/openebs/alpine-bash:4.1.0
+docker.io/bitnami/etcd:3.5.6-debian-11-r10
+docker.io/openebs/alpine-sh:4.1.0
+grafana/loki:2.6.1
+nats:2.9.17-alpine
+natsio/nats-server-config-reloader:0.10.1
+natsio/prometheus-nats-exporter:0.11.0
+bats/bats:1.8.2
+natsio/nats-box:0.13.8

scripts/helm/install.sh

@@ -99,4 +99,4 @@ else
   set +x
 fi
 
-kubectl get pods -n mayastor -o wide
+kubectl get pods -n "$K8S_NAMESPACE" -o wide

scripts/helm/template-images.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
+ROOT_DIR="$SCRIPT_DIR/../.."
+CHART_DIR="$ROOT_DIR/chart"
+IMAGES="$CHART_DIR/images.txt"
+
+EXIT_CODE=
+DRY_RUN=
+DEP_UPDATE=
+HELM="helm"
+ENABLE_ANALYTICS="eventing.enabled=true,obs.callhome.enabled=true,obs.callhome.sendReport=true,localpv-provisioner.analytics.enabled=true"
+
+help() {
+  cat <<EOF
+Usage: $(basename "$0") [COMMAND] [OPTIONS]
+
+Options:
+  -h, --help                 Display this text.
+  --exit-code                Exit with error code if the images file changed ($IMAGES).
+  --dry-run                  Show which commands we'd run, but don't run them.
+  --dependency-update        Run helm dependency update as the first step.
+Examples:
+  $(basename "$0")
+EOF
+}
+
+echo_stderr() {
+  echo -e "${1}" >&2
+}
+
+die() {
+  local _return="${2:-1}"
+  echo_stderr "$1"
+  exit "${_return}"
+}
+
+while [ "$#" -gt 0 ]; do
+  case $1 in
+    -h|--help)
+      help
+      exit 0
+      shift;;
+    --exit-code)
+      EXIT_CODE="true"
+      shift;;
+    --dry-run)
+      DRY_RUN="true"
+      HELM="echo $HELM"
+      shift;;
+    --dependency-update)
+      DEP_UPDATE="true"
+      shift;;
+    *)
+      die "Unknown argument $1!"
+      shift;;
+  esac
+done
+
+cd "$CHART_DIR"
+
+if [ "$DEP_UPDATE" = "true" ]; then
+    $HELM dependency update
+fi
+
+$HELM template . --set "$ENABLE_ANALYTICS" | grep "image:" | awk '{ print $2 }' | tr -d \" > "$IMAGES"
+
+if [ "$EXIT_CODE" = "true" ]; then
+  git diff --exit-code "$IMAGES"
+fi

scripts/helm/verify-kind-images.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+
+SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")"
+ROOT_DIR="$SCRIPT_DIR/../.."
+CHART_DIR="$ROOT_DIR/chart"
+HELM_TPL_IMAGES="$CHART_DIR/images.txt"
+HELM_DEP_IMAGES="$CHART_DIR/dependencies-images.txt"
+TMP_KIND="/tmp/kind/mayastor"
+KIND_IMAGES="kind-images.txt"
+DOCKER="docker"
+
+echo_stderr() {
+  echo -e "${1}" >&2
+}
+
+die() {
+  local _return="${2:-1}"
+  echo_stderr "$1"
+  exit "${_return}"
+}
+
+grep_image() {
+  local image="$1"
+  local file="$2"
+
+  grep -q "^$image$" "$file" || grep -q "^${image##docker.io/}$" "$file"
+}
+
+if ! [ -f "$HELM_TPL_IMAGES" ]; then
+  die "Missing helm template images, please generate them"
+fi
+
+cd "$TMP_KIND"
+for worker in kind-worker*; do
+  if ! [ -f "$worker/$KIND_IMAGES" ]; then
+    die "Missing $KIND_IMAGES file"
+  fi
+
+  curr_images=$($DOCKER exec $worker crictl image | tail -n+2 | awk '{ print $1 ":" $2 }')
+  for image in ${curr_images[@]}; do
+    if grep_image "$image" "$worker/$KIND_IMAGES"; then
+      # if it's there before the install, then ignore it.
+      continue
+    fi
+    if ! grep_image "$image" "$HELM_TPL_IMAGES" && ! grep_image "$image" "$HELM_DEP_IMAGES"; then
+      echo "$image not found"
+    fi
+  done
+done

scripts/k8s/deployer.sh

@@ -17,6 +17,8 @@ DOCKER="docker"
 HUGE_PAGES=1800
 LABEL=
 SUDO=${SUDO:-"sudo"}
+DUMP_KIND_IMG=
+KIND_IMAGES="kind-images.txt"
 
 help() {
   cat <<EOF
@@ -31,6 +33,7 @@ Options:
   --dry-run                         Don't do anything, just output steps.
   --hugepages     <num>             Add <num> 2MiB hugepages (Default: $HUGE_PAGES).
   --label                           Label worker nodes with the io-engine selector.
+  --dump-kind-images                Dump the used images on the worker nodes to $TMP_KIND/$node-name/kind-images.txt
 
 Command:
   start                             Start the k8s cluster.
@@ -95,6 +98,9 @@ while [ "$#" -gt 0 ]; do
           test $# -lt 1 && die "Missing hugepage number"
           HUGE_PAGES=$1
           shift;;
+        --dump-kind-images)
+          DUMP_KIND_IMG="true"
+          shift;;
         --dry-run)
           if [ -z "$DRY_RUN" ]; then
             DRY_RUN="--dry-run"
@@ -106,6 +112,7 @@ while [ "$#" -gt 0 ]; do
           fi
           shift;;
         *)
+          die "Unknown cli argument: $1"
           shift;;
       esac
   esac
@@ -189,6 +196,10 @@ echo "HostIP: $host_ip"
 for node in ${nodes[@]}; do
   $DOCKER exec $node mount -o remount,rw /sys
 
+  if [ "$DUMP_KIND_IMG" = "true" ] && [ -z "$DRY_RUN" ]; then
+    $DOCKER exec $node crictl image | tail -n+2 | awk '{ print $1 ":" $2 }' > "$TMP_KIND/$node/$KIND_IMAGES"
+  fi
+
   # Note: this will go away if the node restarts...
   $DOCKER exec $node bash -c 'printf "'$host_ip' kvmhost\n" >> /etc/hosts'
 done