diff --git a/.github/bors.toml b/.github/bors.toml index bb05fb792..b5bfb1414 100644 --- a/.github/bors.toml +++ b/.github/bors.toml @@ -1,5 +1,5 @@ status = [ "bors-ci" ] -pr_status = [ "helm-chart-test", "submodule-branch", "commitlint", "DCO" ] +pr_status = [ "helm-chart-test", "helm-images", "submodule-branch", "commitlint", "DCO" ] timeout_sec = 10000 required_approvals = 2 delete_merged_branches = true diff --git a/.github/workflows/develop-to-release.yml b/.github/workflows/develop-to-release.yml index 79349d358..eade5ca22 100644 --- a/.github/workflows/develop-to-release.yml +++ b/.github/workflows/develop-to-release.yml @@ -18,11 +18,16 @@ jobs: run: | branch="${{ github.ref_name }}" nix-shell --pure --run "./scripts/helm/publish-chart-yaml.sh --check-chart "$branch" --develop-to-release" ./scripts/helm/shell.nix - nix-shell --pure --run "SKIP_GIT=1 ./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix - name: Check if the submodules are correct run: | branch="${{ github.ref_name }}" ./scripts/git/set-submodule-branches.sh --branch "$branch" + - name: Update the Chart Readme.md + run: nix-shell ./scripts/helm/shell.nix --pure --run "SKIP_GIT=1 ./scripts/helm/generate-readme.sh" + - name: Update the Chart images + run: | + nix-shell ./scripts/helm/shell.nix --pure --run "./scripts/helm/images.sh generate --dependency-update" + nix-shell ./scripts/helm/shell.nix --pure --run "./scripts/helm/images.sh patch" - name: Create Pull Request id: cpr uses: peter-evans/create-pull-request@v5 @@ -36,14 +41,16 @@ jobs: automated-pr draft: false signoff: true - token: ${{ secrets.ORG_CI_GITHUB }} - - name: Approve Pull Request by CI Bot + delete-branch: true + branch-suffix: "random" + token: ${{ github.token }} + - name: Approve Pull Request by CI User 1 if: ${{ steps.cpr.outputs.pull-request-number }} run: | gh pr review ${{ steps.cpr.outputs.pull-request-number }} --approve env: - GH_TOKEN: ${{ github.token }} - - name: Approve Pull Request by CI User + GH_TOKEN: ${{ secrets.ORG_CI_GITHUB }} + - name: Approve Pull Request by CI User 2 if: ${{ steps.cpr.outputs.pull-request-number }} run: | gh pr review ${{ steps.cpr.outputs.pull-request-number }} --approve diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 21ef02b7a..504af9e58 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -29,3 +29,35 @@ jobs: run: nix-shell --run "./scripts/helm/generate-readme.sh" ./scripts/helm/shell.nix - name: HelmChart Template run: nix-shell --pure --run "./scripts/helm/test-template.sh" ./scripts/helm/shell.nix + + helm-images: + runs-on: ubuntu-latest-8-cores + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: DeterminateSystems/nix-installer-action@v11 + with: + kvm: true + - uses: DeterminateSystems/magic-nix-cache-action@v6 + - name: Setup Nix Path + run: | + export NIX_PATH=nixpkgs=$(jq '.nixpkgs.url' nix/sources.json -r) + echo "NIX_PATH=$NIX_PATH" >> $GITHUB_ENV + - name: Pre-populate K8s nix-shell + run: nix-shell ./scripts/k8s/shell.nix --run "echo" + - name: Pre-populate helm nix-shell + run: nix-shell ./scripts/helm/shell.nix --run "echo" + - name: Generate image list + run: nix-shell ./scripts/helm/shell.nix --run "./scripts/helm/images.sh generate --dependency-update --exit-code" + - name: BootStrap k8s cluster + if: github.event_name != 'pull_request' + run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" + - name: Install helm chart + if: github.event_name != 'pull_request' + run: nix-shell ./scripts/helm/shell.nix --run "./scripts/helm/install.sh --wait" + - name: Verify image list + if: github.event_name != 'pull_request' + run: nix-shell ./scripts/helm/shell.nix --run "./scripts/helm/images.sh verify" + - name: Patch chart/Chart.yaml + run: nix-shell ./scripts/helm/shell.nix --run "./scripts/helm/images.sh patch --exit-code" diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index 368819cce..358c1a440 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -4,7 +4,7 @@ on: jobs: k8s-ci: - runs-on: ubuntu-latest-16-cores + runs-on: ubuntu-latest-8-cores steps: - name: Bind mount /dev/sda1 to /nix run: | @@ -63,7 +63,7 @@ jobs: title: Test results k8s-ci-vm: - runs-on: ubuntu-latest-16-cores + runs-on: ubuntu-latest-8-cores steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@v11 diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index f25270925..be1a4b249 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -80,14 +80,14 @@ jobs: delete-branch: true branch-suffix: "random" base: ${{ env.BASE_BRANCH }} - token: ${{ secrets.ORG_CI_GITHUB }} - - name: Approve Pull Request by CI Bot + token: ${{ github.token }} + - name: Approve Pull Request by CI User 1 if: ${{ steps.cpr.outputs.pull-request-number }} run: | gh pr review ${{ steps.cpr.outputs.pull-request-number }} --approve env: - GH_TOKEN: ${{ github.token }} - - name: Approve Pull Request by CI User + GH_TOKEN: ${{ secrets.ORG_CI_GITHUB }} + - name: Approve Pull Request by CI User 2 if: ${{ steps.cpr.outputs.pull-request-number }} run: | gh pr review ${{ steps.cpr.outputs.pull-request-number }} --approve diff --git a/chart/.helmignore b/chart/.helmignore index eaba7b26b..39fed2c16 100644 --- a/chart/.helmignore +++ b/chart/.helmignore @@ -24,4 +24,6 @@ README.md.tmpl # Nix Shell *.nix -kubectl-plugin/ \ No newline at end of file +kubectl-plugin/ +# Chart Metadata +images.txt diff --git a/chart/Chart.yaml b/chart/Chart.yaml index e011634b6..109749a59 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -48,3 +48,63 @@ dependencies: version: 4.2.0 repository: https://openebs.github.io/dynamic-localpv-provisioner condition: localpv-provisioner.enabled +annotations: + helm.sh/images: | + - name: bats + image: bats/bats:1.8.2 + - name: etcd + image: docker.io/bitnami/etcd:3.5.6-debian-11-r10 + - name: promtail + image: docker.io/grafana/promtail:2.8.3 + - name: alpine-bash + image: docker.io/openebs/alpine-bash:4.1.0 + - name: alpine-sh + image: docker.io/openebs/alpine-sh:4.1.0 + - name: mayastor-agent-core + image: docker.io/openebs/mayastor-agent-core:develop + - name: mayastor-agent-ha-cluster + image: docker.io/openebs/mayastor-agent-ha-cluster:develop + - name: mayastor-agent-ha-node + image: docker.io/openebs/mayastor-agent-ha-node:develop + - name: mayastor-api-rest + image: docker.io/openebs/mayastor-api-rest:develop + - name: mayastor-csi-controller + image: docker.io/openebs/mayastor-csi-controller:develop + - name: mayastor-csi-node + image: docker.io/openebs/mayastor-csi-node:develop + - name: mayastor-io-engine + image: docker.io/openebs/mayastor-io-engine:develop + - name: mayastor-metrics-exporter-io-engine + image: docker.io/openebs/mayastor-metrics-exporter-io-engine:develop + - name: mayastor-obs-callhome + image: docker.io/openebs/mayastor-obs-callhome:develop + - name: mayastor-obs-callhome-stats + image: docker.io/openebs/mayastor-obs-callhome-stats:develop + - name: mayastor-operator-diskpool + image: docker.io/openebs/mayastor-operator-diskpool:develop + - name: loki + image: grafana/loki:2.6.1 + - name: nats + image: nats:2.9.17-alpine + - name: nats-box + image: natsio/nats-box:0.13.8 + - name: nats-server-config-reloader + image: natsio/nats-server-config-reloader:0.10.1 + - name: prometheus-nats-exporter + image: natsio/prometheus-nats-exporter:0.11.0 + - name: linux-utils + image: openebs/linux-utils:4.0.0 + - name: provisioner-localpv + image: openebs/provisioner-localpv:4.1.2 + - name: csi-attacher + image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + - name: csi-node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.3 + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3 + - name: snapshot-controller + image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.3 diff --git a/chart/images.txt b/chart/images.txt new file mode 100644 index 000000000..34fb2aff7 --- /dev/null +++ b/chart/images.txt @@ -0,0 +1,29 @@ +bats/bats:1.8.2 +docker.io/bitnami/etcd:3.5.6-debian-11-r10 +docker.io/grafana/promtail:2.8.3 +docker.io/openebs/alpine-bash:4.1.0 +docker.io/openebs/alpine-sh:4.1.0 +docker.io/openebs/mayastor-agent-core:develop +docker.io/openebs/mayastor-agent-ha-cluster:develop +docker.io/openebs/mayastor-agent-ha-node:develop +docker.io/openebs/mayastor-api-rest:develop +docker.io/openebs/mayastor-csi-controller:develop +docker.io/openebs/mayastor-csi-node:develop +docker.io/openebs/mayastor-io-engine:develop +docker.io/openebs/mayastor-metrics-exporter-io-engine:develop +docker.io/openebs/mayastor-obs-callhome-stats:develop +docker.io/openebs/mayastor-obs-callhome:develop +docker.io/openebs/mayastor-operator-diskpool:develop +grafana/loki:2.6.1 +nats:2.9.17-alpine +natsio/nats-box:0.13.8 +natsio/nats-server-config-reloader:0.10.1 +natsio/prometheus-nats-exporter:0.11.0 +openebs/linux-utils:4.1.0 +openebs/provisioner-localpv:4.2.0 +registry.k8s.io/sig-storage/csi-attacher:v4.3.0 +registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 +registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 +registry.k8s.io/sig-storage/csi-resizer:v1.9.3 +registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3 +registry.k8s.io/sig-storage/snapshot-controller:v6.3.3 diff --git a/chart/shell.nix b/chart/shell.nix index 6df345629..86b69c0ca 100644 --- a/chart/shell.nix +++ b/chart/shell.nix @@ -11,5 +11,6 @@ pkgs.mkShell { kubernetes-helm-wrapped semver-tool yq-go + jq ]; } diff --git a/scripts/helm/images.sh b/scripts/helm/images.sh new file mode 100755 index 000000000..02477be80 --- /dev/null +++ b/scripts/helm/images.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." +CHART_DIR="$ROOT_DIR/chart" +CHART="$CHART_DIR/Chart.yaml" +IMAGES="$CHART_DIR/images.txt" +NAMESPACE="mayastor" + +source "$ROOT_DIR/scripts/utils/yaml.sh" +source "$ROOT_DIR/scripts/utils/log.sh" + +EXIT_CODE= +DEP_UPDATE= +HELM="helm" +ENABLE_ANALYTICS="eventing.enabled=true,obs.callhome.enabled=true,obs.callhome.sendReport=true,localpv-provisioner.analytics.enabled=true" + +helm_dep_value() { + local dep_chart="${1:-}" + local dep_name="${2:-}" + local key="${3:-}" + local value="" + + # First, check if we're setting the value in our values.yaml + value="$($HELM show values "$CHART_DIR" --kubeconfig "$CHART_DIR/fake" | yq -r ".$dep_name.$key")" + if [ -n "$value" ] && [ "$value" != "null" ]; then + echo "$value" + return 0 + fi + + # Otherwise, get it from the source (or rather, the dependency :)) + if ! value="$($HELM show values "$dep_chart" --kubeconfig "$CHART_DIR/fake" | yq -r ".$key")"; then + log_fatal "Can't show the helm dependency $dep_name: $dep_chart" + fi + if [ -n "$value" ] && [ "$value" != "null" ]; then + echo "$value" + return 0 + fi +} + +helm_dep_value_required() { + local dep_chart="${1:-}" + local dep_name="${2:-}" + local key="${3:-}" + local value + value=$(helm_dep_value "$dep_chart" "$dep_name" "$key") + [ -z "$value" ] && log_fatal "I can't find $dep_name.$key neither in our chart nor $dep_chart" + echo "$value" +} + +helm_dep_version() { + $HELM show chart "$CHART_DIR" --kubeconfig "$CHART_DIR/fake" | dep="${1:-}" yq '.dependencies[]|select(.name == strenv(dep)).version' +} + +helm_localpv_prov_helper_image() { + local registry + local repository + local tag + local version + local name="localpv-provisioner" + local chart + + version=$(helm_dep_version "$name") + if [ "$version" = "" ]; then + log_fatal "Can't find the version of the helm dependency: $name" + fi + chart="$CHART_DIR/charts/$name-$version.tgz" + if ! [ -f "$chart" ]; then + log_fatal "Can't find the helm dependency: $chart" + fi + + registry="$(helm_dep_value "$chart" "$name" "helperPod.image.registry")" + repository="$(helm_dep_value_required "$chart" "$name" "helperPod.image.repository")" + tag="$(helm_dep_value_required "$chart" "$name" "helperPod.image.tag")" + + [ -n "$registry" ] && registry="${registry%/}/" + echo "$registry$repository:$tag" +} + +helm_on_demand_images() { + local images="${1:-}" + + helm_localpv_prov_helper_image >> "$images" +} + +# This fetches the dependencies in an exact version from the Chart.yaml +# NOTE: This won't work if we ever modify the Chart.yaml to specify non-pinned versions, ex: 14 vs 14.0.0 +# Update can be forced with global var DEP_UPDATE="true". +helm_dep_update() { + local update="false" + + if [ "$DEP_UPDATE" = "true" ]; then + update="true" + else + local deps + + if ! deps=$($HELM show chart "$CHART_DIR" --kubeconfig "$CHART_DIR/fake" | yq -ojson '.dependencies[]|select(.repository != "")' | jq -c); then + log_fatal "Can't find the helm dependencies in $CHART_DIR" + fi + + for chart in ${deps[@]}; do + version=$(echo "$chart" | jq -r '.version') + name=$(echo "$chart" | jq -r '.name') + if [ "$(semver validate "$version")" != "valid" ]; then + log_fatal "Found $name with version $version only pinned versions are supported!" + fi + if ! [ -f "$CHART_DIR/charts/$name-$version.tgz" ]; then + update="true" + break + fi + done + fi + + if [ "$update" = "true" ]; then + $HELM dependency update "$CHART_DIR" --kubeconfig "$CHART_DIR/fake" + fi +} + +cleanup() { + if [ -f "${TEMPLATE_IMAGES:-}" ]; then + rm "${TEMPLATE_IMAGES:-}" + fi + if [ -d "${CHART_VERSIONED:-}" ]; then + rm -rf "${CHART_VERSIONED:-}" + fi + if [ -f "${LIVE_IMAGES:-}" ]; then + rm "${LIVE_IMAGES:-}" + fi +} + +help() { + cat < "$TEMPLATE_IMAGES" + + # Second, we handle images which are deployed on-demand by the product and its dependencies. + # An example of this is the init pod's deployed by the localpv-provisioner in order to prepare the filesystem for + # a PVC. + helm_on_demand_images "$TEMPLATE_IMAGES" + + cat "$TEMPLATE_IMAGES" | LC_ALL=C sort | uniq > "$IMAGES" + + if [ "$EXIT_CODE" = "true" ]; then + git diff --exit-code "$IMAGES" + fi + + echo "Finished generating the images:" + cat "$IMAGES" + ;; + patch) + LIST="" + while IFS= read -r image; do + name=$(echo "$image" | awk -F '[/:]' '{print $(NF-1)}') + if [ -z "$LIST" ]; then + LIST=$(echo -n "{ \"name\": \"$name\", \"image\": \"$image\" }") + else + LIST=$(echo -n "$LIST,{ \"name\": \"$name\", \"image\": \"$image\" }") + fi + done < "$IMAGES" + + yq_ibl ".annotations.\"helm.sh/images\" |= [$LIST]" "$CHART" + sed -i 's/ helm.sh\/images:/ helm.sh\/images: |/' "$CHART" + + if [ "$EXIT_CODE" = "true" ]; then + git diff --exit-code "$IMAGES" + fi + + echo "Finished patching $CHART" + cat "$CHART" + ;; + verify) + LIVE_IMAGES=$(mktemp helm-XXXXXX.txt) + MISSING_IMAGES="" + + kubectl -n "$NAMESPACE" get pods -o json | jq -r '.items[].spec.containers[]?.image, .items[].spec.initContainers[]?.image' | LC_ALL=C sort | uniq > "$LIVE_IMAGES" + + while IFS= read -r image; do + if ! grep -xq "$image" "$IMAGES"; then + if [ -z "$MISSING_IMAGES" ]; then + MISSING_IMAGES="$image" + else + MISSING_IMAGES="$MISSING_IMAGES, $image" + fi + fi + done < "$LIVE_IMAGES" + + if [ -n "$MISSING_IMAGES" ]; then + log_fatal "Missing images: $MISSING_IMAGES" + fi + + echo "Finished verifying the images" + ;; + *) + log_fatal "Missing Command" + ;; +esac diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index 52c3be3c3..1c1b1e092 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -24,6 +24,7 @@ TIMEOUT="5m" WAIT= DRY_RUN= HELM_DRY_RUN="" +HELM_ARGS= SCRIPT_DIR="$(dirname "$0")" CHART_DIR="$SCRIPT_DIR"/../../chart CHART_SOURCE=$CHART_DIR @@ -51,8 +52,9 @@ Options: --dep-update Run helm dependency update. --fail-if-installed Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace. --hosted-chart Install a hosted chart instead of the local chart. - --version Set the version/version-range for the chart. Works only when used with the '--hosted' option. + --version Set the version/version-range for the chart. Works only when used with the '--hosted' option. --registry Set the registry URL for the hosted chart. Works only when used with the '--hosted' option. (Default: $DEFAULT_REGISTRY) + --helm Pass Helm Args directly to the install/upgrade commands. Examples: $(basename "$0") @@ -118,6 +120,11 @@ while [ "$#" -gt 0 ]; do REGISTRY="${1#*=}" fi shift;; + --helm) + shift + test $# -lt 1 && die "Missing helm args" + HELM_ARGS="${HELM_ARGS:-} $1" + shift;; *) die "Unknown argument $1!" shift;; @@ -157,8 +164,8 @@ else $HELM install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \ --set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \ --set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \ - --set="eventing.enabled=false" \ - $HELM_DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG + --set="eventing.enabled=true,nats.cluster.enabled=false,nats.cluster.replicas=1" \ + $HELM_DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG ${HELM_ARGS:-} set +x fi diff --git a/scripts/k8s/deployer.sh b/scripts/k8s/deployer.sh index c97bbff2a..c15cd6616 100755 --- a/scripts/k8s/deployer.sh +++ b/scripts/k8s/deployer.sh @@ -60,7 +60,7 @@ while [ "$#" -gt 0 ]; do -h|--help) help exit 0 - shift;; + ;; start) COMMAND="start" DO_ARGS="y" @@ -111,7 +111,8 @@ while [ "$#" -gt 0 ]; do fi shift;; *) - shift;; + die "Unknown cli argument: $1" + ;; esac esac done @@ -147,13 +148,13 @@ EOF start_core=1 nodes=() -for node_index in $(seq 1 $WORKERS); do +for node_index in $(seq 1 "$WORKERS"); do if [ "$node_index" == 1 ]; then node="kind-worker" else node="kind-worker$node_index" fi - nodes+=($node) + nodes+=("$node") host_path="$TMP_KIND/$node" cat <> "$TMP_KIND_CONFIG" @@ -175,12 +176,12 @@ EOF openebs.io/engine: mayastor EOF fi - mkdir -p $host_path/io-engine + mkdir -p "$host_path"/io-engine if [ -n "$POOL_SIZE" ]; then - $FALLOCATE -l $POOL_SIZE $host_path/io-engine/disk.io + $FALLOCATE -l "$POOL_SIZE" "$host_path"/io-engine/disk.io fi corelist=$(seq -s, $((start_core+((node_index-1)*CORES))) 1 $((start_core-1+((node_index)*CORES)))) - printf "eal_opts:\n core_list: $corelist\n developer_delay: $DELAY\n" >$host_path/io-engine/config.yaml + printf "eal_opts:\n core_list: %s\n developer_delay: %s\n" "$corelist" "$DELAY" >"$host_path"/io-engine/config.yaml done if [ -n "$DRY_RUN" ]; then @@ -191,13 +192,14 @@ $KIND create cluster --config "$TMP_KIND_CONFIG" $KUBECTL cluster-info --context kind-kind if [ -z "$DRY_RUN" ]; then - host_ip=$($DOCKER network inspect kind | jq '.[0].IPAM.Config[0].Gateway') + host_ip=$($DOCKER network inspect kind | jq -r 'first (.[0].IPAM.Config[].Gateway | select(.))') fi echo "HostIP: $host_ip" +# shellcheck disable=SC2068 for node in ${nodes[@]}; do - $DOCKER exec $node mount -o remount,rw /sys + $DOCKER exec "$node" mount -o remount,rw /sys # Note: this will go away if the node restarts... - $DOCKER exec $node bash -c 'printf "'$host_ip' kvmhost\n" >> /etc/hosts' + $DOCKER exec "$node" bash -c 'printf "'"$host_ip"' kvmhost\n" >> /etc/hosts' done diff --git a/scripts/utils/yaml.sh b/scripts/utils/yaml.sh index 5a72ebe40..49086d2f7 100644 --- a/scripts/utils/yaml.sh +++ b/scripts/utils/yaml.sh @@ -5,7 +5,7 @@ yq_ibl() { error=0 - diff_out=$(diff -B <(yq '.' "$2") <(yq "$1" "$2")) || error=$? + diff_out=$(diff -B <(cat "$2") <(yq "$1" "$2")) || error=$? if [ "$error" != "0" ] && [ "$error" != "1" ]; then exit "$error" fi