ci.yaml

name: CI

on:
  pull_request: {}
  push:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: write

jobs:
  git-sha:
    name: Git SHA
    runs-on: ubuntu-latest
    outputs:
      git_sha: ${{ steps.git_sha.outputs.git_sha }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/git-sha
        id: git_sha

  sanitize:
    name: Sanitize
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Go vet
        run: |
          make vet
      - name: Lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.60.3
          args: --build-tags exclude_graphdriver_btrfs

  test:
    name: Unit tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Unit tests
        run: |
          make unit-tests

  int-tests:
    name: Integration tests
    runs-on: ubuntu-latest
    needs:
      - int-tests-kind
    steps:
      - name: Succeed if all tests passed
        run: echo "Integration tests succeeded"

  int-tests-kind:
    name: Integration tests (kind)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Install kind
        run: |
          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
          chmod +x ./kind
          sudo mv ./kind /usr/local/bin/kind
      - name: Run tests
        run: |
          make -C tests/integration test-kind

  dryrun-tests:
    name: Dryrun tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Go cache
        uses: actions/cache@v4
        with:
          path: |
            ./dev/.gocache
            ./dev/.gomodcache
          key: dryrun-tests-go-cache
      - name: Dryrun tests
        run: |
          make dryrun-tests

  check-operator-crds:
    name: Check operator CRDs
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Make manifests
      run: make -C operator manifests
    - name: Check CRDs
      run: |
        git diff --exit-code --name-only
        if [ $? -eq 0 ]; then
          echo "CRDs are up to date"
        else
          echo "CRDs are out of date"
          exit 1
        fi

  buildtools:
    name: Build buildtools
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Compile buildtools
        run: |
          make buildtools
      - name: Upload buildtools artifact
        uses: actions/upload-artifact@v4
        with:
          name: buildtools
          path: output/bin/buildtools

  build-current:
    name: Build current
    runs-on: embedded-cluster-2
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Install dagger
        run: |
          curl -fsSL https://dl.dagger.io/dagger/install.sh | sh
          sudo mv ./bin/dagger /usr/local/bin/dagger

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-install
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          export K0S_VERSION=$(make print-K0S_VERSION)
          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}
          # avoid rate limiting
          export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2)

          ./scripts/build-and-release.sh

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: current-release
          path: |
            output/bin/embedded-cluster
            output/bin/embedded-cluster-original
            output/bin/embedded-cluster-release-builder

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  build-legacydr:
    name: Build legacy DR
    runs-on: embedded-cluster-2
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Install dagger
        run: |
          curl -fsSL https://dl.dagger.io/dagger/install.sh | sh
          sudo mv ./bin/dagger /usr/local/bin/dagger

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-install-legacydr
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          export K0S_VERSION=$(make print-K0S_VERSION)
          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-legacydr
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-legacydr
          # avoid rate limiting
          export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2)

          ./scripts/build-and-release.sh
          cp output/bin/embedded-cluster output/bin/embedded-cluster-legacydr

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: legacydr-release
          path: |
            output/bin/embedded-cluster-legacydr

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  build-previous-k0s:
    name: Build previous k0s
    runs-on: embedded-cluster-2
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Install dagger
        run: |
          curl -fsSL https://dl.dagger.io/dagger/install.sh | sh
          sudo mv ./bin/dagger /usr/local/bin/dagger

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-install
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
          MANGLE_METADATA: "1"
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          export K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION)
          export K0S_GO_VERSION=$(make print-PREVIOUS_K0S_GO_VERSION)
          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-previous-k0s
          # avoid rate limiting
          export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2)

          ./scripts/build-and-release.sh
          cp output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: previous-k0s-release
          path: |
            output/bin/embedded-cluster-previous-k0s

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-PREVIOUS_K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  find-previous-stable:
    name: Determine previous stable version
    runs-on: ubuntu-latest
    needs:
      - git-sha
    outputs:
      ec_version: ${{ steps.export.outputs.ec_version }}
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Export k0s version
        id: export
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          k0s_majmin_version="$(make print-PREVIOUS_K0S_VERSION | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')"
          if [ "$k0s_majmin_version" == "1.28" ]; then
            k0s_majmin_version="1.29"
          fi
          EC_VERSION="$(gh release list --repo replicatedhq/embedded-cluster \
            --exclude-drafts --exclude-pre-releases --json name \
            --jq '.[] | .name' \
            | grep "k8s-${k0s_majmin_version}" \
            | head -n1)"

          gh release download "$EC_VERSION" --repo replicatedhq/embedded-cluster --pattern 'metadata.json'
          K0S_VERSION="$(jq -r '.Versions.Kubernetes' metadata.json)"

          echo "EC_VERSION=\"$EC_VERSION\""
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "ec_version=$EC_VERSION" >> "$GITHUB_OUTPUT"
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  build-upgrade:
    name: Build upgrade
    runs-on: embedded-cluster-2
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Install dagger
        run: |
          curl -fsSL https://dl.dagger.io/dagger/install.sh | sh
          sudo mv ./bin/dagger /usr/local/bin/dagger

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-upgrade
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
          MANGLE_METADATA: "1"
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          export K0S_VERSION=$(make print-K0S_VERSION)
          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-upgrade
          # avoid rate limiting
          export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2)

          ./scripts/build-and-release.sh
          cp output/bin/embedded-cluster output/bin/embedded-cluster-upgrade

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: upgrade-release
          path: |
            output/bin/embedded-cluster-upgrade

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  check-images:
    name: Check images
    runs-on: ubuntu-latest
    needs:
      - buildtools
      - build-current
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download buildtools artifact
        uses: actions/download-artifact@v4
        with:
          name: buildtools
          path: output/bin
      - name: Download embedded-cluster artifact
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin
      - name: Check for missing images
        run: |
          chmod +x ./output/bin/buildtools
          chmod +x ./output/bin/embedded-cluster-original
          ./output/bin/embedded-cluster-original version metadata > version-metadata.json
          ./output/bin/embedded-cluster-original version list-images > expected.txt
          printf "Expected images:\n$(cat expected.txt)\n"
          ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt
          printf "Found images:\n$(cat images.txt)\n"
          missing_images=0
          while read img; do
            grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; }
          done <images.txt
          if [ $missing_images -gt 0 ]; then
            echo "Found $missing_images missing images"
            exit 1
          fi

  export-version-specifier:
    name: Export version specifier
    runs-on: ubuntu-latest
    needs:
      - git-sha
    outputs:
      version_specifier: ${{ steps.export-version-specifier.outputs.version_specifier }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Export version specifier
        id: export-version-specifier
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          echo "version_specifier=$SHORT_SHA" >> "$GITHUB_OUTPUT"

  release-app:
    name: Create app releases
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    needs:
      - git-sha
      - build-current
      - build-legacydr
      - build-previous-k0s
      - build-upgrade
      - find-previous-stable
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install replicated CLI
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          gh release download --repo replicatedhq/replicated --pattern '*linux_amd64.tar.gz' --output replicated.tar.gz
          tar xf replicated.tar.gz replicated && rm replicated.tar.gz
          mv replicated /usr/local/bin/replicated
      - name: Create CI releases
        env:
          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
          APP_CHANNEL: CI
          USES_DEV_BUCKET: "0"
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}

          # re-promote a release containing an old version of embedded-cluster to test upgrades
          export APP_VERSION="appver-${SHORT_SHA}-pre-minio-removal"
          replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"

          # re-promote a release containing an old version of embedded-cluster to test upgrades
          export APP_VERSION="appver-${SHORT_SHA}-1.8.0-k8s-1.28"
          replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
          replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}"

          # promote a release containing the previous stable version of embedded-cluster to test upgrades
          export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}"
          export APP_VERSION="appver-${SHORT_SHA}-previous-stable"
          export RELEASE_YAML_DIR=e2e/kots-release-install-stable
          ./scripts/ci-release-app.sh

          # install the previous k0s version to ensure an upgrade occurs
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s"
          export APP_VERSION="appver-${SHORT_SHA}-previous-k0s"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # then install the current k0s version
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh
          
          # then install a version with alternate unsupported overrides
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}-unsupported-overrides"
          export RELEASE_YAML_DIR=e2e/kots-release-unsupported-overrides
          ./scripts/ci-release-app.sh
          
          # then install a version with additional failing host preflights
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}-failing-preflights"
          export RELEASE_YAML_DIR=e2e/kots-release-install-failing-preflights
          ./scripts/ci-release-app.sh
          
          # then install a version with additional warning host preflights
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}-warning-preflights"
          export RELEASE_YAML_DIR=e2e/kots-release-install-warning-preflights
          ./scripts/ci-release-app.sh

          # promote a release with improved dr support
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-legacydr"
          export APP_VERSION="appver-${SHORT_SHA}-legacydr"
          export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr
          ./scripts/ci-release-app.sh

          # then a noop upgrade
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}-noop"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # and finally an app upgrade
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade"
          export APP_VERSION="appver-${SHORT_SHA}-upgrade"
          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
          ./scripts/ci-release-app.sh

      - name: Create airgap releases
        env:
          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
          APP_CHANNEL: CI-airgap
          USES_DEV_BUCKET: "0"
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}

          # promote a release containing the previous stable version of embedded-cluster to test upgrades
          export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}"
          export APP_VERSION="appver-${SHORT_SHA}-previous-stable"
          export RELEASE_YAML_DIR=e2e/kots-release-install-stable
          ./scripts/ci-release-app.sh

          # install the previous k0s version to ensure an upgrade occurs
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s"
          export APP_VERSION="appver-${SHORT_SHA}-previous-k0s"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # then install the current k0s version
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # promote a release with improved dr support
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-legacydr"
          export APP_VERSION="appver-${SHORT_SHA}-legacydr"
          export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr
          ./scripts/ci-release-app.sh

          # then a noop upgrade
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}-noop"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # and finally an app upgrade
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-upgrade"
          export APP_VERSION="appver-${SHORT_SHA}-upgrade"
          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
          ./scripts/ci-release-app.sh

      - name: Create download link message text
        if: github.event_name == 'pull_request'
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}"

          echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt
          echo "" >> download-link.txt
          echo "Online Installer:" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "curl \"https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci/${APP_VERSION}\" -H \"Authorization: \$EC_SMOKE_TEST_LICENSE_ID\" -o embedded-cluster-smoke-test-staging-app-ci.tgz" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "Airgap Installer (may take a few minutes before the airgap bundle is built):" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "curl \"https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci-airgap/${APP_VERSION}?airgap=true\" -H \"Authorization: \$EC_SMOKE_TEST_LICENSE_ID\" -o embedded-cluster-smoke-test-staging-app-ci.tgz" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "Happy debugging!" >> download-link.txt
          cat download-link.txt

      - name: Comment download link
        if: github.event_name == 'pull_request'
        uses: mshick/add-pr-comment@v2
        with:
          message-path: download-link.txt

  # e2e-docker runs the e2e tests inside a docker container rather than a full VM
  e2e-docker:
    name: E2E docker # this name is used by .github/workflows/automated-prs-manager.yaml
    runs-on: ubuntu-latest
    needs:
      - git-sha
      - build-current
      - build-legacydr
      - build-previous-k0s
      - build-upgrade
      - find-previous-stable
      - release-app
      - export-version-specifier
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestPreflights
          - TestMaterialize
          - TestHostPreflightCustomSpec
          - TestHostPreflightInBuiltSpec
          - TestSingleNodeInstallation
          - TestSingleNodeInstallationAlmaLinux8
          - TestSingleNodeInstallationDebian11
          - TestSingleNodeInstallationDebian12
          - TestSingleNodeInstallationCentos9Stream
          - TestSingleNodeUpgradePreviousStable
          - TestInstallFromReplicatedApp
          - TestUpgradeFromReplicatedApp
          - TestUpgradeEC18FromReplicatedApp
          - TestResetAndReinstall
          - TestOldVersionUpgrade
          - TestInstallSnapshotFromReplicatedApp
          - TestMultiNodeInstallation
          - TestMultiNodeHAInstallation
          - TestSingleNodeDisasterRecovery
          - TestSingleNodeLegacyDisasterRecovery
          - TestSingleNodeResumeDisasterRecovery
          - TestMultiNodeHADisasterRecovery
          - TestSingleNodeInstallationNoopUpgrade
          - TestCustomCIDR
          - TestLocalArtifactMirror
          - TestMultiNodeReset
          - TestCollectSupportBundle
          - TestUnsupportedOverrides
          - TestHostCollectSupportBundleInCluster
          - TestInstallWithConfigValues
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download binary
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin
      - name: Download new DR binary
        uses: actions/download-artifact@v4
        with:
          name: legacydr-release
          path: output/bin
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Login to DockerHub to avoid rate limiting
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}
      - name: Free up runner disk space
        uses: ./.github/actions/free-disk-space
      - name: Write license files
        run: |
          echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml
          echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}" | base64 --decode > e2e/snapshot-license.yaml
      - name: Run test
        env:
          SHORT_SHA: dev-${{ needs.git-sha.outputs.git_sha }}
          LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
          AIRGAP_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
          SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
          AIRGAP_SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
          DR_AWS_S3_ENDPOINT: https://s3.amazonaws.com
          DR_AWS_S3_REGION: us-east-1
          DR_AWS_S3_BUCKET: kots-testim-snapshots
          DR_AWS_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
          DR_AWS_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
          DR_AWS_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
          DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
          EXPECT_K0S_VERSION: ${{ needs.build-current.outputs.k0s_version }}
          EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.build-previous-k0s.outputs.k0s_version }}
          EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }}
        run: |
          make e2e-test TEST_NAME=${{ matrix.test }}
      - name: Troubleshoot
        if: ${{ !cancelled() }}
        uses: ./.github/actions/e2e-troubleshoot
        with:
          test-name: '${{ matrix.test }}'

  e2e:
    name: E2E # this name is used by .github/workflows/automated-prs-manager.yaml
    runs-on: ${{ matrix.runner || 'ubuntu-22.04' }}
    needs:
      - build-current
      - build-legacydr
      - build-previous-k0s
      - build-upgrade
      - find-previous-stable
      - release-app
      - export-version-specifier
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestVersion
          - TestCommandsRequireSudo
          - TestResetAndReinstallAirgap
          - TestSingleNodeAirgapUpgrade
          - TestSingleNodeAirgapUpgradeConfigValues
          - TestSingleNodeAirgapUpgradeCustomCIDR
          - TestSingleNodeDisasterRecoveryWithProxy
          - TestProxiedEnvironment
          - TestProxiedCustomCIDR
          - TestInstallWithPrivateCAs
          - TestInstallWithMITMProxy
        include:
          - test: TestMultiNodeAirgapUpgrade
            runner: embedded-cluster-2
          - test: TestMultiNodeAirgapUpgradeSameK0s
            runner: embedded-cluster-2
          - test: TestMultiNodeAirgapUpgradePreviousStable
            runner: embedded-cluster-2
          - test: TestAirgapUpgradeFromEC18
            runner: embedded-cluster-2
          - test: TestSingleNodeAirgapDisasterRecovery
            runner: embedded-cluster-2
          - test: TestMultiNodeAirgapHAInstallation
            runner: embedded-cluster-2
          - test: TestMultiNodeAirgapHADisasterRecovery
            runner: embedded-cluster-2
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download current binary
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin

      - uses: ./.github/actions/e2e
        with:
          test-name: '${{ matrix.test }}'
          is-large-runner: ${{ matrix.runner == 'embedded-cluster-2' }}
          airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
          snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
          snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}
          airgap-snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
          license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
          license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
          dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
          dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
          k0s-version: ${{ needs.build-current.outputs.k0s_version }}
          k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }}
          k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }}
          version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }}

  e2e-main:
    name: E2E (on merge)
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    runs-on: ${{ matrix.runner || 'ubuntu-22.04' }}
    needs:
      - build-current
      - build-legacydr
      - build-previous-k0s
      - build-upgrade
      - find-previous-stable
      - release-app
      - export-version-specifier
    strategy:
      fail-fast: false
      matrix:
        include:
          - test: TestFiveNodesAirgapUpgrade
            runner: embedded-cluster-2
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download current binary
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin

      - uses: ./.github/actions/e2e
        with:
          test-name: '${{ matrix.test }}'
          is-large-runner: ${{ matrix.runner == 'embedded-cluster-2' }}
          airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
          snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
          snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}
          airgap-snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
          license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
          license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
          dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
          dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
          k0s-version: ${{ needs.build-current.outputs.k0s_version }}
          k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }}
          k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }}
          version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }}

  # this job will validate that all the tests passed
  # it is used for the github branch protection rule
  validate-success:
    name: Validate success # this name is used by .github/workflows/automated-prs-manager.yaml
    runs-on: ubuntu-20.04
    needs:
      - e2e
      - e2e-main
      - e2e-docker
      - sanitize
      - test
      - int-tests
      - dryrun-tests
      - check-images
      - check-operator-crds
    if: always()
    steps:
      # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
      - name: fail if e2e job was not successful
        if: needs.e2e.result != 'success'
        run: exit 1
      - name: fail if e2e-main job was not successful
        if: needs.e2e-main.result != 'success' && needs.e2e-main.result != 'skipped'
        run: exit 1
      - name: fail if e2e-docker job was not successful
        if: needs.e2e-docker.result != 'success'
        run: exit 1
      - name: fail if sanitize job was not successful
        if: needs.sanitize.result != 'success'
        run: exit 1
      - name: fail if test job was not successful
        if: needs.test.result != 'success'
        run: exit 1
      - name: fail if check-images job was not successful
        if: needs.check-images.result != 'success'
        run: exit 1
      - name: fail if check-operator-crds job was not successful
        if: needs.check-operator-crds.result != 'success'
        run: exit 1
      - name: fail if e2e-docker-install2 job was not successful
        if: needs.e2e-docker-install2.result != 'success'
        run: exit 1
      - name: fail if e2e-docker-install2-failing job WAS successful
        if: needs.e2e-docker-install2-failing.result == 'success'
        run: exit 1
      - name: fail if e2e-install2 job was not successful
        if: needs.e2e-install2.result != 'success'
        run: exit 1
      - name: fail if e2e-install2-failing job WAS successful
        if: needs.e2e-install2-failing.result == 'success'
        run: exit 1
      - name: succeed if everything else passed
        run: echo "Validation succeeded"