dapr-test.yml

#
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Required secrets:
# - AZURE_CREDENTIALS: JSON object containing the Azure service principal credentials. Docs: https://github.com/Azure/login#configure-a-service-principal-with-a-secret
# - DAPR_BOT_TOKEN: Token for the Dapr bot
#
# Optional secrets:
# - AZURE_DIAG_LOG_ANALYTICS_WORKSPACE_ID: Resource ID of the Log Analytics Workspace where to store certain diagnostic logs (e.g. `/subscriptions/<subscription>/resourcegroups/<resource group>/providers/Microsoft.OperationalInsights/workspaces/<workspace name>`)
# - AZURE_DIAG_STORAGE_ID: Resource ID of the Azure Storage account where to store certain diagnostic logs (e.g. `/subscriptions/<subscription>/resourcegroups/<resource group>/providers/Microsoft.Storage/storageAccounts/<storage account name>`)

name: dapr-test

on:
  # Run every 2 hours
  schedule:
    - cron: '0 */2 * * *'
  # Manual trigger
  workflow_dispatch:
  # Dispatch on external events
  repository_dispatch:
    types: [e2e-test]

env:
  # Version of Go
  GOVER: "1.18"
  # Configure proxy for Go modules
  GOPROXY: https://proxy.golang.org
  # Version of kubectl
  KUBECTLVER: "v1.22.6"
  # If not empty, uses cloud resources for testing
  TEST_CLOUD_ENV: "azure"
  # Version of Helm
  HELMVER: "v3.7.2"
  # Kubernetes namespace to use
  DAPR_NAMESPACE: "dapr-tests"
  # Timeout for tests
  MAX_TEST_TIMEOUT: 5400
  # Enable HA mode for tests
  HA_MODE: true
  # Space-separated of supported Azure regions: one will be picked randomly for each cluster
  AZURE_REGIONS: "westus3"
  # Container registry where to cache e2e test images
  DAPR_CACHE_REGISTRY: "dapre2eacr.azurecr.io"

jobs:
  deploy-infrastructure:
    name: Deploy test infrastructure
    runs-on: ubuntu-latest
    steps:
      - name: Set up for scheduled test
        if: github.event_name != 'repository_dispatch'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/master" >> $GITHUB_ENV
        shell: bash
      - name: Parse test payload
        if: github.event_name == 'repository_dispatch'
        uses: actions/github-script@0.3.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            if (testPayload && testPayload.command == "ok-to-test") {
              var fs = require('fs');
              // Set environment variables
              fs.appendFileSync(process.env.GITHUB_ENV,
                `CHECKOUT_REPO=${testPayload.pull_head_repo}\n`+
                `CHECKOUT_REF=${testPayload.pull_head_ref}`);
            }
      - name: Check out code
        if: env.CHECKOUT_REPO != ''
        uses: actions/checkout@v2
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Login to Azure
        if: env.CHECKOUT_REPO != ''
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Build test prefix
        if: env.CHECKOUT_REPO != ''
        run: |
          BASE_STR="E2E|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          TEST_PREFIX="dapre2e${SUFFIX}"
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Deploy the test cluster
        if: env.TEST_PREFIX != ''
        run: |
          # Select two random Azure regions
          REGIONS=(${{ env.AZURE_REGIONS }})
          REGIONS_SIZE=${#REGIONS[@]}
          REGIONS_IDX=$(($RANDOM % $REGIONS_SIZE))
          REGION1=${REGIONS[$REGIONS_IDX]}
          REGIONS_IDX=$(($RANDOM % $REGIONS_SIZE))
          REGION2=${REGIONS[$REGIONS_IDX]}
          echo "Deploying to Azure regions: Linux=${REGION1} Windows=${REGION2}"

          # Tags
          DATE_TAG=$(date --iso-8601=seconds)
          echo "Tags: date=${DATE_TAG}"

          # Deploy both Linux and Windows cluster
          # Retry the deployment twice in case of transient failures (such as capacity constraints)
          success=false
          for i in 1 2 3; do 
            az deployment sub create \
              --name "${{ env.TEST_PREFIX }}" \
              --location ${REGION1} \
              --template-file ./tests/test-infra/azure-all.bicep \
              --parameters \
                namePrefix="${{ env.TEST_PREFIX }}" \
                location1=${REGION1} \
                location2=${REGION2} \
                dateTag="${DATE_TAG}" \
                diagLogAnalyticsWorkspaceResourceId="${{ secrets.AZURE_DIAG_LOG_ANALYTICS_WORKSPACE_ID }}" \
                diagStorageResourceId="${{ secrets.AZURE_DIAG_STORAGE_ID }}" \
              && success=true \
              && break \
              || sleep 120
          done
          # Exit with error if failed
          $success || exit 1
        shell: bash

  build:
    name: Build for  ${{ matrix.target_os }}
    runs-on: ${{ matrix.os }}
    env:
      GOOS: ${{ matrix.target_os }}
      GOARCH: ${{ matrix.target_arch }}
      TARGET_OS: ${{ matrix.target_os }}
      TARGET_ARCH: ${{ matrix.target_arch }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-2019]
        target_arch: [amd64]
        include:
          - os: ubuntu-latest
            target_os: linux
          - os: windows-2019
            target_os: windows
    steps:
      - name: Set up for scheduled test
        if: github.event_name != 'repository_dispatch'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/master" >> $GITHUB_ENV
        shell: bash
      - name: Parse test payload
        if: github.event_name == 'repository_dispatch'
        uses: actions/github-script@0.3.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            if (testPayload && testPayload.command == "ok-to-test") {
              var fs = require('fs');
              // Set environment variables
              fs.appendFileSync(process.env.GITHUB_ENV,
                `CHECKOUT_REPO=${testPayload.pull_head_repo}\n`+
                `CHECKOUT_REF=${testPayload.pull_head_ref}`);
            }
      # In windows-2019 images, WSL comes with bash.exe (but no distribution) and that causes issues
      # See: https://github.community/t/wsl-not-available-for-hosted-windows-machine/124389
      - name: Remove bash.exe from WSL
        if: runner.os == 'Windows'
        run: |
          rm.exe "C:/WINDOWS/system32/bash.EXE"
      - name: Set up Go ${{ env.GOVER }}
        if: env.CHECKOUT_REPO != ''
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GOVER }}
      - name: Check out code
        if: env.CHECKOUT_REPO != ''
        uses: actions/checkout@v2
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Cache Go modules (Linux)
        if: env.CHECKOUT_REPO != '' && runner.os == 'Linux'
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-build-
      - name: Cache Go modules (Windows)
        if: env.CHECKOUT_REPO != '' && runner.os == 'Windows'
        uses: actions/cache@v3
        with:
          path: |
            ~\AppData\Local\go-build
            ~\go\pkg\mod
          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-build-
      - name: Login to Azure
        if: env.CHECKOUT_REPO != ''
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Login to cache registry
        if: env.CHECKOUT_REPO != '' && env.DAPR_CACHE_REGISTRY != ''
        run: |
          az acr login --name ${{ env.DAPR_CACHE_REGISTRY }}
        shell: bash
      - name: Build test prefix and set env vars
        if: env.CHECKOUT_REPO != ''
        run: |
          BASE_STR="E2E|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          TEST_PREFIX=""
          if [ "${{ env.TARGET_OS }}" == "windows" ] ; then
            TEST_PREFIX="dapre2e${SUFFIX}w"
          else
            TEST_PREFIX="dapre2e${SUFFIX}l"
          fi
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_REGISTRY=${TEST_PREFIX}acr.azurecr.io" >> $GITHUB_ENV
          echo "TEST_CLUSTER=${TEST_PREFIX}-aks" >> $GITHUB_ENV
          echo "DAPR_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_TEST_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "TEST_RESOURCE_GROUP=Dapr-E2E-${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Build dapr and its docker image
        if: env.TEST_PREFIX != ''
        run: |
          make build

          if [ "${{ env.TARGET_OS }}" = "linux" ] && [ "${{ env.TARGET_ARCH }}" != "arm" ]; then
            # For Linux, we use images based on Mariner
            DOCKERFILE=Dockerfile-mariner make docker-build
          else
            make docker-build
          fi
        shell: bash
      - name: Build e2e test apps
        if: env.TEST_PREFIX != ''
        run: |
          make build-e2e-app-all
        shell: bash
      - name: Wait for Azure Container Registry deployment
        if: env.TEST_PREFIX != ''
        run: |
          elapsed=0
          until [ $elapsed -ge 1800 ] || az acr show --name ${{ env.TEST_PREFIX }}acr --query "id"
          do
              echo "Azure Container Registry not ready yet: sleeping for 20 seconds"
              sleep 20
              elapsed=$(expr $elapsed + 20)
          done

          if [ $elapsed -ge 1800 ]; then
            echo "Azure Container Registry not ready on time"
            exit 1
          fi
        shell: bash
      - name: Login to Azure Container Registry
        if: env.TEST_PREFIX != ''
        run: |
          az acr login --name ${{ env.TEST_PREFIX }}acr
        shell: bash
      - name: Push all container images to the registry
        if: env.TEST_PREFIX != ''
        run: |
          if [ "${{ env.TARGET_OS }}" = "linux" ] && [ "${{ env.TARGET_ARCH }}" != "arm" ]; then
            # For Linux, we use images based on Mariner
            DOCKERFILE=Dockerfile-mariner make docker-push
          else
            make docker-push
          fi
          make push-e2e-app-all
        shell: bash

  test-e2e:
    name: End-to-end ${{ matrix.target_os }} tests
    needs:
      - build
      - deploy-infrastructure
    # Always run on Linux as the local OS is irrelevant and this is faster
    runs-on: ubuntu-latest
    env:
      TARGET_OS: ${{ matrix.target_os }}
      TARGET_ARCH: ${{ matrix.target_arch }}
      TEST_OUTPUT_FILE_PREFIX: "test_report"
      PULL_POLICY: IfNotPresent
    strategy:
      fail-fast: false
      matrix:
        target_os: [linux, windows]
        target_arch: [amd64]
    steps:
      - name: Set up log paths
        run: |
          echo "DAPR_CONTAINER_LOG_PATH=$GITHUB_WORKSPACE/container_logs/${{ matrix.target_os }}_${{ matrix.target_arch }}" | sed 's/\\/\//g' >> $GITHUB_ENV
          echo "DAPR_TEST_LOG_PATH=$GITHUB_WORKSPACE/test_logs/${{ matrix.target_os }}_${{ matrix.target_arch }}" | sed 's/\\/\//g' >> $GITHUB_ENV
        shell: bash
      - name: Set up for scheduled test
        if: github.event_name != 'repository_dispatch'
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/master" >> $GITHUB_ENV
        shell: bash
      - name: Parse test payload
        if: github.event_name == 'repository_dispatch'
        uses: actions/github-script@0.3.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            if (testPayload && testPayload.command == "ok-to-test") {
              var fs = require('fs');
              // Set environment variables
              fs.appendFileSync(process.env.GITHUB_ENV,
                `CHECKOUT_REPO=${testPayload.pull_head_repo}\n`+
                `CHECKOUT_REF=${testPayload.pull_head_ref}`);
            }
      - name: Set up Go ${{ env.GOVER }}
        if: env.CHECKOUT_REPO != ''
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GOVER }}
      - name: Check out code
        if: env.CHECKOUT_REPO != ''
        uses: actions/checkout@v2
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Cache Go modules (test)
        if: env.CHECKOUT_REPO != '' && runner.os == 'Linux'
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-tests-
      - uses: azure/setup-kubectl@v1
        with:
          version: ${{ env.KUBECTLVER }}
        id: install
      - name: Set up Helm ${{ env.HELMVER }}
        uses: azure/setup-helm@v1
        with:
          version: ${{ env.HELMVER }}
      - name: Login to Azure
        if: env.CHECKOUT_REPO != ''
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Build test prefix and set env vars
        if: env.CHECKOUT_REPO != ''
        run: |
          BASE_STR="E2E|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          TEST_PREFIX=""
          if [ "${{ env.TARGET_OS }}" == "windows" ] ; then
            TEST_PREFIX="dapre2e${SUFFIX}w"
          else
            TEST_PREFIX="dapre2e${SUFFIX}l"
          fi
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_REGISTRY=${TEST_PREFIX}acr.azurecr.io" >> $GITHUB_ENV
          echo "TEST_CLUSTER=${TEST_PREFIX}-aks" >> $GITHUB_ENV
          echo "DAPR_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "DAPR_TEST_TAG=${TEST_PREFIX}" >> $GITHUB_ENV
          echo "TEST_RESOURCE_GROUP=Dapr-E2E-${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Connect to Kubernetes
        if: env.TEST_PREFIX != ''
        run: |
          az aks get-credentials -n "${{ env.TEST_CLUSTER }}" -g "${{ env.TEST_RESOURCE_GROUP }}"
          kubectl create namespace ${{ env.DAPR_NAMESPACE }}
        shell: bash
      - name: Setup for cloud resources
        if: env.TEST_PREFIX != '' && env.TEST_CLOUD_ENV != ''
        run: ./tests/test-infra/setup_${{ env.TEST_CLOUD_ENV }}.sh
        shell: bash
      - name: Preparing AKS cluster for test
        if: env.TEST_PREFIX != ''
        run: |
          make setup-helm-init
          make setup-test-env-redis
          make setup-test-env-kafka
          if [ "${{ env.DAPR_TEST_STATE_STORE }}" == "" ] ; then
            make setup-test-env-mongodb
          fi
          kubectl get pods -n ${{ env.DAPR_NAMESPACE }}
        shell: bash
      - name: Deploy dapr to AKS cluster
        if: env.TEST_PREFIX != ''
        env:
          ADDITIONAL_HELM_SET: "dapr_operator.logLevel=debug,dapr_operator.watchInterval=20s"
        run: make docker-deploy-k8s
      - name: Deploy test components
        if: env.TEST_PREFIX != ''
        run: make setup-test-components
      - name: Show dapr configurations
        if: env.TEST_PREFIX != ''
        run: kubectl get configurations daprsystem -n ${{ env.DAPR_NAMESPACE }} -o yaml
      - name: Run E2E tests
        if: env.TEST_PREFIX != ''
        run: make test-e2e-all
      - name: Save control plane K8s resources
        if: always() && env.TEST_PREFIX != ''
        run: |
          make save-dapr-control-plane-k8s-resources || true
      - name: Save control plane logs
        if: always() && env.TEST_PREFIX != ''
        run: |
          make save-dapr-control-plane-k8s-logs
      # Container log files can be bigger than the maximum file size allowed by GitHub
      - name: Compress logs
        if: always()
        run: |
          gzip --fast -r ${{ env.DAPR_CONTAINER_LOG_PATH }}
          gzip --fast -r ${{ env.DAPR_TEST_LOG_PATH }}
        shell: bash
      - name: Upload container logs
        if: always()
        uses: actions/upload-artifact@master
        with:
          name: ${{ matrix.target_os }}_${{ matrix.target_arch }}_container_logs
          path: ${{ env.DAPR_CONTAINER_LOG_PATH }}
      - name: Upload test logs
        if: always()
        uses: actions/upload-artifact@master
        with:
          name: ${{ matrix.target_os }}_${{ matrix.target_arch }}_test_logs
          path: ${{ env.DAPR_TEST_LOG_PATH }}
      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@master
        with:
          #TODO: .json suffix can be removed from artifact name after test analytics scripts are updated
          name: ${{ matrix.target_os }}_${{ matrix.target_arch }}_e2e.json
          path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_e2e.*
      - name: Add test result comment to PR
        if: always() && github.event_name == 'repository_dispatch' && env.TEST_PREFIX != ''
        env:
          JOB_CONTEXT: ${{ toJson(job) }}
        uses: actions/github-script@0.3.0
        with:
          github-token: ${{secrets.DAPR_BOT_TOKEN}}
          script: |
            const testPayload = context.payload.client_payload;
            const jobContext = JSON.parse(process.env.JOB_CONTEXT);
            const jobStatus = jobContext.status.toLowerCase();
            const targetOs = process.env.TARGET_OS;

            console.log(`Current Job Status: ${jobStatus}`);

            var message = "";

            const checkLink = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;

            if (jobStatus == "cancelled") {
              message = `End-to-end tests cancelled on ${targetOs}. Please check the [build logs](${checkLink})`;
            } else if (jobStatus == "success") {
              message = `Congrats! All [end-to-end tests](${checkLink}) have passed on ${targetOs}. Thanks for your contribution!`;
            } else if (jobStatus == "failure") {
              message = `End-to-end tests failed on ${targetOs}. Please check the [build logs](${checkLink})`;
            }

            if (message) {
              await github.issues.createComment({
                owner: testPayload.issue.owner,
                repo: testPayload.issue.repo,
                issue_number: testPayload.issue.number,
                body: message
              });
            }

  cleanup:
    name: Clean up Azure resources
    runs-on: ubuntu-latest
    needs:
      - test-e2e
    if: always()
    steps:
      - name: Login to Azure
        uses: azure/login@v1
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}
      - name: Build test prefix
        run: |
          BASE_STR="E2E|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          SUFFIX=$(echo $BASE_STR | sha1sum | head -c 10)
          TEST_PREFIX="dapre2e${SUFFIX}"
          echo "Test prefix is ${TEST_PREFIX}"
          echo "TEST_PREFIX=${TEST_PREFIX}" >> $GITHUB_ENV
        shell: bash
      - name: Delete Linux cluster
        run: |
          # We are not waiting for these commands to complete, and we're ignoring errors
          echo "Starting removal of resource group Dapr-E2E-${{ env.TEST_PREFIX }}l"
          az group delete --no-wait --yes --name "Dapr-E2E-${{ env.TEST_PREFIX }}l" || true
        shell: bash
      - name: Delete Windows cluster
        run: |
          # We are not waiting for these commands to complete, and we're ignoring errors
          echo "Starting removal of resource group Dapr-E2E-${{ env.TEST_PREFIX }}w"
          az group delete --no-wait --yes --name "Dapr-E2E-${{ env.TEST_PREFIX }}w" || true
        shell: bash