Remove Azure eshop from PR tests (#547)

* Update semver checking

* Add condition to Azure test

* Add conditional logic

* Update conditional logic

* Explicitly disable Dapr

* Add explicit check

* Remove Dapr check

* Move Dapr logic to setup step

.github/scripts/validate_semver.py

@@ -29,12 +29,13 @@ def main():
     SEMVER_REGEX = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
 
     pattern = re.compile(SEMVER_REGEX)
-
     version = sys.argv[1]
     match = pattern.search(version)
-
-    # If no match, then return an error (provided version is not valid semver)
-    if match is None:
+
+    if version == "edge":
+        print("Provided version is edge")
+        sys.exit(0)
+    elif match is None:
         print("Provided version is not valid semver")
         sys.exit(1)
     else:

.github/workflows/test.yaml

@@ -1,12 +1,12 @@
-name: Test Quickstarts
+name: Test Samples
 
 on:
   workflow_dispatch:
     inputs:
       version:
-        description: 'Radius version number to use (e.g. 0.22.0, 0.23.0-rc1)'
+        description: 'Radius version number to use (e.g. 0.22.0, 0.23.0-rc1, edge). Defaults to edge.'
         required: false
-        default: ''
+        default: 'edge'
         type: string
   push:
     branches:
@@ -15,6 +15,7 @@ on:
     paths:
       - "quickstarts/**"
       - "reference-apps/**"
+      - "demo/**"
       - ".github/workflows/**"
   pull_request:
     types: [opened, synchronize, reopened]
@@ -28,72 +29,102 @@ env:
 jobs:
   test:
     name: Sample tests
-    if: github.event.action != 'closed'
     runs-on: [self-hosted, 1ES.Pool=1ES-Radius-Samples]
     strategy:
       fail-fast: false
       matrix:
         include:
           - name: demo
+            runOnPullRequest: true
             app: demo
             path: ./demo/app.bicep
             args: --application demo
             uiTestFile: tests/demo.app.spec.ts
             port: 3000
             container: demo
+            enableDapr: false
           - name: dapr
+            runOnPullRequest: true
             app: dapr-quickstart
             path: ./quickstarts/dapr/dapr.bicep
             enableDapr: true
           - name: environment-variables
+            runOnPullRequest: true
             app: myapp
             path: ./quickstarts/environment-variables/app.bicep
+            enableDapr: false
           - name: volumes
+            runOnPullRequest: true
             app: myapp
             path: ./quickstarts/volumes/app.bicep
+            enableDapr: false
           - name: eshop
+            runOnPullRequest: true
             app: eshop
             path: ./reference-apps/eshop/iac/eshop.bicep
             args: --application eshop
             uiTestFile: tests/eshop/container.app.spec.ts
+            enableDapr: false
           - name: eshop-azure
+            runOnPullRequest: false
             app: eshop
             path: ./reference-apps/eshop/iac/eshop.bicep
             args: --application eshop -p platform=azure
             uiTestFile: tests/eshop/container.app.spec.ts
             credential: azure
+            enableDapr: false
     env:
       BRANCH: ${{ github.base_ref || github.ref_name }}
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       AZURE_LOCATION: westus3
     steps:
+      # Setup the test assets and configuration
       - name: Generate output variables
         id: gen-id
         run: |
           BASE_STR="SAMPLES|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
           UNIQUE_ID=$(echo $BASE_STR | sha1sum | head -c 10)
+          
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ matrix.runOnPullRequest }}" == "false" ]]; then
+            RUN_TEST=false
+          else
+            RUN_TEST=true
+          fi
+
+          if [[ "${{ matrix.enableDapr }}" == "true" ]]; then
+            ENABLE_DAPR=true
+          else
+            ENABLE_DAPR=false
+          fi
 
           # Set output variables to be used in the other jobs
           echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT
           echo "TEST_RESOURCE_GROUP_PREFIX=samplestest-${UNIQUE_ID}" >> $GITHUB_OUTPUT
+          echo "RUN_TEST=${RUN_TEST}" >> $GITHUB_OUTPUT
+          echo "ENABLE_DAPR=${ENABLE_DAPR}" >> $GITHUB_OUTPUT
       - name: Checkout code
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         uses: actions/checkout@v3
       - name: Ensure inputs.version is valid semver
-        if: inputs.version != ''
+        if: inputs.version != '' && steps.gen-id.outputs.RUN_TEST == 'true'
         run: |
           python ./.github/scripts/validate_semver.py ${{ inputs.version }}
       - name: Setup Node
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         uses: actions/setup-node@v3
         with:
           node-version: 16
       - name: az CLI login
+        if: matrix.credential == 'azure' && steps.gen-id.outputs.RUN_TEST == 'true'
         run: |
           az login --service-principal \
             --username ${{ secrets.AZURE_SP_TESTS_APPID }} \
             --password ${{ secrets.AZURE_SP_TESTS_PASSWORD }} \
             --tenant ${{ secrets.AZURE_SP_TESTS_TENANTID }}
+      # Create and install test environment
       - name: Create Azure resource group
-        if: matrix.credential == 'azure'
+        id: create-azure-resource-group
+        if: matrix.credential == 'azure' && steps.gen-id.outputs.RUN_TEST == 'true'
         env:
           RESOURCE_GROUP: ${{ steps.gen-id.outputs.TEST_RESOURCE_GROUP_PREFIX }}-${{ matrix.name }}
           SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
@@ -109,16 +140,15 @@ jobs:
             sleep 5
           done
       - name: Download k3d
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         run: wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
       - name: Download rad CLI
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         run: |
           for attempt in 1 2 3 4 5; do
             if [[ -n "${{ inputs.version }}" ]]; then
               echo "Downloading rad CLI version ${{ inputs.version }}"
               wget -q "${{ env.RAD_CLI_URL }}" -O - | /bin/bash -s ${{ inputs.version }}
-            elif [ "$BRANCH" = "edge" ]; then
-              echo "Downloading edge rad CLI"
-              wget -q "${{ env.RAD_CLI_URL }}" -O - | /bin/bash -s edge
             else
               echo "Downloading latest rad CLI"
               wget -q "${{ env.RAD_CLI_URL }}" -O - | /bin/bash
@@ -129,13 +159,15 @@ jobs:
             fi
           done
       - name: Create k3d cluster
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         run: k3d cluster create --agents 2 -p "80:80@loadbalancer" --k3s-arg "--disable=traefik@server:0"
       - name: Install Dapr
-        if: ${{ matrix.enableDapr }}
+        if: steps.gen-id.outputs.RUN_TEST == 'true' && steps.gen-id.outputs.ENABLE_DAPR == 'true'
         run: |
           helm repo add dapr https://dapr.github.io/helm-charts/
           helm install dapr dapr/dapr --version=1.6 --namespace dapr-system --create-namespace --wait
       - name: Init local environment
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         env:
           RESOURCE_GROUP: ${{ steps.gen-id.outputs.TEST_RESOURCE_GROUP_PREFIX }}-${{ matrix.name }}
           SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
@@ -152,16 +184,19 @@ jobs:
             rad env update default --azure-subscription-id $SUBSCRIPTION_ID --azure-resource-group $RESOURCE_GROUP
             rad credential register azure --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} --client-secret ${{ secrets.AZURE_SP_TESTS_PASSWORD }} --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }}
           fi
+      # Deploy application and run tests
       - name: Deploy app
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         run: rad deploy ${{ matrix.path }} ${{ matrix.args }}
       - name: Wait for all pods to be ready
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         run: |
           namespace="default-${{ matrix.app }}"
           label="radius.dev/application=${{ matrix.app }}"
           kubectl wait --for=condition=Ready pod -l $label -n $namespace --timeout=5m
       - name: Run Playwright Test
         id: run-playwright-test
-        if: matrix.uiTestFile != ''
+        if: matrix.uiTestFile != '' && steps.gen-id.outputs.RUN_TEST == 'true'
         run: |
           if [[ "${{ matrix.container }}" != "" ]]; then
             rad resource expose containers ${{ matrix.container }} ${{ matrix.args }} --port ${{ matrix.port }} &
@@ -170,8 +205,18 @@ jobs:
           npm ci
           npx playwright install --with-deps
           npx playwright test ${{ matrix.uiTestFile }} --retries=3
-      - name: Get Pod Logs For Failed Tests
-        if: failure() && matrix.uiTestFile != ''
+      - name: Upload Playwright Results
+        uses: actions/upload-artifact@v3
+        if: always() && ( steps.run-playwright-test.outcome == 'success' || steps.run-playwright-test.outcome == 'failure' )
+        with:
+          name: playwright-report-${{ matrix.name }}
+          path: ui-tests/playwright-report/
+          retention-days: 30
+          if-no-files-found: error
+      # Handle failures
+      - name: Get Pod logs for failed tests
+        id: get-pod-logs
+        if: failure() && steps.run-playwright-test.outcome == 'failure'
         run: |
           # Create pod-logs directory
           mkdir -p ui-tests/pod-logs/${{ matrix.name }}
@@ -183,26 +228,25 @@ jobs:
             kubectl logs $pod_name -n $namespace > ui-tests/pod-logs/${{ matrix.name }}/${pod_name}.txt
           done
           echo "Pod logs saved to ui-tests/pod-logs/${{ matrix.name }}/"
-      - name: Upload Playwright Results
+          # Get kubernetes events and save to file
+          kubectl get events -n $namespace > ui-tests/pod-logs/${{ matrix.name }}/events.txt
+      - name: Upload Pod logs for failed tests
         uses: actions/upload-artifact@v3
-        if: always() && matrix.uiTestFile != ''
-        with:
-          name: playwright-report-${{ matrix.name }}
-          path: ui-tests/playwright-report/
-          retention-days: 30
-          if-no-files-found: error
-      - name: Upload Pod Logs
-        uses: actions/upload-artifact@v3
-        if: failure() && matrix.uiTestFile != ''
+        if: failure() && steps.get-pod-logs.outcome == 'success'
         with:
           name: ${{ matrix.name }}-pod-logs
           path: ui-tests/pod-logs/${{ matrix.name }}
           retention-days: 30
           if-no-files-found: error
+      - name: Create GitHub issue on failure
+        if: failure() && github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch'
+        run: gh issue create --title "Samples deployment failed for ${{ matrix.app }}" --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." --repo ${{ github.repository }}
+      # Cleanup
       - name: Delete app
+        if: steps.gen-id.outputs.RUN_TEST == 'true'
         run: rad app delete ${{ matrix.app }} -y
       - name: Delete Azure resource group
-        if: always() && matrix.credential == 'azure'
+        if: always() && steps.create-azure-resource-group.outcome == 'success'
         env:
           RESOURCE_GROUP: ${{ steps.gen-id.outputs.TEST_RESOURCE_GROUP_PREFIX }}-${{ matrix.name }}
           SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
@@ -212,6 +256,3 @@ jobs:
             --subscription $SUBSCRIPTION_ID \
             --name $RESOURCE_GROUP \
             --yes
-      - name: Create GitHub issue on failure
-        if: failure() && github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch'
-        run: gh issue create --title "Samples deployment failed for ${{ matrix.app }}" --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." --repo ${{ github.repository }}