diff --git a/.github/scripts/validate_semver.py b/.github/scripts/validate_semver.py index 7f182606..6d06b9d3 100755 --- a/.github/scripts/validate_semver.py +++ b/.github/scripts/validate_semver.py @@ -29,12 +29,13 @@ def main(): SEMVER_REGEX = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" pattern = re.compile(SEMVER_REGEX) - version = sys.argv[1] match = pattern.search(version) - - # If no match, then return an error (provided version is not valid semver) - if match is None: + + if version == "edge": + print("Provided version is edge") + sys.exit(0) + elif match is None: print("Provided version is not valid semver") sys.exit(1) else: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 67f9dbfa..11a07d9f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,12 +1,12 @@ -name: Test Quickstarts +name: Test Samples on: workflow_dispatch: inputs: version: - description: 'Radius version number to use (e.g. 0.22.0, 0.23.0-rc1)' + description: 'Radius version number to use (e.g. 0.22.0, 0.23.0-rc1, edge). Defaults to edge.' required: false - default: '' + default: 'edge' type: string push: branches: @@ -15,6 +15,7 @@ on: paths: - "quickstarts/**" - "reference-apps/**" + - "demo/**" - ".github/workflows/**" pull_request: types: [opened, synchronize, reopened] @@ -28,72 +29,102 @@ env: jobs: test: name: Sample tests - if: github.event.action != 'closed' runs-on: [self-hosted, 1ES.Pool=1ES-Radius-Samples] strategy: fail-fast: false matrix: include: - name: demo + runOnPullRequest: true app: demo path: ./demo/app.bicep args: --application demo uiTestFile: tests/demo.app.spec.ts port: 3000 container: demo + enableDapr: false - name: dapr + runOnPullRequest: true app: dapr-quickstart path: ./quickstarts/dapr/dapr.bicep enableDapr: true - name: environment-variables + runOnPullRequest: true app: myapp path: ./quickstarts/environment-variables/app.bicep + enableDapr: false - name: volumes + runOnPullRequest: true app: myapp path: ./quickstarts/volumes/app.bicep + enableDapr: false - name: eshop + runOnPullRequest: true app: eshop path: ./reference-apps/eshop/iac/eshop.bicep args: --application eshop uiTestFile: tests/eshop/container.app.spec.ts + enableDapr: false - name: eshop-azure + runOnPullRequest: false app: eshop path: ./reference-apps/eshop/iac/eshop.bicep args: --application eshop -p platform=azure uiTestFile: tests/eshop/container.app.spec.ts credential: azure + enableDapr: false env: BRANCH: ${{ github.base_ref || github.ref_name }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} AZURE_LOCATION: westus3 steps: + # Setup the test assets and configuration - name: Generate output variables id: gen-id run: | BASE_STR="SAMPLES|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}" UNIQUE_ID=$(echo $BASE_STR | sha1sum | head -c 10) + + if [[ "${{ github.event_name }}" == "pull_request" && "${{ matrix.runOnPullRequest }}" == "false" ]]; then + RUN_TEST=false + else + RUN_TEST=true + fi + + if [[ "${{ matrix.enableDapr }}" == "true" ]]; then + ENABLE_DAPR=true + else + ENABLE_DAPR=false + fi # Set output variables to be used in the other jobs echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT echo "TEST_RESOURCE_GROUP_PREFIX=samplestest-${UNIQUE_ID}" >> $GITHUB_OUTPUT + echo "RUN_TEST=${RUN_TEST}" >> $GITHUB_OUTPUT + echo "ENABLE_DAPR=${ENABLE_DAPR}" >> $GITHUB_OUTPUT - name: Checkout code + if: steps.gen-id.outputs.RUN_TEST == 'true' uses: actions/checkout@v3 - name: Ensure inputs.version is valid semver - if: inputs.version != '' + if: inputs.version != '' && steps.gen-id.outputs.RUN_TEST == 'true' run: | python ./.github/scripts/validate_semver.py ${{ inputs.version }} - name: Setup Node + if: steps.gen-id.outputs.RUN_TEST == 'true' uses: actions/setup-node@v3 with: node-version: 16 - name: az CLI login + if: matrix.credential == 'azure' && steps.gen-id.outputs.RUN_TEST == 'true' run: | az login --service-principal \ --username ${{ secrets.AZURE_SP_TESTS_APPID }} \ --password ${{ secrets.AZURE_SP_TESTS_PASSWORD }} \ --tenant ${{ secrets.AZURE_SP_TESTS_TENANTID }} + # Create and install test environment - name: Create Azure resource group - if: matrix.credential == 'azure' + id: create-azure-resource-group + if: matrix.credential == 'azure' && steps.gen-id.outputs.RUN_TEST == 'true' env: RESOURCE_GROUP: ${{ steps.gen-id.outputs.TEST_RESOURCE_GROUP_PREFIX }}-${{ matrix.name }} SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} @@ -109,16 +140,15 @@ jobs: sleep 5 done - name: Download k3d + if: steps.gen-id.outputs.RUN_TEST == 'true' run: wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash - name: Download rad CLI + if: steps.gen-id.outputs.RUN_TEST == 'true' run: | for attempt in 1 2 3 4 5; do if [[ -n "${{ inputs.version }}" ]]; then echo "Downloading rad CLI version ${{ inputs.version }}" wget -q "${{ env.RAD_CLI_URL }}" -O - | /bin/bash -s ${{ inputs.version }} - elif [ "$BRANCH" = "edge" ]; then - echo "Downloading edge rad CLI" - wget -q "${{ env.RAD_CLI_URL }}" -O - | /bin/bash -s edge else echo "Downloading latest rad CLI" wget -q "${{ env.RAD_CLI_URL }}" -O - | /bin/bash @@ -129,13 +159,15 @@ jobs: fi done - name: Create k3d cluster + if: steps.gen-id.outputs.RUN_TEST == 'true' run: k3d cluster create --agents 2 -p "80:80@loadbalancer" --k3s-arg "--disable=traefik@server:0" - name: Install Dapr - if: ${{ matrix.enableDapr }} + if: steps.gen-id.outputs.RUN_TEST == 'true' && steps.gen-id.outputs.ENABLE_DAPR == 'true' run: | helm repo add dapr https://dapr.github.io/helm-charts/ helm install dapr dapr/dapr --version=1.6 --namespace dapr-system --create-namespace --wait - name: Init local environment + if: steps.gen-id.outputs.RUN_TEST == 'true' env: RESOURCE_GROUP: ${{ steps.gen-id.outputs.TEST_RESOURCE_GROUP_PREFIX }}-${{ matrix.name }} SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} @@ -152,16 +184,19 @@ jobs: rad env update default --azure-subscription-id $SUBSCRIPTION_ID --azure-resource-group $RESOURCE_GROUP rad credential register azure --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} --client-secret ${{ secrets.AZURE_SP_TESTS_PASSWORD }} --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }} fi + # Deploy application and run tests - name: Deploy app + if: steps.gen-id.outputs.RUN_TEST == 'true' run: rad deploy ${{ matrix.path }} ${{ matrix.args }} - name: Wait for all pods to be ready + if: steps.gen-id.outputs.RUN_TEST == 'true' run: | namespace="default-${{ matrix.app }}" label="radius.dev/application=${{ matrix.app }}" kubectl wait --for=condition=Ready pod -l $label -n $namespace --timeout=5m - name: Run Playwright Test id: run-playwright-test - if: matrix.uiTestFile != '' + if: matrix.uiTestFile != '' && steps.gen-id.outputs.RUN_TEST == 'true' run: | if [[ "${{ matrix.container }}" != "" ]]; then rad resource expose containers ${{ matrix.container }} ${{ matrix.args }} --port ${{ matrix.port }} & @@ -170,8 +205,18 @@ jobs: npm ci npx playwright install --with-deps npx playwright test ${{ matrix.uiTestFile }} --retries=3 - - name: Get Pod Logs For Failed Tests - if: failure() && matrix.uiTestFile != '' + - name: Upload Playwright Results + uses: actions/upload-artifact@v3 + if: always() && ( steps.run-playwright-test.outcome == 'success' || steps.run-playwright-test.outcome == 'failure' ) + with: + name: playwright-report-${{ matrix.name }} + path: ui-tests/playwright-report/ + retention-days: 30 + if-no-files-found: error + # Handle failures + - name: Get Pod logs for failed tests + id: get-pod-logs + if: failure() && steps.run-playwright-test.outcome == 'failure' run: | # Create pod-logs directory mkdir -p ui-tests/pod-logs/${{ matrix.name }} @@ -183,26 +228,25 @@ jobs: kubectl logs $pod_name -n $namespace > ui-tests/pod-logs/${{ matrix.name }}/${pod_name}.txt done echo "Pod logs saved to ui-tests/pod-logs/${{ matrix.name }}/" - - name: Upload Playwright Results + # Get kubernetes events and save to file + kubectl get events -n $namespace > ui-tests/pod-logs/${{ matrix.name }}/events.txt + - name: Upload Pod logs for failed tests uses: actions/upload-artifact@v3 - if: always() && matrix.uiTestFile != '' - with: - name: playwright-report-${{ matrix.name }} - path: ui-tests/playwright-report/ - retention-days: 30 - if-no-files-found: error - - name: Upload Pod Logs - uses: actions/upload-artifact@v3 - if: failure() && matrix.uiTestFile != '' + if: failure() && steps.get-pod-logs.outcome == 'success' with: name: ${{ matrix.name }}-pod-logs path: ui-tests/pod-logs/${{ matrix.name }} retention-days: 30 if-no-files-found: error + - name: Create GitHub issue on failure + if: failure() && github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' + run: gh issue create --title "Samples deployment failed for ${{ matrix.app }}" --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." --repo ${{ github.repository }} + # Cleanup - name: Delete app + if: steps.gen-id.outputs.RUN_TEST == 'true' run: rad app delete ${{ matrix.app }} -y - name: Delete Azure resource group - if: always() && matrix.credential == 'azure' + if: always() && steps.create-azure-resource-group.outcome == 'success' env: RESOURCE_GROUP: ${{ steps.gen-id.outputs.TEST_RESOURCE_GROUP_PREFIX }}-${{ matrix.name }} SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} @@ -212,6 +256,3 @@ jobs: --subscription $SUBSCRIPTION_ID \ --name $RESOURCE_GROUP \ --yes - - name: Create GitHub issue on failure - if: failure() && github.event_name != 'pull_request' && github.event_name != 'workflow_dispatch' - run: gh issue create --title "Samples deployment failed for ${{ matrix.app }}" --body "Test failed on ${{ github.repository }}. See [workflow logs](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for more details." --repo ${{ github.repository }}