From d0e5085e2da1348358c353fa61a84bc97aced4a7 Mon Sep 17 00:00:00 2001 From: Sagar Khalasi Date: Wed, 28 Aug 2024 11:52:37 +0530 Subject: [PATCH] ci: Added code for PR comment results with repeat command (#35921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description **Challenge**: In our existing process, when users performed a limited run with a specified count, the results were recorded solely in the logs. This approach introduced an additional step, requiring users to manually navigate through each workflow to check the results, which was both time-consuming and prone to oversight. **Solution**: To address this inefficiency, I took the initiative to develop a custom npm library specifically designed to enhance the user experience. This library automates the generation of a summary file that consolidates the run data into a single, easily accessible document. Furthermore, I integrated this solution into our existing workflow by updating the process to automatically post the contents of this summary file as a comment on the relevant PR upon completion of the run. This improvement not only streamlines the workflow by eliminating the need for users to manually sift through logs, but it also ensures that critical information is immediately available at a glance. By centralizing the results in the PR comments, we’ve enhanced transparency and efficiency, allowing for quicker decision-making and more effective collaboration. This solution is a step forward in optimizing our processes and reducing the manual effort required to track and assess test outcomes. **Looks like this:** Screenshot 2024-08-28 at 11 06 37 AM Fixes #`35920` ## Automation /ok-to-test tags="@tag.Sanity" ### :mag: Cypress test results > [!TIP] > 🟢 🟢 🟢 All cypress tests have passed! 🎉 🎉 🎉 > Workflow run: > Commit: ef6ef69fb812ba43abba5d9d273623a914569c81 > Cypress dashboard. > Tags: `@tag.Sanity` > Spec: >
Wed, 28 Aug 2024 06:19:47 UTC ## Communication Should the DevRel and Marketing teams inform users about this change? - [ ] Yes - [x] No ## Summary by CodeRabbit - **New Features** - Introduced a summary of Cypress test results in pull request comments for improved visibility. - Added functionality to upload Cypress summary logs, ensuring availability regardless of previous test outcomes. - **Dependency Updates** - Updated the testing package from `cypress-repeat-updated` to `cypress-repeat-pro`, potentially enhancing testing capabilities. --- .../workflows/build-client-server-count.yml | 39 ++++++++++++++++++- .../workflows/ci-test-limited-with-count.yml | 11 +++++- app/client/package.json | 2 +- app/client/yarn.lock | 10 ++--- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-client-server-count.yml b/.github/workflows/build-client-server-count.yml index b2ad99cac36..bb3bbcfc5ef 100644 --- a/.github/workflows/build-client-server-count.yml +++ b/.github/workflows/build-client-server-count.yml @@ -252,6 +252,18 @@ jobs: echo "$new_failed_spec_env" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV + - name: Download the client build artifact + uses: actions/download-artifact@v4 + with: + name: cypress-repeat-logs + path: app/client + + - name: Read and Set File Content as ENV Variable + id: set-summary-content + run: | + summary_content=$(cat app/client/cy-repeat-summary.txt | tr '\n' ' ') + echo "summary_content=$summary_content" >> $GITHUB_ENV + - name: Add a comment on the PR with new CI failures if: needs.ci-test-limited.result != 'success' && needs.file-check.outputs.pr != '0' uses: peter-evans/create-or-update-comment@v3 @@ -262,6 +274,9 @@ jobs: Cypress dashboard: Click here! The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} To know the list of identified flaky tests - Refer here + ``` + ${{ env.summary_content }} + ``` - name: Add a comment on the PR when ci-test-limited is success if: needs.ci-test-limited.result == 'success' && needs.file-check.outputs.pr != '0' @@ -272,6 +287,9 @@ jobs: Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. Cypress dashboard url: Click here! All cypress tests have passed 🎉🎉🎉 + ``` + ${{ env.summary_content }} + ``` - name: Check ci-test-limited set status if: needs.ci-test-limited.result != 'success' @@ -369,6 +387,19 @@ jobs: echo "new_failed_spec_env<> $GITHUB_ENV echo "$new_failed_spec_env" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV + + - name: Download the client build artifact + uses: actions/download-artifact@v4 + with: + name: cypress-repeat-logs + path: app/client + + - name: Read and Set File Content as ENV Variable + id: set-summary-content + run: | + summary_content=$(cat app/client/cy-repeat-summary.txt | tr '\n' ' ') + echo "summary_content=$summary_content" >> $GITHUB_ENV + - name: Add a comment on the PR with new CI failures if: needs.ci-test-limited-existing-docker-image.result != 'success' && needs.file-check.outputs.pr != '0' @@ -380,6 +411,9 @@ jobs: Cypress dashboard: Click here! The following are new failures, please fix them before merging the PR: ${{env.new_failed_spec_env}} To know the list of identified flaky tests - Refer here + ``` + ${{ env.summary_content }} + ``` - name: Add a comment on the PR when ci-test-limited-existing-docker-image is success if: needs.ci-test-limited-existing-docker-image.result == 'success' && needs.file-check.outputs.pr != '0' @@ -390,7 +424,10 @@ jobs: Workflow run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>. Cypress dashboard url: Click here! All cypress tests have passed 🎉🎉🎉 - + ``` + ${{ env.summary_content }} + ``` + - name: Check ci-test-limited-existing-docker-image set status if: needs.ci-test-limited-existing-docker-image.result != 'success' run: exit 1 \ No newline at end of file diff --git a/.github/workflows/ci-test-limited-with-count.yml b/.github/workflows/ci-test-limited-with-count.yml index 8790ca23188..beebd72452c 100644 --- a/.github/workflows/ci-test-limited-with-count.yml +++ b/.github/workflows/ci-test-limited-with-count.yml @@ -348,13 +348,22 @@ jobs: cd app/client npx cypress-repeat run -n ${{ inputs.run_count }} --force \ --spec ${{ env.specs_to_run }} \ - --config-file "cypress_ci_custom.config.ts" + --config-file "cypress_ci_custom.config.ts" + cat cy-repeat-summary.txt - name: Trim number of cypress log files if: failure() run: | find ${{ github.workspace }}/app/client/cypress/cypress-logs -name '*.json' -type f | tail -n +11 | xargs -I {} rm -- {} + - name: Upload repeat-pro logs artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: cypress-repeat-logs + path: ${{ github.workspace }}/app/client/cy-repeat-summary.txt + overwrite: true + - name: Upload failed test cypress logs artifact if: failure() uses: actions/upload-artifact@v4 diff --git a/app/client/package.json b/app/client/package.json index 40afddb093f..cf1766d99b6 100644 --- a/app/client/package.json +++ b/app/client/package.json @@ -120,7 +120,7 @@ "craco-babel-loader": "^1.0.4", "cssnano": "^6.0.1", "cypress-log-to-output": "^1.1.2", - "cypress-repeat-updated": "^1.0.1", + "cypress-repeat-pro": "^1.0.1", "d3-geo": "^3.1.0", "dayjs": "^1.10.6", "deep-diff": "^1.0.2", diff --git a/app/client/yarn.lock b/app/client/yarn.lock index e2c9c3e3958..b8350d99ecf 100644 --- a/app/client/yarn.lock +++ b/app/client/yarn.lock @@ -13301,7 +13301,7 @@ __metadata: cypress-network-idle: ^1.14.2 cypress-plugin-tab: ^1.0.5 cypress-real-events: ^1.8.1 - cypress-repeat-updated: ^1.0.1 + cypress-repeat-pro: ^1.0.1 cypress-tags: ^1.1.2 cypress-terminal-report: ^5.3.6 cypress-wait-until: ^1.7.2 @@ -16928,9 +16928,9 @@ __metadata: languageName: node linkType: hard -"cypress-repeat-updated@npm:^1.0.1": +"cypress-repeat-pro@npm:^1.0.1": version: 1.0.1 - resolution: "cypress-repeat-updated@npm:1.0.1" + resolution: "cypress-repeat-pro@npm:1.0.1" dependencies: arg: 5.0.2 bluebird: 3.7.2 @@ -16939,8 +16939,8 @@ __metadata: peerDependencies: cypress: ">=5.3.0" bin: - cypress-repeat: index.js - checksum: 842107e5e1c17b9b1e4d910bfe429b6c1ca8ae6765cd985fc5f08087da36b47e19c52fbc99f936a698cfa18f364c2f674f4af0e2ed9d41fd216f594e72b2506b + cypress-repeat-pro: index.js + checksum: e62a21479b62a5d5624ce21bdd8204afd754082d851d5fc712ad76e39ca2cfe6d446c2a1434025be77fa9d76fa257609285879e9841efecc26be4d25454fb732 languageName: node linkType: hard