Smoke test daily #58
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke test daily | |
| on: | |
| schedule: | |
| - cron: '25 3 * * *' | |
| workflow_dispatch: | |
| env: | |
| API_ARTIFACT: api-daily--run-${{ github.run_number }} | |
| E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }} | |
| FORCE_COLOR: 1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: {} | |
| jobs: | |
| api-tests: | |
| name: API tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report | |
| BASE_URL: ${{ secrets.SMOKE_TEST_URL }} | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install-filters: woocommerce | |
| build: false | |
| - name: Download and install Chromium browser. | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright install chromium | |
| - name: Run 'Update WooCommerce' test. | |
| working-directory: plugins/woocommerce | |
| env: | |
| UPDATE_WC: nightly | |
| run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js update-woocommerce.spec.js | |
| - name: Run API tests. | |
| working-directory: plugins/woocommerce | |
| env: | |
| USER_KEY: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| USER_SECRET: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| run: pnpm exec playwright test --config=tests/api-core-tests/playwright.config.js hello.test.js | |
| - name: Generate API Test report. | |
| if: success() || failure() | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive API test report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| e2e-tests: | |
| name: E2E tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| env: | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_ADMIN_PASSWORD }} | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_ADMIN_USER }} | |
| ADMIN_USER_EMAIL: ${{ secrets.SMOKE_TEST_ADMIN_USER_EMAIL }} | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| BASE_URL: ${{ secrets.SMOKE_TEST_URL }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_CUSTOMER_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_CUSTOMER_USER }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install-filters: woocommerce | |
| build: false | |
| - name: Download and install Chromium browser. | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright install chromium | |
| - name: Run E2E tests. | |
| timeout-minutes: 60 | |
| working-directory: plugins/woocommerce | |
| env: | |
| E2E_MAX_FAILURES: 25 | |
| RESET_SITE: true | |
| run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js | |
| - name: Generate Playwright E2E Test report. | |
| if: success() || failure() | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive E2E test report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| k6-tests: | |
| name: k6 tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| if: success() || failure() | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install-filters: woocommerce | |
| build: false | |
| - name: Download and install Chromium browser. | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright install chromium | |
| - name: Update performance test site with E2E test | |
| working-directory: plugins/woocommerce | |
| env: | |
| BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/ | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| UPDATE_WC: nightly | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| run: | | |
| pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js update-woocommerce.spec.js | |
| continue-on-error: true | |
| - name: Install k6 | |
| run: | | |
| curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 | |
| - name: Run k6 smoke tests | |
| env: | |
| URL: ${{ secrets.SMOKE_TEST_PERF_URL }} | |
| HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }} | |
| A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| P_ID: 22733 | |
| run: | | |
| ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js | |
| test-plugins: | |
| name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| env: | |
| USE_WP_ENV: 1 | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooCommerce Payments' | |
| repo: 'automattic/woocommerce-payments' | |
| - plugin: 'WooCommerce PayPal Payments' | |
| repo: 'woocommerce/woocommerce-paypal-payments' | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| repo: 'automattic/woocommerce-services' | |
| - plugin: 'WooCommerce Subscriptions' | |
| repo: WC_SUBSCRIPTIONS_REPO | |
| private: true | |
| - plugin: 'Gutenberg' | |
| repo: 'WordPress/gutenberg' | |
| - plugin: 'Gutenberg - Nightly' | |
| repo: 'bph/gutenberg' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| build-filters: woocommerce | |
| - name: Launch wp-env e2e environment | |
| working-directory: plugins/woocommerce | |
| run: pnpm env:test --filter=woocommerce | |
| - name: Download and install Chromium browser. | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright install chromium | |
| - name: Run 'Upload plugin' test | |
| working-directory: plugins/woocommerce | |
| env: | |
| PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| run: pnpm test:e2e-pw upload-plugin.spec.js | |
| - name: Run the rest of E2E tests | |
| working-directory: plugins/woocommerce | |
| env: | |
| E2E_MAX_FAILURES: 15 | |
| run: pnpm test:e2e-pw | |
| - name: Generate E2E Test report. | |
| if: success() || failure() | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive E2E test report | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| trunk-results: | |
| name: Publish report on smoke tests on nightly build | |
| if: | | |
| ( success() || failure() ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| steps: | |
| - name: Create dirs | |
| run: | | |
| mkdir -p repo | |
| mkdir -p artifacts/api | |
| mkdir -p artifacts/e2e | |
| mkdir -p output | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| path: repo | |
| - name: Download API test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: artifacts/api | |
| - name: Download E2E test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: artifacts/e2e | |
| - name: Post test summary | |
| uses: actions/github-script@v6 | |
| env: | |
| API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json | |
| E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json | |
| with: | |
| result-encoding: string | |
| script: | | |
| const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' ) | |
| return await script( { core } ) | |
| - name: Publish report | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| gh workflow run publish-test-reports-daily.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f api_artifact="$API_ARTIFACT" \ | |
| -f e2e_artifact="$E2E_ARTIFACT" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| plugins-results: | |
| name: Publish report on Smoke tests on trunk with plugins | |
| if: | | |
| ( success() || failure() ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooCommerce Payments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| slug: woocommerce-services | |
| - plugin: 'WooCommerce Subscriptions' | |
| slug: woocommerce-subscriptions | |
| - plugin: 'Gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| slug: gutenberg-nightly | |
| steps: | |
| - name: Download test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.ARTIFACT }} | |
| - name: Publish reports | |
| run: | | |
| gh workflow run publish-test-reports-daily-plugins.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f artifact="${{ env.ARTIFACT }}" \ | |
| -f plugin="${{ matrix.plugin }}" \ | |
| -f slug="${{ matrix.slug }}" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports |