[TLM CI] Add unique queries for tests + add re-run failed tests ci #58
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: /test-tlm CI | |
| on: | |
| issue_comment: | |
| types: [created] | |
| jobs: | |
| test: | |
| name: "TLM Test: Python ${{ matrix.python }} on ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| - windows-latest | |
| python: | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| steps: | |
| - name: Checkout latest commit to pull request | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: refs/pull/${{ github.event.issue.number }}/head | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install . | |
| pip install -r tests/requirements_test.txt | |
| - name: Install Cleanlab Studio client | |
| run: pip install -e . | |
| - name: Set env variables (Linux/macOS/Windows) | |
| run: | | |
| echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Cleanlab login | |
| run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}" | |
| - name: Run tests | |
| run: | | |
| pytest tests/tlm/ --ignore=tests/tlm/test_properties.py --verbose | |
| propertytest: | |
| name: "TLM Property Test: Python ${{ matrix.python }} on ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" | |
| strategy: | |
| matrix: | |
| os: | |
| - macos-latest | |
| python: | |
| - "3.11" | |
| steps: | |
| - name: Checkout latest commit to pull request | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: refs/pull/${{ github.event.issue.number }}/head | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install . | |
| pip install -r tests/requirements_test.txt | |
| - name: Install Cleanlab Studio client | |
| run: pip install -e . | |
| - name: Set env variables (Linux/macOS/Windows) | |
| run: | | |
| echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Cleanlab login | |
| run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}" | |
| - name: Run tests | |
| run: | | |
| pytest -n auto tests/tlm/test_properties.py --verbose | |
| launch-tlm-test-notif: | |
| name: Comment TLM test start on PR | |
| runs-on: ubuntu-latest | |
| if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" | |
| steps: | |
| - name: Log PR Number and Pull request ref | |
| run: echo "PR Number is ${{ github.event.issue.number }}. PR ref is refs/pull/${{ github.event.issue.number }}/head." | |
| - name: Find Comment | |
| id: fc | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body-regex: '^/test-tlm$' | |
| direction: last | |
| - name: Update comment | |
| uses: peter-evans/create-or-update-comment@v4 | |
| if: steps.fc.outputs.comment-id != '' | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| body: | | |
| :sparkles: **Starting TLM tests...** :sparkles: | |
| If you want to run the TLM tests again (in case they failed or you have further updates to the TLM code), comment '/test-tlm' on this PR. | |
| [View full GitHub Actions run log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| reactions: rocket | |
| finish-tlm-test-notif: | |
| name: Comment TLM test completion on PR | |
| runs-on: ubuntu-latest | |
| needs: | |
| - launch-tlm-test-notif | |
| - test | |
| - propertytest | |
| if: always() && github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" but still run if needs failed | |
| continue-on-error: true | |
| steps: | |
| - name: Find Comment | |
| id: fc | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body-includes: Starting TLM tests | |
| direction: last | |
| - name: Build Comment Body | |
| run: | | |
| # Set up emojis based on test results | |
| if [[ "${{ needs.test.result }}" == "success" ]]; then | |
| TLM_TEST_RESULT="✅" | |
| else | |
| TLM_TEST_RESULT="❌" | |
| fi | |
| if [[ "${{ needs.propertytest.result }}" == "success" ]]; then | |
| PROPERTY_TEST_RESULT="✅" | |
| else | |
| PROPERTY_TEST_RESULT="❌" | |
| fi | |
| echo "PROPERTY_TEST_RESULT=$PROPERTY_TEST_RESULT" >> $GITHUB_ENV | |
| echo "TLM_TEST_RESULT=$TLM_TEST_RESULT" >> $GITHUB_ENV | |
| - name: Update comment | |
| if: steps.fc.outputs.comment-id != '' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| body: | | |
| :sparkles: **Tests completed!** :sparkles: | |
| TLM Tests Results: ${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }} | |
| TLM Property Tests Results: ${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }} | |
| Click the Github Actions run log for more information. |