[TLM] Add custom eval as part of TLMOptions #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: /rerun-failed-test-tlm CI | |
| on: | |
| issue_comment: | |
| types: [created] | |
| jobs: | |
| propertytestrerun: | |
| name: "TLM Property Test: Python ${{ matrix.python }} on ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| if: github.event.comment.body == '/rerun-failed-test-tlm' # Only run if the comment is "/rerun-failed-test-tlm" | |
| strategy: | |
| matrix: | |
| os: | |
| - macos-latest | |
| python: | |
| - "3.11" | |
| steps: | |
| - name: Checkout latest commit to pull request | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: refs/pull/${{ github.event.issue.number }}/head | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install . | |
| pip install -r tests/requirements_test.txt | |
| - name: Restore pytest cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: .pytest_cache | |
| key: pytest-cache-${{ github.run_id }} | |
| restore-keys: pytest-cache- | |
| - name: Install Cleanlab Studio client | |
| run: pip install -e . | |
| - name: Set env variables (Linux/macOS/Windows) | |
| run: | | |
| echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Cleanlab login | |
| run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}" | |
| - name: Run tests | |
| run: | | |
| pytest tests/tlm/test_properties.py --last-failed --verbose | |
| - name: Cache pytest results | |
| uses: actions/cache@v2 | |
| with: | |
| path: .pytest_cache | |
| key: pytest-cache-${{ github.run_id }} | |
| launch-tlm-test-notif: | |
| name: Comment TLM test start on PR | |
| runs-on: ubuntu-latest | |
| if: github.event.comment.body == '/rerun-failed-test-tlm' # Only run if the comment is "/rerun-failed-test-tlm" | |
| steps: | |
| - name: Log PR Number and Pull request ref | |
| run: echo "PR Number is ${{ github.event.issue.number }}. PR ref is refs/pull/${{ github.event.issue.number }}/head." | |
| - name: Find Comment | |
| id: fc | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body-regex: '^/rerun-failed-test-tlm$' | |
| direction: last | |
| - name: Update comment | |
| uses: peter-evans/create-or-update-comment@v4 | |
| if: steps.fc.outputs.comment-id != '' | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| body: | | |
| :sparkles: **Starting Rerun of Failed TLM tests...** :sparkles: | |
| If you want to run all the TLM tests (because TLM code is ready for review), comment '/test-tlm' on this PR. | |
| If you want to re-run only the failed tests again (you are still developing), comment '/rerun-failed-test-tlm' on this PR. | |
| [View full GitHub Actions run log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| reactions: rocket | |
| finish-tlm-test-notif: | |
| name: Comment TLM test completion on PR | |
| runs-on: ubuntu-latest | |
| needs: | |
| - launch-tlm-test-notif | |
| - propertytestrerun | |
| if: always() && github.event.comment.body == '/rerun-failed-test-tlm' # Only run if the comment is "/rerun-failed-test-tlm" | |
| continue-on-error: true | |
| steps: | |
| - name: Find Comment | |
| id: fc | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body-includes: Starting Rerun of Failed TLM tests | |
| direction: last | |
| - name: Build Comment Body | |
| run: | | |
| # Set up emojis based on test results | |
| if [[ "${{ needs.propertytestrerun.result }}" == "success" ]]; then | |
| PROPERTY_TEST_RESULT="✅" | |
| else | |
| PROPERTY_TEST_RESULT="❌" | |
| fi | |
| echo "PROPERTY_TEST_RESULT=$PROPERTY_TEST_RESULT" >> $GITHUB_ENV | |
| - name: Update comment | |
| if: steps.fc.outputs.comment-id != '' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| body: | | |
| :sparkles: **Failed Property Test rerun completed!** :sparkles: | |
| TLM Property Previously Failed Tests Results: ${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }} | |
| **Note:** These results are only for the tests that failed on the previous run and not for all tests. | |
| Click the Github Actions run log for more information. |