[TLM] Add TLMCalibrated class #66
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: /test-tlm CI | |
| on: | |
| issue_comment: | |
| types: [created] | |
| jobs: | |
| test: | |
| name: "TLM Test: Python ${{ matrix.python }} on ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| - macos-latest | |
| - windows-latest | |
| python: | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| - "3.11" | |
| - "3.12" | |
| steps: | |
| - name: Checkout latest commit to pull request | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: refs/pull/${{ github.event.issue.number }}/head | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install . | |
| pip install -r tests/requirements_test.txt | |
| - name: Install Cleanlab Studio client | |
| run: pip install -e . | |
| - name: Set env variables (Linux/macOS/Windows) | |
| run: | | |
| echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Cleanlab login | |
| run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}" | |
| - name: Run tests | |
| run: | | |
| pytest tests/tlm/ --ignore=tests/tlm/test_properties.py --verbose | |
| propertytest: | |
| name: "TLM Property Test: Python ${{ matrix.python }} on ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" | |
| strategy: | |
| matrix: | |
| os: | |
| - macos-latest | |
| python: | |
| - "3.11" | |
| steps: | |
| - name: Checkout latest commit to pull request | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: refs/pull/${{ github.event.issue.number }}/head | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install . | |
| pip install -r tests/requirements_test.txt | |
| - name: Restore pytest cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: .pytest_cache | |
| key: pytest-cache-${{ github.run_id }} | |
| restore-keys: pytest-cache- | |
| - name: Install Cleanlab Studio client | |
| run: pip install -e . | |
| - name: Set env variables (Linux/macOS/Windows) | |
| run: | | |
| echo "CLEANLAB_API_BASE_URL=${{ secrets.CLEANLAB_API_BASE_URL }}" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Cleanlab login | |
| run: cleanlab login --key "${{ secrets.CLEANLAB_STUDIO_CI_API_KEY }}" | |
| - name: Run tests | |
| run: | | |
| pytest -n auto tests/tlm/test_properties.py --verbose | |
| - name: Cache pytest results | |
| uses: actions/cache@v2 | |
| with: | |
| path: .pytest_cache | |
| key: pytest-cache-${{ github.run_id }} | |
| launch-tlm-test-notif: | |
| name: Comment TLM test start on PR | |
| runs-on: ubuntu-latest | |
| if: github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" | |
| steps: | |
| - name: Log PR Number and Pull request ref | |
| run: echo "PR Number is ${{ github.event.issue.number }}. PR ref is refs/pull/${{ github.event.issue.number }}/head." | |
| - name: Find Comment | |
| id: fc | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body-regex: '^/test-tlm$' | |
| direction: last | |
| - name: Update comment | |
| uses: peter-evans/create-or-update-comment@v4 | |
| if: steps.fc.outputs.comment-id != '' | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| body: | | |
| :sparkles: **Starting TLM tests...** :sparkles: | |
| If you want to run all the TLM tests again (because TLM code is ready for review), comment '/test-tlm' on this PR. | |
| If you want to re-run only the failed tests (you are still developing), comment '/rerun-failed-test-tlm' on this PR. | |
| [View full GitHub Actions run log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| reactions: rocket | |
| finish-tlm-test-notif: | |
| name: Comment TLM test completion on PR | |
| runs-on: ubuntu-latest | |
| needs: | |
| - launch-tlm-test-notif | |
| - test | |
| - propertytest | |
| if: always() && github.event.comment.body == '/test-tlm' # Only run if the comment is "/test-tlm" but still run if needs failed | |
| continue-on-error: true | |
| steps: | |
| - name: Find Comment | |
| id: fc | |
| uses: peter-evans/find-comment@v3 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body-includes: Starting TLM tests | |
| direction: last | |
| - name: Build Comment Body | |
| run: | | |
| # Set up emojis based on test results | |
| if [[ "${{ needs.test.result }}" == "success" ]]; then | |
| TLM_TEST_RESULT="✅" | |
| else | |
| TLM_TEST_RESULT="❌" | |
| fi | |
| if [[ "${{ needs.propertytest.result }}" == "success" ]]; then | |
| PROPERTY_TEST_RESULT="✅" | |
| else | |
| PROPERTY_TEST_RESULT="❌" | |
| fi | |
| echo "PROPERTY_TEST_RESULT=$PROPERTY_TEST_RESULT" >> $GITHUB_ENV | |
| echo "TLM_TEST_RESULT=$TLM_TEST_RESULT" >> $GITHUB_ENV | |
| - name: Update comment | |
| if: steps.fc.outputs.comment-id != '' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| body: | | |
| :sparkles: **Tests completed!** :sparkles: | |
| TLM Tests Results: ${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }}${{ env.TLM_TEST_RESULT }} | |
| TLM Property Tests Results: ${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }}${{ env.PROPERTY_TEST_RESULT }} | |
| Click the Github Actions run log for more information. |