Skip to content

PR comment github action tests #25

PR comment github action tests

PR comment github action tests #25

name: PR comment github action tests
on:
workflow_dispatch:
branches-ignore:
- main
issue_comment:
types:
- created
- labeled
branches-ignore:
- main
pull_request_review_comment:
types: [created, deleted]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
HF_HOME: /mnt/cache
TRANSFORMERS_IS_CI: yes
OMP_NUM_THREADS: 8
MKL_NUM_THREADS: 8
RUN_SLOW: yes
# For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
# This token is created under the bot `hf-transformers-bot`.
HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
TF_FORCE_GPU_ALLOW_GROWTH: true
RUN_PT_TF_CROSS_TESTS: 1
CUDA_VISIBLE_DEVICES: 0,1
jobs:
reply-and-trigger:
if: contains(github.event.comment.body,'/pytest')
runs-on: ubuntu-latest
permissions: write-all
steps:
- name: Set up GitHub CLI
uses: actions/checkout@v4
- name: Reply to the comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
-f "body=This comment contains /pytest, running the specified job..."
- name: Create Run
id: create_run
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs/${{ github.job_id }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/OWNER/REPO/statuses/SHA \
-f "state=pending" -f "target_url=$GITHUB_RUN_URL" -f "description=The build succeeded!" -f "context=continuous-integration/pytest"
- name: Create Check Run
id: create_check_run
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs/${{ github.job }}
run: |
CHECK_RUN_ID=$(gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/check-runs \
-f "name=Custom slow tests" \
-f "head_sha=${{ github.event.pull_request.head.sha }}" \
-f "details_url=$GITHUB_RUN_URL"
-f "status=in_progress" \
-f "external_id=42" \
-f "started_at=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-f "output[title]=Mighty Readme report" \
-f "output[summary]=The current GitHub Actions run can be viewed here: $GITHUB_RUN_URL" \
-f "output[text]=Detailed output goes here." \
| jq -r '.id')
echo "CHECK_RUN_ID=$CHECK_RUN_ID" >> $GITHUB_ENV
- name: Trigger specific job
run: |
echo "Running the specific job because /pytest was found"
# Add the logic to trigger your specific job here
- name: Update Check Run Status
if: always()
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CHECK_RUN_ID: ${{ env.CHECK_RUN_ID }}
run: |
gh api \
--method PATCH \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/check-runs/${{ env.CHECK_RUN_ID }} \
-f "name=mighty_readme" \
-f "started_at=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-f "status=completed" \
-f "conclusion=success" \
-f "completed_at=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-f "output[title]=Mighty Readme report" \
-f "output[summary]=There are 0 failures, 2 warnings, and 1 notices." \
-f "output[text]=You may have some misspelled words on lines 2 and 4. You also may want to add a section in your README about how to install your app." \
-f "output[annotations[][path]]=README.md" \
-f "output[annotations[][annotation_level]]=warning" \
-f "output[annotations[][title]]=Spell Checker" \
-f "output[annotations[][message]]=Check your spelling for 'banaas'." \
-f "output[annotations[][raw_details]]=Do you mean 'bananas' or 'banana'?" \
-f "output[annotations[][start_line]]=2" \
-f "output[annotations[][end_line]]=2" \
-f "output[annotations[][path]]=README.md" \
-f "output[annotations[][annotation_level]]=warning" \
-f "output[annotations[][title]]=Spell Checker" \
-f "output[annotations[][message]]=Check your spelling for 'aples'" \
-f "output[annotations[][raw_details]]=Do you mean 'apples' or 'Naples'" \
-f "output[annotations[][start_line]]=4" \
-f "output[annotations[][end_line]]=4" \
-f "output[images[][alt]]=Super bananas" \
-f "output[images[][image_url]]=http://example.com/images/42"
run_if_label_matches:
if: github.event.label.name == 'run-slow'
runs-on: ubuntu-latest
steps:
- run: echo 'The label was run-slow'
pr_commented:
# This job only runs for pull request comments
name: PR comment
if: ${{ github.event.issue.pull_request }}
runs-on: ubuntu-latest
steps:
- run: |
echo A comment on PR $NUMBER
env:
NUMBER: ${{ github.event.issue.number }}
testing-stuff-around:
if: contains(github.event.comment.html_url, '/pull/') # check if the comments come from pull request, exclude those from issue.
runs-on: ubuntu-22.04
steps:
- name: say hello
if: contains(github.event.comment.body, '/pytest') # check the comment if it contains the keywords
run: |
echo say hello
- name: test
if: contains(github.event.comment.body, '/test') # check the comment if it contains the keywords
run: |
echo test
find_models_to_run:
runs-on: ubuntu-22.04
name: Find models to run slow tests
# Triggered only if the required label `run-slow` is added
if: github.event.issue.pull_request && contains(github.event.comment.body,'/pytest')
outputs:
models: ${{ steps.models_to_run.outputs.models }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: "0"
ref: ${{ github.event.pull_request.head.sha }}
- name: Get commit message
run: |
echo "commit_message=$(git show -s --format=%s)" >> $GITHUB_ENV
- name: Parse comment and set machine type
if: github.event_name == 'issue_comment'
run: |
COMMENT=${{ github.event.comment.body }}
if [[ "$COMMENT" == *"--device"* ]]; then
DEVICE=$(echo "$COMMENT" | grep -oP '(?<=--device\s)\S+')
echo "machine_type=${DEVICE}" >> $GITHUB_OUTPUT
fi
- name: Get models to run slow tests
run: |
echo "${{ env.commit_message }}"
python -m pip install GitPython
python utils/pr_slow_ci_models.py --commit_message "${{ env.commit_message }}" | tee output.txt
echo "models=$(tail -n 1 output.txt)" >> $GITHUB_ENV
- name: Models to run slow tests
id: models_to_run
run: |
echo "${{ env.models }}"
echo "models=${{ env.models }}" >> $GITHUB_OUTPUT
run_models_gpu:
name: Run all tests for the model
if: ${{ needs.find_models_to_run.outputs.models != '[]' || github.event_name == 'issue_comment' }}
needs: find_models_to_run
strategy:
fail-fast: false
matrix:
folders: ${{ fromJson(needs.find_models_to_run.outputs.models) || '[]' }}
machine_type: [single-gpu, multi-gpu]
runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, ci]
container:
image: huggingface/transformers-all-latest-gpu
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
steps:
- name: Parse command from PR comment
if: ${{ github.event_name == 'issue_comment' }}
id: parse_command
run: |
comment="${{ github.event.comment.body }}"
if [[ "$comment" =~ /run[[:space:]]+pytest ]]; then
export pytest_command=$(echo "$comment" | sed -n 's/\/run[[:space:]]\+pytest[[:space:]]\+\(.*\)/\1/p')
echo "pytest_command=$pytest_command" >> $GITHUB_ENV
else
echo "Invalid command. Exiting."
exit 1
env:
pytest_command: ""
- name: Echo input and matrix info
if: ${{ github.event_name == 'pull_request' || env.pytest_command != '' }}
shell: bash
run: |
echo "${{ matrix.folders }}"
- name: Echo folder ${{ matrix.folders }}
if: ${{ github.event_name == 'pull_request' || env.pytest_command != '' }}
shell: bash
run: |
echo "${{ matrix.folders }}"
matrix_folders=${{ matrix.folders }}
matrix_folders=${matrix_folders/'models/'/'models_'}
echo "$matrix_folders"
echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
- name: Update clone
working-directory: /transformers
run: git fetch && git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge && git checkout pull/${{ github.event.pull_request.number }}/merge
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
working-directory: /transformers
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
- name: NVIDIA-SMI
run: |
nvidia-smi
- name: Environment
working-directory: /transformers
run: |
python3 utils/print_env.py
- name: Show installed libraries and their versions
working-directory: /transformers
run: pip freeze
- name: Run all tests on GPU
working-directory: /transformers
run: |
export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py --test_folder ${{ matrix.folders }})"
echo $CUDA_VISIBLE_DEVICES
python3 -m pytest -v -rsfE --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Run custom tests from comment
working-directory: /transformers
if: ${{ github.event_name == 'issue_comment' && env.pytest_command != '' }}
run: |
export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py)"
echo $CUDA_VISIBLE_DEVICES
python3 -m pytest -v ${{ env.pytest_command }}
- name: Failure short reports
if: ${{ failure() }}
continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: Make sure report directory exists
shell: bash
run: |
mkdir -p /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
echo "hello" > /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
echo "${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports