Skip to content

BNB from source self-hosted runner with slow tests (scheduled) #358

BNB from source self-hosted runner with slow tests (scheduled)

BNB from source self-hosted runner with slow tests (scheduled) #358

Workflow file for this run

name: BNB from source self-hosted runner with slow tests (scheduled)
on:
workflow_dispatch:
schedule:
- cron: "0 2 * * *"
env:
RUN_SLOW: "yes"
IS_GITHUB_CI: "1"
# To be able to run tests on CUDA 12.2
NVIDIA_DISABLE_REQUIRE: "1"
SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
jobs:
run_all_tests_single_gpu:
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"]
runs-on:
group: aws-g6-4xlarge-plus
env:
CUDA_VISIBLE_DEVICES: "0"
TEST_TYPE: "single_gpu_${{ matrix.docker-image-name }}"
container:
image: ${{ matrix.docker-image-name }}
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
- name: Pip install
run: |
source activate peft
pip install -e . --no-deps
pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26
mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone # rename to transformers clone to avoid modules conflict
if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
cd transformers-clone
transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
echo "Checking out tag for Transformers version: v$transformers_version"
git fetch --tags
git checkout tags/v$transformers_version
cd ..
fi
- name: Test bnb import
id: import
if: always()
run: |
source activate peft
python3 -m bitsandbytes
python3 -c "import bitsandbytes as bnb"
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes import
status: ${{ steps.import.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Run examples on single GPU
id: examples_tests
if: always()
run: |
source activate peft
make tests_examples_single_gpu_bnb
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes examples tests - single GPU
status: ${{ steps.examples_tests.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Run core tests on single GPU
id: core_tests
if: always()
run: |
source activate peft
make tests_core_single_gpu_bnb
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes core tests - single GPU
status: ${{ steps.core_tests.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
# TODO: this is a test to see if BNB multi-backend single-GPU tests succeed w/o regression tests
# - name: Run BNB regression tests on single GPU
# id: regression_tests
# if: always()
# run: |
# source activate peft
# make tests_gpu_bnb_regression
# - name: Post to Slack
# if: always()
# uses: huggingface/hf-workflows/.github/actions/post-slack@main
# with:
# slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
# title: 🤗 Results of bitsandbytes regression tests - single GPU
# status: ${{ steps.regression_tests.outcome }}
# slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Run transformers tests on single GPU
id: transformers_tests
if: always()
run: |
source activate peft
make transformers_tests
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes transformers tests - single GPU
status: ${{ steps.transformers_tests.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Generate Report
if: always()
run: |
pip install slack_sdk tabulate
python scripts/log_reports.py --slack_channel_name bnb-daily-ci-collab >> $GITHUB_STEP_SUMMARY
run_all_tests_multi_gpu:
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"]
runs-on:
group: aws-g6-12xlarge-plus
env:
CUDA_VISIBLE_DEVICES: "0,1"
TEST_TYPE: "multi_gpu_${{ matrix.docker-image-name }}"
container:
image: ${{ matrix.docker-image-name }}
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
- name: Pip install
run: |
source activate peft
pip install -e . --no-deps
pip install pytest-reportlog pytest-cov parameterized datasets scipy einops
pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26
mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone
if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then
cd transformers-clone
transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//')
echo "Checking out tag for Transformers version: v$transformers_version"
git fetch --tags
git checkout tags/v$transformers_version
cd ..
fi
- name: Test bnb import
id: import
if: always()
run: |
source activate peft
python3 -m bitsandbytes
python3 -c "import bitsandbytes as bnb"
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes import
status: ${{ steps.import.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Run core GPU tests on multi-gpu
if: always()
run: |
source activate peft
- name: Run examples on multi GPU
id: examples_tests
if: always()
run: |
source activate peft
make tests_examples_multi_gpu_bnb
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes examples tests - multi GPU
status: ${{ steps.examples_tests.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Run core tests on multi GPU
id: core_tests
if: always()
run: |
source activate peft
make tests_core_multi_gpu_bnb
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes core tests - multi GPU
status: ${{ steps.core_tests.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Run transformers tests on multi GPU
id: transformers_tests
if: always()
run: |
source activate peft
make transformers_tests
- name: Post to Slack
if: always()
uses: huggingface/hf-workflows/.github/actions/post-slack@main
with:
slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }}
title: 🤗 Results of bitsandbytes transformers tests - multi GPU
status: ${{ steps.transformers_tests.outcome }}
slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
- name: Generate Report
if: always()
run: |
pip install slack_sdk tabulate
python scripts/log_reports.py --slack_channel_name bnb-daily-ci-collab >> $GITHUB_STEP_SUMMARY