BNB from source self-hosted runner with slow tests (scheduled) #357
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: BNB from source self-hosted runner with slow tests (scheduled) | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 2 * * *" | |
| env: | |
| RUN_SLOW: "yes" | |
| IS_GITHUB_CI: "1" | |
| # To be able to run tests on CUDA 12.2 | |
| NVIDIA_DISABLE_REQUIRE: "1" | |
| SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }} | |
| jobs: | |
| run_all_tests_single_gpu: | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"] | |
| runs-on: | |
| group: aws-g6-4xlarge-plus | |
| env: | |
| CUDA_VISIBLE_DEVICES: "0" | |
| TEST_TYPE: "single_gpu_${{ matrix.docker-image-name }}" | |
| container: | |
| image: ${{ matrix.docker-image-name }} | |
| options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Pip install | |
| run: | | |
| source activate peft | |
| pip install -e . --no-deps | |
| pip install pytest-reportlog pytest-cov parameterized datasets scipy einops | |
| pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26 | |
| mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone # rename to transformers clone to avoid modules conflict | |
| if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then | |
| cd transformers-clone | |
| transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//') | |
| echo "Checking out tag for Transformers version: v$transformers_version" | |
| git fetch --tags | |
| git checkout tags/v$transformers_version | |
| cd .. | |
| fi | |
| - name: Test bnb import | |
| id: import | |
| if: always() | |
| run: | | |
| source activate peft | |
| python3 -m bitsandbytes | |
| python3 -c "import bitsandbytes as bnb" | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes import | |
| status: ${{ steps.import.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Run examples on single GPU | |
| id: examples_tests | |
| if: always() | |
| run: | | |
| source activate peft | |
| make tests_examples_single_gpu_bnb | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes examples tests - single GPU | |
| status: ${{ steps.examples_tests.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Run core tests on single GPU | |
| id: core_tests | |
| if: always() | |
| run: | | |
| source activate peft | |
| make tests_core_single_gpu_bnb | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes core tests - single GPU | |
| status: ${{ steps.core_tests.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| # TODO: this is a test to see if BNB multi-backend single-GPU tests succeed w/o regression tests | |
| # - name: Run BNB regression tests on single GPU | |
| # id: regression_tests | |
| # if: always() | |
| # run: | | |
| # source activate peft | |
| # make tests_gpu_bnb_regression | |
| # - name: Post to Slack | |
| # if: always() | |
| # uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| # with: | |
| # slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| # title: 🤗 Results of bitsandbytes regression tests - single GPU | |
| # status: ${{ steps.regression_tests.outcome }} | |
| # slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Run transformers tests on single GPU | |
| id: transformers_tests | |
| if: always() | |
| run: | | |
| source activate peft | |
| make transformers_tests | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes transformers tests - single GPU | |
| status: ${{ steps.transformers_tests.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Generate Report | |
| if: always() | |
| run: | | |
| pip install slack_sdk tabulate | |
| python scripts/log_reports.py --slack_channel_name bnb-daily-ci-collab >> $GITHUB_STEP_SUMMARY | |
| run_all_tests_multi_gpu: | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| docker-image-name: ["huggingface/peft-gpu-bnb-source:latest", "huggingface/peft-gpu-bnb-latest:latest", "huggingface/peft-gpu-bnb-multi-source:latest"] | |
| runs-on: | |
| group: aws-g6-12xlarge-plus | |
| env: | |
| CUDA_VISIBLE_DEVICES: "0,1" | |
| TEST_TYPE: "multi_gpu_${{ matrix.docker-image-name }}" | |
| container: | |
| image: ${{ matrix.docker-image-name }} | |
| options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/ | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Pip install | |
| run: | | |
| source activate peft | |
| pip install -e . --no-deps | |
| pip install pytest-reportlog pytest-cov parameterized datasets scipy einops | |
| pip install "pytest>=7.2.0,<8.0.0" # see: https://github.com/huggingface/transformers/blob/ce4fff0be7f6464d713f7ac3e0bbaafbc6959ae5/setup.py#L148C6-L148C26 | |
| mkdir transformers-clone && git clone https://github.com/huggingface/transformers.git transformers-clone | |
| if [ "${{ matrix.docker-image-name }}" == "huggingface/peft-gpu-bnb-latest:latest" ]; then | |
| cd transformers-clone | |
| transformers_version=$(pip show transformers | grep '^Version:' | cut -d ' ' -f2 | sed 's/\.dev0//') | |
| echo "Checking out tag for Transformers version: v$transformers_version" | |
| git fetch --tags | |
| git checkout tags/v$transformers_version | |
| cd .. | |
| fi | |
| - name: Test bnb import | |
| id: import | |
| if: always() | |
| run: | | |
| source activate peft | |
| python3 -m bitsandbytes | |
| python3 -c "import bitsandbytes as bnb" | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes import | |
| status: ${{ steps.import.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Run core GPU tests on multi-gpu | |
| if: always() | |
| run: | | |
| source activate peft | |
| - name: Run examples on multi GPU | |
| id: examples_tests | |
| if: always() | |
| run: | | |
| source activate peft | |
| make tests_examples_multi_gpu_bnb | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes examples tests - multi GPU | |
| status: ${{ steps.examples_tests.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Run core tests on multi GPU | |
| id: core_tests | |
| if: always() | |
| run: | | |
| source activate peft | |
| make tests_core_multi_gpu_bnb | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes core tests - multi GPU | |
| status: ${{ steps.core_tests.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Run transformers tests on multi GPU | |
| id: transformers_tests | |
| if: always() | |
| run: | | |
| source activate peft | |
| make transformers_tests | |
| - name: Post to Slack | |
| if: always() | |
| uses: huggingface/hf-workflows/.github/actions/post-slack@main | |
| with: | |
| slack_channel: ${{ secrets.BNB_SLACK_CHANNEL_ID }} | |
| title: 🤗 Results of bitsandbytes transformers tests - multi GPU | |
| status: ${{ steps.transformers_tests.outcome }} | |
| slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} | |
| - name: Generate Report | |
| if: always() | |
| run: | | |
| pip install slack_sdk tabulate | |
| python scripts/log_reports.py --slack_channel_name bnb-daily-ci-collab >> $GITHUB_STEP_SUMMARY |