test.yml

name: Test Python package
on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:

# When this workflow is queued, automatically cancel any previous running
# or pending jobs from the same branch
concurrency:
  group: test-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_and_test:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.10"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install NeMo-Curator and pytest
        # TODO: Remove pytest when optional test dependencies are added to setup.py

        # Installing wheel beforehand due to fasttext issue:
        # https://github.com/facebookresearch/fastText/issues/512#issuecomment-1837367666
        # Explicitly install cython: https://github.com/VKCOM/YouTokenToMe/issues/94
        run: |
          pip install wheel cython
          pip install --no-cache-dir --extra-index-url https://pypi.nvidia.com .
          pip install pytest
      - name: Run tests
        # TODO: Remove env variable when gpu dependencies are optional
        run: |
          RAPIDS_NO_INITIALIZE=1 python -m pytest -v --cpu