ci-testing.yml

name: CI testing

# see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
on: # Trigger the workflow on push or pull request, but only for the master branch
  push:
    branches: ["master", "release/*"]
  pull_request:
    branches: ["master", "release/*"]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
  cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}

defaults:
  run:
    shell: bash

jobs:
  pytester:

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      # max-parallel: 6
      matrix:
        # PyTorch 1.5 is failing on Win and bolts requires torchvision>=0.5
        os: [ubuntu-20.04, macOS-12, windows-2022]
        python-version: [3.7, 3.9]
        requires: ['oldest', 'latest']
        topic: ['core']
        extra: [[]]
        exclude:
          # Skip if torch<1.8 and py3.9 on Linux: https://github.com/pytorch/pytorch/issues/50014
          - { python-version: 3.9, requires: 'oldest' }
          - { os: 'macOS-12', requires: 'oldest' }
          - { os: 'windows-2022', requires: 'oldest' }
        include:
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'core', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_extra']}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_baal']}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_segm']}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_vissl']}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'video', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'tabular', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'text', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.8, topic: 'pointcloud', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'serve', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'graph', extra: []}
          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'audio', extra: []}

    # Timeout: https://stackoverflow.com/a/59076067/4521646
    timeout-minutes: 50
    env:
      FREEZE_REQUIREMENTS: 1

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Set Swap Space
      if: runner.os == 'Linux'
      uses: pierotofy/set-swap-space@master
      with:
        swap-size-gb: 10

    # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
    - name: Setup macOS
      if: runner.os == 'macOS'
      run: brew install libomp openblas lapack

    - name: Setup Ubuntu
      if: runner.os == 'Linux'
      run: sudo apt-get install -y libsndfile1 graphviz

    - name: Set min. dependencies
      if: matrix.requires == 'oldest'
      run: |
        import glob, os
        # FixMe: shall be minimal for ALL dependencies not only base
        # files = glob.glob(os.path.join("requirements", "*.txt")) + ['requirements.txt']
        files = ['requirements.txt']
        for fname in files:
          lines = [line.replace('>=', '==') for line in open(fname).readlines()]
          open(fname, 'w').writelines(lines)
      shell: python

    - name: Adjust extras
      run: |
        import os
        extras = ['${{ matrix.topic }}'] + ${{ toJSON(matrix.extra) }}
        with open(os.getenv('GITHUB_ENV'), "a") as gh_env:
            gh_env.write(f"EXTRAS={','.join(extras)}")
      shell: python

    - name: Install dependencies
      env:
        TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
      run: |
        python -m pip install "pip==22.2.1"
        pip install numpy Cython "torch>=1.7.1" -f $TORCH_URL
        pip install .[$EXTRAS,test] --upgrade --prefer-binary -f $TORCH_URL
        # FixMe: this shall be one line command
        pip install --upgrade --prefer-binary \
          -r requirements/testing_${{ matrix.topic }}.txt -f $TORCH_URL
        pip list

    - name: Cache datasets
      uses: actions/cache@v3
      with:
        path: data # This path is specific to Ubuntu
        key: flash-datasets-${{ hashFiles('tests/examples/test_scripts.py') }}
        restore-keys: flash-datasets-

    - name: Tests
      env:
        FIFTYONE_DO_NOT_TRACK: true
      run: |
        # FixMe: include doctests for src/
        coverage run --source flash -m pytest \
          tests/core \
          tests/deprecated_api \
          tests/examples \
          tests/template \
          tests/${{ matrix.topic }} \
          -v # --reruns 3 --reruns-delay 2

    - name: Statistics
      run: |
        coverage report
        coverage xml

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml
        flags: unittests
        env_vars: OS,PYTHON
        name: codecov-umbrella
        fail_ci_if_error: false


  testing-guardian:
    runs-on: ubuntu-latest
    needs: pytester
    if: always()
    steps:
    - run: echo "${{ needs.pytester.result }}"
    - name: failing...
      if: needs.pytester.result == 'failure'
      run: exit 1
    - name: cancelled or skipped...
      if: contains(fromJSON('["cancelled", "skipped"]'), needs.pytester.result)
      timeout-minutes: 1
      run: sleep 90