CI + testing req. (#1488)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.azure/gpu-example-tests.yml

@@ -12,8 +12,6 @@ jobs:
   parameters:
     domains:
     - "image"
-    - "icevision"
-    - "vissl"
     - "text"
     - "tabular"
     - "video"

.azure/gpu-special-tests.yml

@@ -53,7 +53,7 @@ jobs:
 
     - bash: |
         # python -m pip install "pip==20.1"
-        pip install '.[image,test]' learn2learn
+        pip install '.[image,test]' -r requirements/testing_image.txt
         pip list
       env:
         FREEZE_REQUIREMENTS: 1
@@ -70,19 +70,3 @@ jobs:
         python -m codecov --token=$(CODECOV_TOKEN) --commit=$(Build.SourceVersion) --flags=gpu,pytest --name="GPU-coverage" --env=linux,azure
         ls -l
       displayName: 'Statistics'
-
-    - task: PublishTestResults@2
-      displayName: 'Publish test results'
-      inputs:
-        testResultsFiles: '$(Build.StagingDirectory)/test-results.xml'
-        testRunTitle: '$(Agent.OS) - $(Build.DefinitionName) - Python $(python.version)'
-      condition: succeededOrFailed()
-
-    - task: PublishCodeCoverageResults@1
-      displayName: 'Publish coverage report'
-      inputs:
-        codeCoverageTool: 'cobertura'
-        summaryFileLocation: 'coverage.xml'
-        reportDirectory: '$(Build.SourcesDirectory)/htmlcov'
-        testRunTitle: '$(Agent.OS) - $(Build.BuildNumber)[$(Agent.JobName)] - Python $(python.version)'
-      condition: succeededOrFailed()

.azure/template-examples.yml

@@ -14,7 +14,7 @@ jobs:
       # this need to have installed docker in the base image...
       container:
         # base ML image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04
-        image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.10"
+        image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.11"
         # image: "pytorch/pytorch:1.8.1-cuda11.0-cudnn8-runtime"
         options: "-it --rm --gpus=all --shm-size=16g"
 
@@ -40,39 +40,14 @@ jobs:
         displayName: 'Sanity check'
 
       - bash: |
-          # python -m pip install "pip==20.1"
-          if [ "${{dom}}" == "icevision" ]; then
-            pip install '.[image]' icevision effdet icedata;
-          elif [ "${{dom}}" == "vissl" ]; then
-            pip install '.[image]';
-          else
-            pip install '.[${{dom}}]';
-          fi
-          pip install '.[test]' --upgrade-strategy only-if-needed
+          pip install '.[${{dom}},test]' -r requirements/testing_${{dom}}.txt
           pip list
         env:
           FREEZE_REQUIREMENTS: 1
         displayName: 'Install dependencies'
 
       - bash: |
-          pip uninstall -y opencv-python opencv-python-headless
-          pip install opencv-python-headless==4.5.5.64
-        displayName: 'Install OpenCV dependencies'
-        condition: eq('${{ dom }}', 'icevision')
-
-      - bash: |
-          pip install fairscale
-          pip install git+https://github.com/facebookresearch/ClassyVision.git
-          pip install git+https://github.com/facebookresearch/vissl.git
-        displayName: 'Install VISSL dependencies'
-        condition: eq('${{ dom }}', 'vissl')
-
-      - bash: |
-          python -c "import torch; print(f'found GPUs: {torch.cuda.device_count()}')"
-          python -m coverage run --source flash -m pytest \
-            tests/examples/test_scripts.py \
-            tests/image/embedding/test_model.py \
-            -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=30
+          python -m coverage run --source flash -m pytest --durations=30
         env:
           FLASH_TEST_TOPIC: ${{ dom }}
         displayName: 'Testing'

.github/workflows/ci-testing.yml

@@ -32,14 +32,15 @@ jobs:
         exclude:
           # Skip if torch<1.8 and py3.9 on Linux: https://github.com/pytorch/pytorch/issues/50014
           - { python-version: 3.9, requires: 'oldest' }
-          - { os: macOS-12, requires: 'oldest' }
-          - { os: windows-2022, requires: 'oldest' }
+          - { os: 'macOS-12', requires: 'oldest' }
+          - { os: 'windows-2022', requires: 'oldest' }
         include:
           - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'core', extra: []}
-          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_extras']}
-          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_extras_baal']}
+          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_extra']}
+          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_baal']}
+          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_segm']}
+          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'image', extra: ['image_vissl']}
           - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'video', extra: []}
-          - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'video', extra: ['video_extras']}
           - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'tabular', extra: []}
           - { os: 'ubuntu-20.04', python-version: 3.9, topic: 'text', extra: []}
           - { os: 'ubuntu-20.04', python-version: 3.8, topic: 'pointcloud', extra: []}
@@ -68,31 +69,24 @@ jobs:
     # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
     - name: Setup macOS
       if: runner.os == 'macOS'
-      run: |
-        brew update
-        brew install libomp  # https://github.com/pytorch/pytorch/issues/20030
+      run: brew install libomp openblas lapack
 
-    - name: Install graphviz
-      if: contains( matrix.topic , 'serve' )
-      run: sudo apt-get install graphviz
+    - name: Setup Ubuntu
+      if: runner.os == 'Linux'
+      run: sudo apt-get install -y libsndfile1 graphviz
 
     - name: Set min. dependencies
       if: matrix.requires == 'oldest'
       run: |
-        fname = 'requirements.txt'
-        ignore = ['pandas', 'torchmetrics']
-        lines = [line if any([line.startswith(package) for package in ignore]) else line.replace('>', '=') for line in open(fname).readlines()]
-        open(fname, 'w').writelines(lines)
+        import glob, os
+        # FixMe: shall be minimal for ALL dependencies not only base
+        # files = glob.glob(os.path.join("requirements", "*.txt")) + ['requirements.txt']
+        files = ['requirements.txt']
+        for fname in files:
+          lines = [line.replace('>=', '==') for line in open(fname).readlines()]
+          open(fname, 'w').writelines(lines)
       shell: python
 
-    - name: Install graph test dependencies
-      if: contains( matrix.topic , 'graph' )
-      run: |
-        pip install torch==1.11.0+cpu torchvision==0.12.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
-        pip install torch-scatter -f https://data.pyg.org/whl/torch-1.11.0+cpu.html
-        pip install torch-sparse -f https://data.pyg.org/whl/torch-1.11.0+cpu.html
-        pip install torch-cluster -f https://data.pyg.org/whl/torch-1.11.0+cpu.html
-
     - name: Adjust extras
       run: |
         import os
@@ -101,32 +95,16 @@ jobs:
             gh_env.write(f"EXTRAS={','.join(extras)}")
       shell: python
 
-    - name: Install package & dependencies
+    - name: Install dependencies
       env:
-        SYSTEM_VERSION_COMPAT: 1
+        TORCH_URL: https://download.pytorch.org/whl/cpu/torch_stable.html
       run: |
         python -m pip install "pip==22.2.1"
-        pip install cython "torch>=1.7.1" -f https://download.pytorch.org/whl/cpu/torch_stable.html
-        pip install .[$EXTRAS,test] --upgrade --prefer-binary --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-
-    - name: Install vissl
-      if: contains( matrix.topic , 'image_extras' )
-      run: |
-        pip install git+https://github.com/facebookresearch/ClassyVision.git
-        pip install git+https://github.com/facebookresearch/vissl.git
-
-    - name: Install serve test dependencies
-      if: contains( matrix.topic , 'serve' )
-      run: |
-        sudo apt-get install libsndfile1
-        pip install '.[all,audio]' icevision sahi==0.8.19 effdet --upgrade
-
-    - name: Install audio test dependencies
-      if: contains( matrix.topic , 'audio' )
-      run: |
-        sudo apt-get install libsndfile1
-        pip install matplotlib
-        pip install '.[audio,image]' torch==1.11.0 --upgrade
+        pip install numpy Cython "torch>=1.7.1" -f $TORCH_URL
+        pip install .[$EXTRAS,test] \
+          -r requirements/testing_${{ matrix.topic }}.txt  \
+          --upgrade --prefer-binary -f $TORCH_URL
+        pip list
 
     - name: Cache datasets
       uses: actions/cache@v3
@@ -137,10 +115,8 @@ jobs:
 
     - name: Tests
       env:
-        FLASH_TEST_TOPIC: ${{ join(matrix.topic,',') }}
         FIFTYONE_DO_NOT_TRACK: true
       run: |
-        pip list
         # FixMe: include doctests for src/
         coverage run --source flash -m pytest \
           tests/core \

requirements.txt

@@ -1,6 +1,6 @@
 # NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
-packaging
+packaging <23.0
 setuptools <=59.5.0  # Prevent install bug with tensorboard
 numpy <1.24  # strict - freeze for using np.long
 torch >=1.7.1

requirements/datatype_audio.txt

@@ -1,7 +1,7 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 torchaudio <=0.13.1
 torchvision <=0.14.1
-librosa>=0.8.1, <=0.9.2
-transformers>=4.13.0, <=4.25.1
-datasets>=1.16.1, <=2.8.0
+librosa >=0.8.1, <=0.9.2
+transformers >=4.13.0, <=4.25.1
+datasets >=1.16.1, <=2.8.0

requirements/datatype_graph.txt

@@ -1,8 +1,8 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 torch-scatter <=2.1.0
 torch-sparse <=0.6.16
-torch-geometric>=2.0.0, <=2.2.0
+torch-geometric >=2.0.0, <=2.2.0
 torch-cluster <=1.6.0
 networkx <=2.8.8
-class-resolver>=0.3.2, <=0.3.10
+class-resolver >=0.3.2, <=0.3.10

requirements/datatype_image.txt

@@ -1,12 +1,14 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 torchvision <=0.14.1
-timm>=0.4.5, <=0.4.12
-lightning-bolts>=0.3.3, <=0.6.0
-Pillow>=7.2, <=9.3.0
+timm >0.4.5, <=0.6.11  # effdet 0.3.0 depends on timm>=0.4.12
+lightning-bolts >0.3.3, <=0.6.0
+Pillow >7.1, <=9.3.0
 albumentations <=1.3.0
-pystiche>=1.0.0, <=1.0.1
-segmentation-models-pytorch>=0.2.0, <=0.3.1
+pystiche >1.0.0, <=1.0.1
 ftfy <=6.1.1
 regex <=2022.10.31
-sahi <0.11  # strict - Fixes compatibility with icevision
+sahi >=0.8.19, <0.11  # strict - Fixes compatibility with icevision
+
+icevision >0.8
+icedata <=0.5.1 # dead

requirements/datatype_image_extras_baal.txt → requirements/datatype_image_baal.txt

@@ -1,5 +1,4 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 # This is a separate file, as baal integration is affected by vissl installation (conflicts)
-baal>=1.3.2, <=1.7.0
-icevision >=0.8
+baal >=1.3.2, <=1.7.0

requirements/datatype_image_extras.txt

@@ -1,20 +1,19 @@
 # NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
 
 matplotlib <=3.6.2
-fiftyone
-classy_vision
-vissl>=0.1.5
-icevision >=0.8
-sahi >=0.8.19,<0.11.0
-icedata
-effdet
-kornia>=0.5.1
-learn2learn; platform_system != "Windows"  # dead
-fastface
+fiftyone <0.19.0
+classy-vision <=0.6
+effdet <=0.3.0
+kornia >0.5.1, <=0.6.9
+learn2learn <=0.1.7; platform_system != "Windows"  # dead
+fastface <=0.1.3  # dead
 fairscale
 
 # pin PL for testing, remove when fastface is updated
 pytorch-lightning <1.5.0
-torchmetrics<0.8.0  # pinned PL so we force a compatible TM version
+
+# pinned PL so we force a compatible TM version
+torchmetrics<0.8.0
+
 # effdet had an issue with PL 1.12, and icevision doesn't support effdet's latest version yet (0.3.0)
 torch <1.12

requirements/datatype_image_segm.txt

@@ -0,0 +1,4 @@
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
+
+# This is a separate file, as segmentation integration is affected by vissl installation (conflicts)
+segmentation-models-pytorch >0.2.0, <=0.3.1

requirements/datatype_image_vissl.txt

@@ -0,0 +1,4 @@
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
+
+# This is a separate file, as vissl integration is affected by baal installation (conflicts)
+vissl >=0.1.5, <=0.1.6 # dead

requirements/datatype_pointcloud.txt

@@ -1,6 +1,7 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 open3d ==0.13
-torch ==1.7.1
-torchvision ==0.8.2
+# Version mismatch: Open3D needs PyTorch version 1.7.*, but version 1.8.1+cpu is installed!
+torch >=1.7.0, <1.8.0
+torchvision >=0.8.0, <0.9.0
 tensorboard <=2.11.0

requirements/datatype_tabular.txt

@@ -1,8 +1,8 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 scikit-learn <=1.2.0
-pytorch-forecasting>=0.9.0, <=0.10.3
-# pytorch-tabular>=1.0.2, <1.0.3  # pending requirements resolving
-https://github.com/Lightning-Sandbox/pytorch_tabular/archive/refs/heads/req/replace.zip
-torchmetrics>=0.10.0
-omegaconf<=2.1.1, <=2.1.1
+pytorch-forecasting >=0.9.0, <=0.10.3
+# pytorch-tabular >=1.0.2, <1.0.3  # pending requirements resolving
+https://github.com/manujosephv/pytorch_tabular/archive/refs/heads/main.zip
+torchmetrics >=0.10.0
+omegaconf <=2.1.1, <=2.1.1

requirements/datatype_text.txt

@@ -1,11 +1,11 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 torchvision <=0.14.1
-sentencepiece>=0.1.95, <=0.1.97
+sentencepiece >=0.1.95, <=0.1.97
 filelock <=3.8.2
-transformers>=4.5, <=4.25.1
-torchmetrics[text]>=0.5.1, <0.11.0
-datasets>=1.8, <=2.8.0
+transformers >4.13.0, <=4.25.1
+torchmetrics[text] >0.5.0, <0.11.0
+datasets >2.0.0, <=2.8.0
 sentence-transformers <=2.2.2
 ftfy <=6.1.1
 regex <=2022.10.31

requirements/datatype_video.txt

@@ -1,6 +1,8 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 torchvision <=0.14.1
-Pillow>=7.2, <=9.3.0
-kornia>=0.5.1, <=0.6.9
-pytorchvideo==0.1.2
+Pillow >7.1, <=9.3.0
+kornia >=0.5.1, <=0.6.9
+pytorchvideo ==0.1.2
+
+fiftyone <=0.18.0

requirements/docs.txt

@@ -1,4 +1,4 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
 sphinx >=4.0, <5.0
 myst-parser >=0.15

requirements/serve.txt

@@ -1,14 +1,14 @@
-# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forecd in setup
+# NOTE: all pins for latest are for CI consistency unless it is `strict`, then it is also forced in setup
 
-pillow <=9.3.0
+pillow >7.1, <=9.3.0
 pyyaml <=6.0
 cytoolz <=0.12.1
 graphviz <=0.20.1
 tqdm <=4.64.1
-fastapi>=0.65.2, <=0.68.2
-pydantic>1.8.1, <=1.10.2
-starlette==0.14.2
-uvicorn[standard]>=0.12.0, <=0.20.0
+fastapi >=0.65.2, <=0.68.2
+pydantic >1.8.1, <=1.10.2
+starlette ==0.14.2
+uvicorn[standard] >=0.12.0, <=0.20.0
 aiofiles <=22.1.0
-jinja2>=3.0.0, <3.1.0
+jinja2 >=3.0.0, <3.1.0
 torchvision <=0.14.1