Merge pull request #429 from Dana-Farber-AIOS/dev

PathML 3.0.0: New Features and User Experience Improvements

.gitattributes

@@ -0,0 +1 @@
+tests/testdata/tilestitching_testdata/*.tif filter=lfs diff=lfs merge=lfs -text

.github/workflows/precommit-hooks.yaml

@@ -3,15 +3,16 @@ name: pre-commit-hooks
 on: [pull_request]
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
-      - name: Linting
-        run: |
-          pip install pre-commit
-          pre-commit run --all-files
+      - uses: actions/checkout@v3
+      - uses: psf/black@stable
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/[email protected]
+      with:
+        extra_args: flake8 --all-files

.github/workflows/publish-to-docker-hub.yml

@@ -27,9 +27,10 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2.7.0
+        uses: docker/build-push-action@v4 
         with:
           context: .
+          platforms: linux/amd64,linux/arm64  
           push: true
           tags: |
             pathml/pathml:latest

.github/workflows/tests-linux.yml

@@ -0,0 +1,166 @@
+name: Tests linux
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev
+      - master
+  push:
+    branches:
+      - dev
+      - master
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    timeout-minutes: 60 # add a timeout
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Display Python version
+      run: python -c "import sys; print(sys.version)"
+    - name: Install dependencies
+      shell: bash -l {0}
+      run: |
+        sudo apt-get update
+        sudo apt-get install openslide-tools
+        sudo apt-get install pandoc
+    - name: Setup Micromamba
+      uses: mamba-org/setup-micromamba@v1
+      with:
+        micromamba-version: '1.5.6-0'
+        environment-name: pathml
+        environment-file: requirements/environment_test.yml
+        init-shell: >-
+          bash
+        create-args: >-
+          python=${{ matrix.python-version }}
+    - name: Debugging
+      run: |
+        echo "Printing the environment.yml file..."
+        cat requirements/environment_test.yml
+        echo "Checking the status of micro mamba..."
+        micromamba config
+        echo "Checking the available disk space..."
+        df -h
+        
+    # - name: Install dependencies with micromamba
+    #   shell: bash -l {0}
+    #   run: micromamba env update --file requirements/environment_test.yml --name pathml
+
+    - name: Micromamba info
+      shell: bash -l {0}
+      run: |
+        micromamba info
+        micromamba config
+        micromamba list
+    
+    - name: Set default Temurin JDK 17
+      run: |
+        sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
+        java -version
+        
+    - name: Install PathML
+      shell: bash -l {0}      
+      run: pip install -e .
+
+    - name: Install torchvision
+      shell: bash -l {0}
+      run: pip install torchvision==0.14.1
+
+    - name: disk usage
+      shell: bash -l {0}
+      run: |
+        sudo df -h
+        sudo du -h
+        
+    - name: free disk space
+      run: |
+        sudo swapoff -a
+        sudo rm -f /swapfile
+        sudo apt clean
+        docker rmi $(docker image ls -aq)
+        sudo df -h
+        sudo du -h
+
+    # - name: Check Coverage Command
+    #   run: |
+    #     which coverage
+    #     coverage --version
+
+    # - name: Test with pytest for tile_stitcher
+    #   run: |
+    #     java -version
+    #     python -m pytest tests/preprocessing_tests/test_tilestitcher.py
+
+    # - name: Test with pytest and generate coverage report
+    #   shell: bash -l {0}
+    #   run: |
+    #     coverage run -m pytest -m "not slow and not exclude"
+    #     coverage xml
+    # - name: Upload coverage to Codecov
+    #   uses: codecov/codecov-action@v2
+    #   with:
+    #     token: ${{ secrets.CODECOV_TOKEN }}
+    #     env_vars: OS,PYTHON
+    #     fail_ci_if_error: true
+    #     files: ./coverage.xml
+    #     name: codecov-umbrella
+    #     verbose: true
+
+    - name: Test other modules with pytest and generate coverage
+      shell: bash -l {0}
+      run: |
+        COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude"
+
+    - name: Test tile_stitcher with pytest and generate coverage
+      shell: bash -l {0}
+      run: |
+        COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py
+    
+    - name: List Files in Directory
+      shell: bash -l {0}
+      run: |
+        ls -la
+
+    - name: Combine Coverage Data
+      shell: bash -l {0}
+      run: |
+        coverage combine .coverage_tilestitcher .coverage_others
+
+    - name: Generate Combined Coverage Report
+      shell: bash -l {0}
+      run: |
+        coverage xml -o combined_coverage.xml
+    
+    # - name: Combine coverage data
+    #   shell: bash -l {0}
+    #   run: |
+    #     coverage combine coverage_tilestitcher.xml coverage_others.xml
+    #     coverage xml -o coverage_combined.xml
+
+    - name: Upload combined coverage to Codecov
+      uses: codecov/codecov-action@v2
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        env_vars: OS,PYTHON
+        fail_ci_if_error: true
+        files: ./combined_coverage.xml
+        name: codecov-umbrella
+        verbose: true
+
+    - name: Compile docs
+      shell: bash -l {0}
+      run: |
+        cd docs
+        pip install -r readthedocs-requirements.txt
+        make html

.github/workflows/tests-macos.yml

@@ -0,0 +1,70 @@
+name: Tests MacOS
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev_graph
+  push:
+    branches:
+      - dev_graph
+
+jobs:
+  build:
+    name: Build (${{ matrix.os }}, Python ${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest]
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install openslide
+        shell: bash -l {0}
+        run: brew install openslide
+
+
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          micromamba-version: '1.5.6-0'
+          environment-name: pathml
+          environment-file: requirements/environment_mac.yml
+          init-shell: >-
+            bash
+          create-args: >-
+            python=${{ matrix.python-version }}
+
+
+      - name: Micromamba info
+        shell: bash -l {0}
+        run: |
+          micromamba info
+          micromamba config
+          micromamba list
+
+      - name: Set default Temurin JDK 17
+        run: |
+          sudo update-java-alternatives --set temurin-17-jdk-amd64 || true
+          java -version
+
+      - name: Install PathML
+        shell: bash -l {0}      
+        run: pip install -e .
+
+      - name: Install torchvision
+        shell: bash -l {0}
+        run: pip install torchvision==0.14.1
+
+      - name: Test other modules with pytest and generate coverage
+        shell: bash -l {0}
+        run: |
+         COVERAGE_FILE=.coverage_others coverage run -m pytest -m "not slow and not exclude"
+
+      - name: Test tile_stitcher with pytest and generate coverage
+        shell: bash -l {0}
+        run: |
+         COVERAGE_FILE=.coverage_tilestitcher coverage run -m pytest tests/preprocessing_tests/test_tilestitcher.py