Testing for macOS and Windows

.github/workflows/ci.yml

@@ -47,9 +47,7 @@ jobs:
 
       - name: Download Coverage Artifact
         uses: actions/download-artifact@v3
-        with:
-          name: coverage
-          path: coverage
+        # if `name` is not specified, all artifacts are downloaded.
 
       - name: Upload Coverage to Coveralls
         uses: coverallsapp/github-action@v2

.github/workflows/test.yml

@@ -8,21 +8,33 @@ permissions:
 
 jobs:
   test:
-    name: ${{ github.repository }}
-    runs-on: ubuntu-latest
+    name: ${{ matrix.os }}
+    runs-on: ${{ matrix.os}}
     strategy:
       # Ensure that a wheel builder finishes even if another fails
       fail-fast: false
       matrix:
         python-version: [3.x] # latest stable release
         poetry-version: [latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
+
 
     steps:
       - name: Checkout ${{ github.repository }}
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
+      - name: Set up Python ${{matrix.python-version}}
+        uses: actions/setup-python@v4
+        id: setup-python
+        with:
+          python-version: ${{matrix.python-version}}
+          check-latest: true
+
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
@@ -31,12 +43,12 @@ jobs:
           virtualenvs-in-project: true
           installer-parallel: true
 
-      - name: Set up Python ${{matrix.python-version}}
-        uses: actions/setup-python@v4
+      - name: Load Cached Virtual Environment
+        id: cached-pip-wheels
+        uses: actions/cache@v3
         with:
-          python-version: ${{matrix.python-version}}
-          check-latest: true
-          cache: "poetry"
+          path: ~/.cache
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
       - name: Set up Poetry Dynamic Versioning
         run: |
@@ -54,12 +66,13 @@ jobs:
 
       - name: Run Tests
         run: |
+          source $VENV
           poetry install --with test --no-interaction --no-root
           poetry run pytest
 
       - name: Archive Coverage
         uses: actions/upload-artifact@v3
         with:
-          name: coverage
+          name: coverage-${{join(matrix.*, '-')}}
           path: |
             coverage.lcov

src/toffy/qc_comp.py

@@ -23,6 +23,7 @@
 
 from toffy import settings
 from toffy.mibitracker_utils import MibiRequests, MibiTrackerError
+from toffy.utils import remove_readonly
 
 
 def create_mibitracker_request_helper(email, password):
@@ -135,7 +136,7 @@ def download_mibitracker_data(
     if os.path.exists(os.path.join(base_dir, tiff_dir)):
         if overwrite_tiff_dir:
             print("Overwriting existing data in tiff_dir %s" % tiff_dir)
-            rmtree(os.path.join(base_dir, tiff_dir))
+            rmtree(os.path.join(base_dir, tiff_dir), onerror=remove_readonly)
         else:
             raise ValueError("tiff_dir %s already exists in %s" % (tiff_dir, base_dir))
 
@@ -179,7 +180,7 @@ def download_mibitracker_data(
                 )
 
                 # clean the FOV: we will not have a folder for it (in case of Moly point)
-                rmtree(os.path.join(base_dir, tiff_dir, img["number"]))
+                rmtree(os.path.join(base_dir, tiff_dir, img["number"]), onerror=remove_readonly)
 
                 # do not attempt to download any more channels
                 break

src/toffy/reorg.py

@@ -4,6 +4,7 @@
 from alpineer import io_utils, misc_utils
 
 from toffy.json_utils import read_json_file, rename_duplicate_fovs, rename_missing_fovs
+from toffy.utils import remove_readonly
 
 
 def merge_partial_runs(cohort_dir, run_string):
@@ -41,7 +42,7 @@ def merge_partial_runs(cohort_dir, run_string):
             shutil.move(os.path.join(cohort_dir, partial, fov), new_path)
 
         # remove partial folder
-        shutil.rmtree(os.path.join(cohort_dir, partial))
+        shutil.rmtree(os.path.join(cohort_dir, partial), onerror=remove_readonly)
 
 
 def combine_runs(cohort_dir):
@@ -65,7 +66,7 @@ def combine_runs(cohort_dir):
         for fov in fovs:
             shutil.move(os.path.join(run_path, fov), os.path.join(output_dir, run + "_" + fov))
 
-        shutil.rmtree(run_path)
+        shutil.rmtree(run_path, onerror=remove_readonly)
 
 
 def rename_fov_dirs(json_run_path, default_run_dir, output_run_dir=None):

src/toffy/rosetta.py

@@ -13,6 +13,7 @@
 
 from toffy.json_utils import read_json_file
 from toffy.streak_detection import streak_correction
+from toffy.utils import remove_readonly
 
 
 def transform_compensation_json(json_path, comp_mat_path):
@@ -160,10 +161,10 @@ def clean_rosetta_test_dir(folder_path):
     # remove the compensated data folders
     comp_folders = io_utils.list_folders(folder_path, substrs="compensated_data_")
     for cf in comp_folders:
-        shutil.rmtree(os.path.join(folder_path, cf))
+        shutil.rmtree(os.path.join(folder_path, cf), onerror=remove_readonly)
 
     # remove the stitched image folder
-    shutil.rmtree(os.path.join(folder_path, "stitched_images"))
+    shutil.rmtree(os.path.join(folder_path, "stitched_images"), onerror=remove_readonly)
 
 
 def flat_field_correction(img, gaus_rad=100):
@@ -551,7 +552,7 @@ def remove_sub_dirs(run_dir, sub_dirs, fovs=None):
 
     for fov in all_fovs:
         for sub_dir in sub_dirs:
-            shutil.rmtree(os.path.join(run_dir, fov, sub_dir))
+            shutil.rmtree(os.path.join(run_dir, fov, sub_dir), onerror=remove_readonly)
 
 
 def create_rosetta_matrices(

src/toffy/utils.py

@@ -0,0 +1,36 @@
+import errno
+import os
+import pathlib
+import shutil
+import stat
+from typing import TYPE_CHECKING, Callable
+
+if TYPE_CHECKING:
+    from shutil import _OnErrorCallback
+
+
+def remove_readonly(func: Callable, path: pathlib.Path | str, excinfo: "_OnErrorCallback"):
+    """
+    Removes readonly files, mainly useful for Windows CI/CD pipelines.
+    References:
+        - https://stackoverflow.com/questions/1889597/deleting-read-only-directory-in-python
+        - https://stackoverflow.com/questions/2656322/shutil-rmtree-fails-on-windows-with-access-is-denied
+
+    Example usage:
+    shutil.rmtree(my_path, onerror=remove_readonly)
+
+    Args:
+        func (Callable): The function to be called, e.g. `shutil.rmtree`.
+        path (pathlib.Path | str): The path to the file / directory.
+        excinfo (shutil._OnErrorCallback): The exception callabck.
+    """
+    # os.chmod(path, stat.S_IWRITE)
+    excvalue = excinfo[1]
+    if func in (os.rmdir, os.remove, shutil.rmtree) and excvalue.errno == errno.EACCES:
+        if not os.access(path, os.W_OK):
+            os.chmod(
+                path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IWRITE | stat.S_IWUSR
+            )  # 0777
+            func(path)
+    else:
+        raise

tests/file_hash_test.py

@@ -7,6 +7,7 @@
 from alpineer.image_utils import save_image
 
 from toffy import file_hash
+from toffy.utils import remove_readonly
 
 
 def test_get_hash():
@@ -54,7 +55,7 @@ def test_compare_directories():
             file_hash.compare_directories(dir_1, dir_2)
 
         # check that warning is raised when sub-folder is present in second directory
-        shutil.rmtree(sub_folder_1)
+        shutil.rmtree(sub_folder_1, onerror=remove_readonly)
         sub_folder_2 = os.path.join(dir_2, "sub_folder")
         os.makedirs(sub_folder_2)
 

tests/image_stitching_test.py

@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 import tempfile
 
@@ -9,6 +10,7 @@
 
 from tests.utils.test_utils import make_run_file
 from toffy import image_stitching
+from toffy.utils import remove_readonly
 
 
 def test_get_max_img_size():
@@ -232,7 +234,7 @@ def test_stitch_images(mocker, tiled, tile_names, nontiled_fov, subdir):
         # test previous stitching raises an error
         with pytest.raises(ValueError, match="The stitch_images subdirectory already exists"):
             image_stitching.stitch_images(tmpdir, test_dir, img_sub_folder=subdir, tiled=tiled)
-        shutil.rmtree(os.path.join(tmpdir, stitched_dir))
+        shutil.rmtree(os.path.join(tmpdir, stitched_dir), onerror=remove_readonly)
 
         # test stitching for specific channels
         image_stitching.stitch_images(
@@ -254,7 +256,7 @@ def test_stitch_images(mocker, tiled, tile_names, nontiled_fov, subdir):
             assert sorted(io_utils.list_files(os.path.join(tmpdir, stitched_dir))) == sorted(
                 ["Au_stitched.tiff", "CD3_stitched.tiff"]
             )
-        shutil.rmtree(os.path.join(tmpdir, stitched_dir))
+        shutil.rmtree(os.path.join(tmpdir, stitched_dir), onerror=remove_readonly)
 
     with tempfile.TemporaryDirectory() as tmpdir:
         run_name = os.path.basename(tmpdir)

tests/normalize_test.py

@@ -14,6 +14,7 @@
 
 from toffy import normalize
 from toffy.json_utils import read_json_file, write_json_file
+from toffy.utils import remove_readonly
 
 from .utils import normalize_test_cases as test_cases
 
@@ -682,8 +683,8 @@ def test_normalize_image_data(tmpdir, metrics):
         )
 
     # mismatch between FOVs
-    shutil.rmtree(os.path.join(img_dir, fovs[0]))
-    shutil.rmtree(norm_dir)
+    shutil.rmtree(os.path.join(img_dir, fovs[0]), onerror=remove_readonly)
+    shutil.rmtree(norm_dir, onerror=remove_readonly)
     os.makedirs(norm_dir)
     with pytest.raises(ValueError, match="image data fovs"):
         normalize.normalize_image_data(

tests/rosetta_test.py

@@ -14,6 +14,7 @@
 
 from toffy import rosetta
 from toffy.rosetta import create_rosetta_matrices
+from toffy.utils import remove_readonly
 
 from .utils import rosetta_test_cases as test_cases
 
@@ -617,7 +618,7 @@ def test_copy_image_files(mocker):
                 assert os.path.exists(os.path.join(extracted_fov_dir, folder, "test_image.tif"))
 
             # test successful folder copy with some runs skipped
-            rmtree(os.path.join(temp_dir2, "cohort_name"))
+            rmtree(os.path.join(temp_dir2, "cohort_name"), onerror=remove_readonly)
             with pytest.warns(UserWarning, match="The following runs will be skipped"):
                 rosetta.copy_image_files(
                     "cohort_name", run_names, temp_dir2, temp_dir, fovs_per_run=5

tests/utils/test_utils.py

@@ -17,6 +17,7 @@
 from toffy.fov_watcher import RunStructure
 from toffy.json_utils import write_json_file
 from toffy.settings import QC_COLUMNS, QC_SUFFIXES
+from toffy.utils import remove_readonly
 
 TEST_CHANNELS = [
     "Calprotectin",
@@ -504,7 +505,7 @@ def __enter__(self):
         return self.tmpdir, self.run_structure
 
     def __exit__(self, exc_type, exc_value, exc_traceback):
-        shutil.rmtree(self.tmpdir)
+        shutil.rmtree(self.tmpdir, onerror=remove_readonly)
 
 
 class RunStructureCases:

tests/utils_test.py

@@ -0,0 +1,36 @@
+import os
+import pathlib
+import shutil
+from stat import S_IREAD, S_IRGRP, S_IROTH
+
+import pytest
+
+from toffy.utils import remove_readonly
+
+
+@pytest.fixture(scope="function")
+def create_readonly_file(tmp_path: pathlib.Path) -> pathlib.Path:
+    """
+    Creates a readonly file.
+
+    Args:
+        tmp_path (pathlib.Path): A temporary path to create the file in.
+
+    Returns:
+        pathlib.Path: The path to the file.
+    """
+
+    ro_path: pathlib.Path = tmp_path / "ro_dir"
+    ro_path.mkdir()
+
+    ro_file: pathlib.Path = ro_path / "ro_file.txt"
+    ro_file.write_text("This is a readonly file.")
+    os.chmod(ro_file, S_IREAD | S_IRGRP | S_IROTH)
+    yield ro_path
+
+
+def test_remove_readonly(create_readonly_file: pathlib.Path):
+    assert not os.access(create_readonly_file / "ro_file.txt", os.W_OK)
+
+    shutil.rmtree(create_readonly_file, onerror=remove_readonly)
+    assert not (create_readonly_file / "ro_file.txt").exists()