diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62035b57..a8e320fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,9 +47,7 @@ jobs: - name: Download Coverage Artifact uses: actions/download-artifact@v3 - with: - name: coverage - path: coverage + # if `name` is not specified, all artifacts are downloaded. - name: Upload Coverage to Coveralls uses: coverallsapp/github-action@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c207b71d..fef5e4c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,14 +8,19 @@ permissions: jobs: test: - name: ${{ github.repository }} - runs-on: ubuntu-latest + name: ${{ matrix.os }} + runs-on: ${{ matrix.os}} strategy: # Ensure that a wheel builder finishes even if another fails fail-fast: false matrix: python-version: [3.x] # latest stable release poetry-version: [latest] + os: [ubuntu-latest, macos-latest, windows-latest] + defaults: + run: + shell: bash + steps: - name: Checkout ${{ github.repository }} @@ -23,6 +28,13 @@ jobs: with: fetch-depth: 0 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v4 + id: setup-python + with: + python-version: ${{matrix.python-version}} + check-latest: true + - name: Install Poetry uses: snok/install-poetry@v1 with: @@ -31,12 +43,12 @@ jobs: virtualenvs-in-project: true installer-parallel: true - - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v4 + - name: Load Cached Virtual Environment + id: cached-pip-wheels + uses: actions/cache@v3 with: - python-version: ${{matrix.python-version}} - check-latest: true - cache: "poetry" + path: ~/.cache + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} - name: Set up Poetry Dynamic Versioning run: | @@ -54,12 +66,13 @@ jobs: - name: Run Tests run: | + source $VENV poetry install --with test --no-interaction --no-root poetry run pytest - name: Archive Coverage uses: actions/upload-artifact@v3 with: - name: coverage + name: coverage-${{join(matrix.*, '-')}} path: | coverage.lcov diff --git a/src/toffy/qc_comp.py b/src/toffy/qc_comp.py index 0aea0945..d6eaff82 100644 --- a/src/toffy/qc_comp.py +++ b/src/toffy/qc_comp.py @@ -23,6 +23,7 @@ from toffy import settings from toffy.mibitracker_utils import MibiRequests, MibiTrackerError +from toffy.utils import remove_readonly def create_mibitracker_request_helper(email, password): @@ -135,7 +136,7 @@ def download_mibitracker_data( if os.path.exists(os.path.join(base_dir, tiff_dir)): if overwrite_tiff_dir: print("Overwriting existing data in tiff_dir %s" % tiff_dir) - rmtree(os.path.join(base_dir, tiff_dir)) + rmtree(os.path.join(base_dir, tiff_dir), onerror=remove_readonly) else: raise ValueError("tiff_dir %s already exists in %s" % (tiff_dir, base_dir)) @@ -179,7 +180,7 @@ def download_mibitracker_data( ) # clean the FOV: we will not have a folder for it (in case of Moly point) - rmtree(os.path.join(base_dir, tiff_dir, img["number"])) + rmtree(os.path.join(base_dir, tiff_dir, img["number"]), onerror=remove_readonly) # do not attempt to download any more channels break diff --git a/src/toffy/reorg.py b/src/toffy/reorg.py index db40905a..0690006f 100644 --- a/src/toffy/reorg.py +++ b/src/toffy/reorg.py @@ -4,6 +4,7 @@ from alpineer import io_utils, misc_utils from toffy.json_utils import read_json_file, rename_duplicate_fovs, rename_missing_fovs +from toffy.utils import remove_readonly def merge_partial_runs(cohort_dir, run_string): @@ -41,7 +42,7 @@ def merge_partial_runs(cohort_dir, run_string): shutil.move(os.path.join(cohort_dir, partial, fov), new_path) # remove partial folder - shutil.rmtree(os.path.join(cohort_dir, partial)) + shutil.rmtree(os.path.join(cohort_dir, partial), onerror=remove_readonly) def combine_runs(cohort_dir): @@ -65,7 +66,7 @@ def combine_runs(cohort_dir): for fov in fovs: shutil.move(os.path.join(run_path, fov), os.path.join(output_dir, run + "_" + fov)) - shutil.rmtree(run_path) + shutil.rmtree(run_path, onerror=remove_readonly) def rename_fov_dirs(json_run_path, default_run_dir, output_run_dir=None): diff --git a/src/toffy/rosetta.py b/src/toffy/rosetta.py index 3d194874..a3065992 100644 --- a/src/toffy/rosetta.py +++ b/src/toffy/rosetta.py @@ -13,6 +13,7 @@ from toffy.json_utils import read_json_file from toffy.streak_detection import streak_correction +from toffy.utils import remove_readonly def transform_compensation_json(json_path, comp_mat_path): @@ -160,10 +161,10 @@ def clean_rosetta_test_dir(folder_path): # remove the compensated data folders comp_folders = io_utils.list_folders(folder_path, substrs="compensated_data_") for cf in comp_folders: - shutil.rmtree(os.path.join(folder_path, cf)) + shutil.rmtree(os.path.join(folder_path, cf), onerror=remove_readonly) # remove the stitched image folder - shutil.rmtree(os.path.join(folder_path, "stitched_images")) + shutil.rmtree(os.path.join(folder_path, "stitched_images"), onerror=remove_readonly) def flat_field_correction(img, gaus_rad=100): @@ -551,7 +552,7 @@ def remove_sub_dirs(run_dir, sub_dirs, fovs=None): for fov in all_fovs: for sub_dir in sub_dirs: - shutil.rmtree(os.path.join(run_dir, fov, sub_dir)) + shutil.rmtree(os.path.join(run_dir, fov, sub_dir), onerror=remove_readonly) def create_rosetta_matrices( diff --git a/src/toffy/utils.py b/src/toffy/utils.py new file mode 100644 index 00000000..d4d497c6 --- /dev/null +++ b/src/toffy/utils.py @@ -0,0 +1,36 @@ +import errno +import os +import pathlib +import shutil +import stat +from typing import TYPE_CHECKING, Callable + +if TYPE_CHECKING: + from shutil import _OnErrorCallback + + +def remove_readonly(func: Callable, path: pathlib.Path | str, excinfo: "_OnErrorCallback"): + """ + Removes readonly files, mainly useful for Windows CI/CD pipelines. + References: + - https://stackoverflow.com/questions/1889597/deleting-read-only-directory-in-python + - https://stackoverflow.com/questions/2656322/shutil-rmtree-fails-on-windows-with-access-is-denied + + Example usage: + shutil.rmtree(my_path, onerror=remove_readonly) + + Args: + func (Callable): The function to be called, e.g. `shutil.rmtree`. + path (pathlib.Path | str): The path to the file / directory. + excinfo (shutil._OnErrorCallback): The exception callabck. + """ + # os.chmod(path, stat.S_IWRITE) + excvalue = excinfo[1] + if func in (os.rmdir, os.remove, shutil.rmtree) and excvalue.errno == errno.EACCES: + if not os.access(path, os.W_OK): + os.chmod( + path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO | stat.S_IWRITE | stat.S_IWUSR + ) # 0777 + func(path) + else: + raise diff --git a/tests/file_hash_test.py b/tests/file_hash_test.py index fa9f411b..6bfd2057 100644 --- a/tests/file_hash_test.py +++ b/tests/file_hash_test.py @@ -7,6 +7,7 @@ from alpineer.image_utils import save_image from toffy import file_hash +from toffy.utils import remove_readonly def test_get_hash(): @@ -54,7 +55,7 @@ def test_compare_directories(): file_hash.compare_directories(dir_1, dir_2) # check that warning is raised when sub-folder is present in second directory - shutil.rmtree(sub_folder_1) + shutil.rmtree(sub_folder_1, onerror=remove_readonly) sub_folder_2 = os.path.join(dir_2, "sub_folder") os.makedirs(sub_folder_2) diff --git a/tests/image_stitching_test.py b/tests/image_stitching_test.py index a1b8b85d..8ea1b1fa 100644 --- a/tests/image_stitching_test.py +++ b/tests/image_stitching_test.py @@ -1,4 +1,5 @@ import os +import re import shutil import tempfile @@ -9,6 +10,7 @@ from tests.utils.test_utils import make_run_file from toffy import image_stitching +from toffy.utils import remove_readonly def test_get_max_img_size(): @@ -232,7 +234,7 @@ def test_stitch_images(mocker, tiled, tile_names, nontiled_fov, subdir): # test previous stitching raises an error with pytest.raises(ValueError, match="The stitch_images subdirectory already exists"): image_stitching.stitch_images(tmpdir, test_dir, img_sub_folder=subdir, tiled=tiled) - shutil.rmtree(os.path.join(tmpdir, stitched_dir)) + shutil.rmtree(os.path.join(tmpdir, stitched_dir), onerror=remove_readonly) # test stitching for specific channels image_stitching.stitch_images( @@ -254,7 +256,7 @@ def test_stitch_images(mocker, tiled, tile_names, nontiled_fov, subdir): assert sorted(io_utils.list_files(os.path.join(tmpdir, stitched_dir))) == sorted( ["Au_stitched.tiff", "CD3_stitched.tiff"] ) - shutil.rmtree(os.path.join(tmpdir, stitched_dir)) + shutil.rmtree(os.path.join(tmpdir, stitched_dir), onerror=remove_readonly) with tempfile.TemporaryDirectory() as tmpdir: run_name = os.path.basename(tmpdir) diff --git a/tests/normalize_test.py b/tests/normalize_test.py index 4a163284..70a4e08c 100644 --- a/tests/normalize_test.py +++ b/tests/normalize_test.py @@ -14,6 +14,7 @@ from toffy import normalize from toffy.json_utils import read_json_file, write_json_file +from toffy.utils import remove_readonly from .utils import normalize_test_cases as test_cases @@ -682,8 +683,8 @@ def test_normalize_image_data(tmpdir, metrics): ) # mismatch between FOVs - shutil.rmtree(os.path.join(img_dir, fovs[0])) - shutil.rmtree(norm_dir) + shutil.rmtree(os.path.join(img_dir, fovs[0]), onerror=remove_readonly) + shutil.rmtree(norm_dir, onerror=remove_readonly) os.makedirs(norm_dir) with pytest.raises(ValueError, match="image data fovs"): normalize.normalize_image_data( diff --git a/tests/rosetta_test.py b/tests/rosetta_test.py index f695fa73..5dba13fa 100644 --- a/tests/rosetta_test.py +++ b/tests/rosetta_test.py @@ -14,6 +14,7 @@ from toffy import rosetta from toffy.rosetta import create_rosetta_matrices +from toffy.utils import remove_readonly from .utils import rosetta_test_cases as test_cases @@ -617,7 +618,7 @@ def test_copy_image_files(mocker): assert os.path.exists(os.path.join(extracted_fov_dir, folder, "test_image.tif")) # test successful folder copy with some runs skipped - rmtree(os.path.join(temp_dir2, "cohort_name")) + rmtree(os.path.join(temp_dir2, "cohort_name"), onerror=remove_readonly) with pytest.warns(UserWarning, match="The following runs will be skipped"): rosetta.copy_image_files( "cohort_name", run_names, temp_dir2, temp_dir, fovs_per_run=5 diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 851d6cd4..3efd3a48 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -17,6 +17,7 @@ from toffy.fov_watcher import RunStructure from toffy.json_utils import write_json_file from toffy.settings import QC_COLUMNS, QC_SUFFIXES +from toffy.utils import remove_readonly TEST_CHANNELS = [ "Calprotectin", @@ -504,7 +505,7 @@ def __enter__(self): return self.tmpdir, self.run_structure def __exit__(self, exc_type, exc_value, exc_traceback): - shutil.rmtree(self.tmpdir) + shutil.rmtree(self.tmpdir, onerror=remove_readonly) class RunStructureCases: diff --git a/tests/utils_test.py b/tests/utils_test.py new file mode 100644 index 00000000..c946d9c7 --- /dev/null +++ b/tests/utils_test.py @@ -0,0 +1,36 @@ +import os +import pathlib +import shutil +from stat import S_IREAD, S_IRGRP, S_IROTH + +import pytest + +from toffy.utils import remove_readonly + + +@pytest.fixture(scope="function") +def create_readonly_file(tmp_path: pathlib.Path) -> pathlib.Path: + """ + Creates a readonly file. + + Args: + tmp_path (pathlib.Path): A temporary path to create the file in. + + Returns: + pathlib.Path: The path to the file. + """ + + ro_path: pathlib.Path = tmp_path / "ro_dir" + ro_path.mkdir() + + ro_file: pathlib.Path = ro_path / "ro_file.txt" + ro_file.write_text("This is a readonly file.") + os.chmod(ro_file, S_IREAD | S_IRGRP | S_IROTH) + yield ro_path + + +def test_remove_readonly(create_readonly_file: pathlib.Path): + assert not os.access(create_readonly_file / "ro_file.txt", os.W_OK) + + shutil.rmtree(create_readonly_file, onerror=remove_readonly) + assert not (create_readonly_file / "ro_file.txt").exists()