From d6d28c908e8e1c87ad52d9c996546af7ac7e789f Mon Sep 17 00:00:00 2001 From: Samuel St-Jean <3030760+samuelstjean@users.noreply.github.com> Date: Thu, 25 Jan 2024 14:11:50 -0500 Subject: [PATCH 1/7] move script to entrypoint --- .gitignore | 3 ++ scripts/nlsam_denoising => nlsam/script.py | 4 +- nlsam/tests/test_script.py | 43 ++++++++++++++++++++++ nlsam/tests/test_scripts1.sh | 36 ------------------ nlsam/tests/test_scripts2.sh | 39 -------------------- pyproject.toml | 4 ++ setup.py | 5 +-- 7 files changed, 52 insertions(+), 82 deletions(-) rename scripts/nlsam_denoising => nlsam/script.py (99%) mode change 100755 => 100644 create mode 100644 nlsam/tests/test_script.py delete mode 100755 nlsam/tests/test_scripts1.sh delete mode 100755 nlsam/tests/test_scripts2.sh diff --git a/.gitignore b/.gitignore index 2c919592..5b88a42e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,6 @@ target/ nlsam.zip *.code-workspace +log +*.nii +*.nii.gz \ No newline at end of file diff --git a/scripts/nlsam_denoising b/nlsam/script.py old mode 100755 new mode 100644 similarity index 99% rename from scripts/nlsam_denoising rename to nlsam/script.py index bc1fc5b1..1e23dc2a --- a/scripts/nlsam_denoising +++ b/nlsam/script.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import os import argparse import logging @@ -521,7 +519,7 @@ def main(): logger.info(f"Difference map saved as {os.path.realpath(args.save_difference)}") -if __name__ == "__main__": +def hook(): # Until joblib.loky support pyinstaller, we use dask instead for the frozen binaries import sys frozen = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') diff --git a/nlsam/tests/test_script.py b/nlsam/tests/test_script.py new file mode 100644 index 00000000..0263bf45 --- /dev/null +++ b/nlsam/tests/test_script.py @@ -0,0 +1,43 @@ +import subprocess +import pytest + +from pathlib import Path + +cwd = Path(__file__).parents[2] / Path("example") +commands_crop = [ + '''python -c "import nibabel as nib; import numpy as np; d = nib.load('dwi.nii.gz').get_fdata(); nib.save(nib.Nifti1Image(d[40:50, 80:85],np.eye(4)), 'dwi_crop.nii.gz')"''', + '''python -c "import nibabel as nib; import numpy as np; d = nib.load('mask.nii.gz').get_fdata(); nib.save(nib.Nifti1Image(d[40:50, 80:85],np.eye(4)), 'mask_crop.nii.gz')"''', + '''python -c "import nibabel as nib; import numpy as np; d = nib.load('mask.nii.gz').get_fdata(); nib.save(nib.Nifti1Image(np.random.rayleigh(10, d[40:50, 80:85].shape),np.eye(4)), 'noise.nii.gz')"''', +] + +commands_nlsam = [ + # Test on example dataset + 'nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -f -N 1 --noise_est local_std --sh_order 0 --log log --cores 1 -m mask_crop.nii.gz', + 'nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f -N 1 --noise_est local_std --sh_order 6 --iterations 5 --verbose --save_sigma sigma.nii.gz', + 'nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --noise_est auto --sh_order 6 --iterations 5 --verbose --save_sigma sigma.nii.gz --save_N N.nii.gz', + 'nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --b0_threshold 10 --noise_mask pmask.nii.gz', + # Test on niftis + "gunzip dwi_crop.nii.gz mask_crop.nii.gz sigma.nii.gz" + "nlsam_denoising dwi_crop.nii dwi_nlsam.nii bvals bvecs -m mask_crop.nii -f --verbose --sh_order 0 -N 1 --no_stabilization --load_sigma sigma.nii --is_symmetric --use_threading --save_difference diff.nii", + "nlsam_denoising dwi_crop.nii dwi_nlsam.nii bvals bvecs -m mask_crop.nii -f --verbose --sh_order 0 --no_denoising --save_sigma sigma.nii --save_stab stab.nii --load_mhat dwi_crop.nii --save_eta eta.nii", + # Test on example dataset + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose -N 1 --noise_est local_std --sh_order 0 --block_size 2,2,2 --save_sigma sigma.nii.gz --cores 1", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --load_sigma sigma.nii.gz --no_subsample --fix_implausible --no_clip_eta", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --noise_map noise.nii.gz --noise_mask pmask.nii.gz --use_f32", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --load_sigma sigma.nii.gz --no_stabilization --no_denoising", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --load_sigma sigma.nii.gz --no_denoising --no_clip_eta", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --noise_est local_std --no_denoising", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --noise_est auto --no_denoising --cores 4", + "nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --no_denoising", +] + + +@pytest.mark.parametrize("command", commands_crop) +def test_crop(command): + print(cwd) + subprocess.run([command], shell=True, cwd=cwd, check=True) + + +@pytest.mark.parametrize("command", commands_nlsam) +def test_script_nlsam(command): + subprocess.run([command], shell=True, cwd=cwd, check=True) diff --git a/nlsam/tests/test_scripts1.sh b/nlsam/tests/test_scripts1.sh deleted file mode 100755 index a7039bb4..00000000 --- a/nlsam/tests/test_scripts1.sh +++ /dev/null @@ -1,36 +0,0 @@ -check_return_code () -{ - if [ $1 -ne 0 ]; then - echo 'Something failed in python apparently' - exit 1 - fi - - echo 'stuff finished' -} - -# Crop example dataset -python -c 'import nibabel as nib; import numpy as np; d = nib.load("dwi.nii.gz").get_fdata(); nib.save(nib.Nifti1Image(d[40:50, 80:85],np.eye(4)), "dwi_crop.nii.gz")' -python -c 'import nibabel as nib; import numpy as np; d = nib.load("mask.nii.gz").get_fdata(); nib.save(nib.Nifti1Image(d[40:50, 80:85],np.eye(4)), "mask_crop.nii.gz")' -python -c 'import nibabel as nib; import numpy as np; d = nib.load("mask.nii.gz").get_fdata(); nib.save(nib.Nifti1Image(np.random.rayleigh(10, d[40:50, 80:85].shape),np.eye(4)), "noise.nii.gz")' - -# Test on example dataset -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -f -N 1 --noise_est local_std --sh_order 0 --log log --cores 1 -m mask_crop.nii.gz -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f -N 1 --noise_est local_std --sh_order 6 --iterations 5 --verbose --save_sigma sigma.nii.gz -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --noise_est auto --sh_order 6 --iterations 5 --verbose --save_sigma sigma.nii.gz --save_N N.nii.gz -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --b0_threshold 10 --noise_mask pmask.nii.gz -check_return_code $? - -# Test on niftis -gunzip dwi_crop.nii.gz mask_crop.nii.gz sigma.nii.gz - -nlsam_denoising dwi_crop.nii dwi_nlsam.nii bvals bvecs -m mask_crop.nii -f --verbose --sh_order 0 -N 1 --no_stabilization --load_sigma sigma.nii --is_symmetric --use_threading --save_difference diff.nii -check_return_code $? - -nlsam_denoising dwi_crop.nii dwi_nlsam.nii bvals bvecs -m mask_crop.nii -f --verbose --sh_order 0 --no_denoising --save_sigma sigma.nii --save_stab stab.nii --load_mhat dwi_crop.nii --save_eta eta.nii -check_return_code $? diff --git a/nlsam/tests/test_scripts2.sh b/nlsam/tests/test_scripts2.sh deleted file mode 100755 index 2b7f589d..00000000 --- a/nlsam/tests/test_scripts2.sh +++ /dev/null @@ -1,39 +0,0 @@ -check_return_code () -{ - if [ $1 -ne 0 ]; then - echo 'Something failed in python apparently' - exit 1 - fi - - echo 'stuff finished' -} - -# Crop example dataset -python -c 'import nibabel as nib; import numpy as np; d = nib.load("dwi.nii.gz").get_fdata(); nib.save(nib.Nifti1Image(d[40:50, 80:85],np.eye(4)), "dwi_crop.nii.gz")' -python -c 'import nibabel as nib; import numpy as np; d = nib.load("mask.nii.gz").get_fdata(); nib.save(nib.Nifti1Image(d[40:50, 80:85],np.eye(4)), "mask_crop.nii.gz")' -python -c 'import nibabel as nib; import numpy as np; d = nib.load("mask.nii.gz").get_fdata(); nib.save(nib.Nifti1Image(np.random.rayleigh(10, d[40:50, 80:85].shape),np.eye(4)), "noise.nii.gz")' - -# Test on example dataset -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose -N 1 --noise_est local_std --sh_order 0 --block_size 2,2,2 --save_sigma sigma.nii.gz --cores 1 -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --load_sigma sigma.nii.gz --no_subsample --fix_implausible --no_clip_eta -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --noise_map noise.nii.gz --noise_mask pmask.nii.gz --use_f32 -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --load_sigma sigma.nii.gz --no_stabilization --no_denoising -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --load_sigma sigma.nii.gz --no_denoising --no_clip_eta -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 -N 1 --noise_est local_std --no_denoising -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --noise_est auto --no_denoising --cores 4 -check_return_code $? - -nlsam_denoising dwi_crop.nii.gz dwi_nlsam.nii.gz bvals bvecs -m mask_crop.nii.gz -f --verbose --sh_order 0 --no_denoising -check_return_code $? diff --git a/pyproject.toml b/pyproject.toml index 27085c20..3b64b68f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,3 +32,7 @@ dependencies = [ homepage = "https://github.com/samuelstjean/nlsam" documentation = "https://nlsam.readthedocs.io/en/latest/" changelog = "https://github.com/samuelstjean/nlsam/blob/master/CHANGELOG.md" + + +[project.scripts] +nlsam_denoising = "nlsam.script:hook" \ No newline at end of file diff --git a/setup.py b/setup.py index 59160b6f..f0d08afa 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,11 @@ import numpy -from setuptools import setup, find_packages +from setuptools import setup from Cython.Build import cythonize ext_modules = cythonize("nlsam/*.pyx") include_dirs = [numpy.get_include()] -scripts = ['scripts/nlsam_denoising'] setup( - scripts=scripts, - packages=find_packages(), include_dirs=include_dirs, ext_modules=ext_modules) From a49137148b6d2cf08092a3e15f1bb9c8af26db9c Mon Sep 17 00:00:00 2001 From: Samuel St-Jean <3030760+samuelstjean@users.noreply.github.com> Date: Thu, 25 Jan 2024 14:25:16 -0500 Subject: [PATCH 2/7] stuff --- nlsam/tests/test_script.py | 1 - pyproject.toml | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nlsam/tests/test_script.py b/nlsam/tests/test_script.py index 0263bf45..0bc257d1 100644 --- a/nlsam/tests/test_script.py +++ b/nlsam/tests/test_script.py @@ -34,7 +34,6 @@ @pytest.mark.parametrize("command", commands_crop) def test_crop(command): - print(cwd) subprocess.run([command], shell=True, cwd=cwd, check=True) diff --git a/pyproject.toml b/pyproject.toml index 3b64b68f..a3a9399c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,4 +35,8 @@ changelog = "https://github.com/samuelstjean/nlsam/blob/master/CHANGELOG.md" [project.scripts] -nlsam_denoising = "nlsam.script:hook" \ No newline at end of file +nlsam_denoising = "nlsam.script:hook" + + +[tool.setuptools.packages] +find = {} # Scanning implicit namespaces is active by default From ef127674bc55cc5e0862285f1ed1be51ac681d29 Mon Sep 17 00:00:00 2001 From: Samuel St-Jean <3030760+samuelstjean@users.noreply.github.com> Date: Thu, 25 Jan 2024 14:46:03 -0500 Subject: [PATCH 3/7] stuff --- pyproject.toml | 4 ---- setup.py | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a3a9399c..0efb8c88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,3 @@ changelog = "https://github.com/samuelstjean/nlsam/blob/master/CHANGELOG.md" [project.scripts] nlsam_denoising = "nlsam.script:hook" - - -[tool.setuptools.packages] -find = {} # Scanning implicit namespaces is active by default diff --git a/setup.py b/setup.py index f0d08afa..b1d2d725 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ import numpy -from setuptools import setup +from setuptools import setup, find_packages from Cython.Build import cythonize ext_modules = cythonize("nlsam/*.pyx") @@ -8,4 +8,5 @@ setup( include_dirs=include_dirs, + packages=find_packages(), ext_modules=ext_modules) From 657df28a540acc022545140ca7b65cc57abb12dd Mon Sep 17 00:00:00 2001 From: Samuel St-Jean <3030760+samuelstjean@users.noreply.github.com> Date: Thu, 25 Jan 2024 14:55:28 -0500 Subject: [PATCH 4/7] stuff --- .github/workflows/cibuildwheel.yml | 4 ++-- nlsam/{script.py => scripts.py} | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename nlsam/{script.py => scripts.py} (99%) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 29cc1946..d95d518f 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -86,9 +86,9 @@ jobs: path=$(python -c "import distributed; print(distributed.__path__[0]);") echo "version=v$(python -c "from importlib.metadata import version; print(version('nlsam'));")" >> $GITHUB_ENV if [ "$RUNNER_OS" == "Windows" ]; then - pyinstaller ./scripts/nlsam_denoising -F --clean --add-data "${path};.\\distributed\\" --noupx + pyinstaller ./nlsam/scripts.py -n nlsam_denoising -F --clean --add-data "${path};.\\distributed\\" --noupx else - pyinstaller ./scripts/nlsam_denoising -F --clean --add-data ${path}:./distributed/ --noupx + pyinstaller ./nlsam/scripts.py -n nlsam_denoising -F --clean --add-data ${path}:./distributed/ --noupx fi - uses: actions/upload-artifact@v4 diff --git a/nlsam/script.py b/nlsam/scripts.py similarity index 99% rename from nlsam/script.py rename to nlsam/scripts.py index 1e23dc2a..55634fe4 100644 --- a/nlsam/script.py +++ b/nlsam/scripts.py @@ -519,7 +519,7 @@ def main(): logger.info(f"Difference map saved as {os.path.realpath(args.save_difference)}") -def hook(): +if __name__ == "__main__": # Until joblib.loky support pyinstaller, we use dask instead for the frozen binaries import sys frozen = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') diff --git a/pyproject.toml b/pyproject.toml index 0efb8c88..5466e50d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,4 +35,4 @@ changelog = "https://github.com/samuelstjean/nlsam/blob/master/CHANGELOG.md" [project.scripts] -nlsam_denoising = "nlsam.script:hook" +nlsam_denoising = "nlsam.scripts:main" From 7b80152117d8fa68189ba2ca1654971fcdb16200 Mon Sep 17 00:00:00 2001 From: Samuel St-Jean <3030760+samuelstjean@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:08:38 -0500 Subject: [PATCH 5/7] pimp cibuildwheel --- .github/workflows/cibuildwheel.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index d95d518f..ec07f8be 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -31,13 +31,6 @@ jobs: CIBW_ARCHS_MACOS: x86_64 arm64 CIBW_TEST_REQUIRES: pytest CIBW_TEST_COMMAND: pytest --pyargs nlsam --verbose - CIBW_TEST_COMMAND_LINUX: > - pytest --pyargs nlsam --verbose && - cd {package}/example && - chmod +x {package}/nlsam/tests/test_scripts1.sh && - bash {package}/nlsam/tests/test_scripts1.sh && - chmod +x {package}/nlsam/tests/test_scripts2.sh && - bash {package}/nlsam/tests/test_scripts2.sh - uses: actions/upload-artifact@v4 with: From 06d3c7aa33a403408e09903e7973994a422bf439 Mon Sep 17 00:00:00 2001 From: Samuel St-Jean <3030760+samuelstjean@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:55:21 -0500 Subject: [PATCH 6/7] stuff --- nlsam/tests/dwi_crop.nii | Bin 0 -> 1574752 bytes nlsam/tests/dwi_crop.nii.gz | Bin 0 -> 128098 bytes nlsam/tests/mask_crop.nii | Bin 0 -> 38752 bytes nlsam/tests/mask_crop.nii.gz | Bin 0 -> 420 bytes nlsam/tests/noise.nii.gz | Bin 0 -> 36625 bytes nlsam/tests/sigma.nii | Bin 0 -> 1574752 bytes nlsam/tests/sigma.nii.gz | Bin 0 -> 30480 bytes 7 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 nlsam/tests/dwi_crop.nii create mode 100644 nlsam/tests/dwi_crop.nii.gz create mode 100644 nlsam/tests/mask_crop.nii create mode 100644 nlsam/tests/mask_crop.nii.gz create mode 100644 nlsam/tests/noise.nii.gz create mode 100644 nlsam/tests/sigma.nii create mode 100644 nlsam/tests/sigma.nii.gz diff --git a/nlsam/tests/dwi_crop.nii b/nlsam/tests/dwi_crop.nii new file mode 100644 index 0000000000000000000000000000000000000000..22435517c2b0eadc1883e9b0dc83409f47e40f5a GIT binary patch literal 1574752 zcmeF)&C71
%8?{
zy4L%NK74re#RJW~;_Dtp*L?rT-Ve{;ji>a{fj4|JAN{8}@k9CcW?pl~zwL2eXsdDj}7I)@kf`u@}J_ZTOQOV
z*t)A<^BwIFe)q+(AAb4lM)3zsAN7+Tzxs-E2Rq(&Kc@;ikA1?aqYq}Ucp;vX-TuxO
zhvI|jFMhfY;_BIhcwoZC*YTtMpwGUZ>gCas{NeW=5bylh4|A&zKL@P(
`3Jvxo-arr9pb`g-n9?s;=M1zJ9*B<
zcYo%h{@q@4^V~TP4_yZ|x7LFvbqG4}(C^+Z-u3qu`dxhM^PD*R(OLZBqRXH6&hMw=
zmuH{wG*{=N$9sitbkU{X9LxvfH=q9dq~_C4)5NdNrOVe>J-_A8FMY^^ I^z>ef$aU
z{IL@^bz@7E*YW(~N>6z7Q~C6ow>f
uL*e>lXRaJoPktla$)Eac@#3KK=s469pMHB^`fltv
z_CtS;;Yok{*^TKFVe4%F=)|LcXb#RB(TR87b55MO98kT^$J>~G@zq}YJaz=l2R*0a
zS$gj8BRZnD^DEwbluaJpbAXr5d)({u+r&9