Skip to content

Commit e1de37a

Browse files
kostrykinkostrykin
authored
Merge pull request #15 from BMCV/develop
Update to 0.3.0
2 parents 03a5677 + 2f78977 commit e1de37a

File tree

1,707 files changed

+119108
-48
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,707 files changed

+119108
-48
lines changed

‎.github/workflows/regressiontests.yml

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: Regression tests
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
branches: ['develop']
7+
paths:
8+
- .github/workflows/regressiontests.yml
9+
- superdsm/**
10+
- examples/**
11+
- superdsm.yml
12+
- tests/regression
13+
- requirements.txt
14+
15+
jobs:
16+
17+
regression_tests:
18+
name: "Test: ${{ matrix.taskdir }}"
19+
timeout-minutes: 1440
20+
runs-on: gpuagrohr-01
21+
container:
22+
image: ubuntu:22.04
23+
defaults:
24+
run:
25+
shell: bash -el {0}
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
taskdir:
30+
- U2OS
31+
- NIH3T3
32+
- GOWT1-1
33+
- GOWT1-2
34+
35+
steps:
36+
37+
- name: Checkout
38+
uses: actions/checkout@v2
39+
40+
- name: Clear cache # otherwise corrupted packages can be reported sometimes
41+
run: rm -rf /github/home/conda_pkgs_dir
42+
43+
- name: Setup Miniconda
44+
uses: conda-incubator/setup-miniconda@v2
45+
with:
46+
miniconda-version: 'latest'
47+
channels: conda-forge, bioconda, defaults
48+
auto-update-conda: true
49+
auto-activate-base: false
50+
activate-environment: superdsm
51+
environment-file: superdsm.yml
52+
53+
- name: Download image data
54+
run: |
55+
cd examples
56+
python load_data.py
57+
58+
- name: Run SuperDSM
59+
run: |
60+
python -m "superdsm.batch" examples --task-dir "${{ matrix.taskdir }}"
61+
python -m "superdsm.batch" examples --task-dir "${{ matrix.taskdir }}" --run
62+
env:
63+
SUPERDSM_INTERMEDIATE_OUTPUT: false
64+
SUPERDSM_NUM_CPUS: 20
65+
66+
- name: Validate results
67+
id: validation
68+
run: |
69+
mkdir "actual_csv"
70+
sh "tests/regression/validate-${{ matrix.taskdir }}.sh" "actual_csv"
71+
72+
- name: Upload artifact
73+
if: failure() && steps.validation.outcome != 'success'
74+
uses: actions/upload-artifact@v3 ## v4 requires GLIBC_2.28 which is not found on host
75+
with:
76+
name: Results ${{ matrix.taskdir }}
77+
path: actual_csv/${{ matrix.taskdir }}

‎.github/workflows/testsuite.yml

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
name: Test suite
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
branches: ['develop']
7+
paths:
8+
- .github/workflows/testsuite.yml
9+
- superdsm/**
10+
- examples/**
11+
- superdsm.yml
12+
- tests/**
13+
- requirements.txt
14+
15+
jobs:
16+
17+
run_testsuite:
18+
name: Test suite
19+
runs-on: gpuagrohr-01
20+
container:
21+
image: ubuntu:22.04
22+
defaults:
23+
run:
24+
shell: bash -el {0}
25+
26+
steps:
27+
28+
- name: Checkout
29+
uses: actions/checkout@v2
30+
31+
- name: Clear cache # otherwise corrupted packages can be reported sometimes
32+
run: rm -rf /github/home/conda_pkgs_dir
33+
34+
- name: Setup Miniconda
35+
uses: conda-incubator/setup-miniconda@v2
36+
with:
37+
miniconda-version: 'latest'
38+
channels: conda-forge, bioconda, defaults
39+
auto-update-conda: true
40+
auto-activate-base: false
41+
activate-environment: superdsm
42+
environment-file: superdsm.yml
43+
44+
- name: Run SuperDSM
45+
run: python -m "unittest"

‎.github/workflows/validate_pr.yml

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Validate pull request
2+
3+
on:
4+
pull_request:
5+
6+
jobs:
7+
8+
validate_branches:
9+
name: Validate branches
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
14+
- name: Validate branches
15+
if: github.base_ref == 'master' && github.head_ref != 'develop'
16+
run: |
17+
echo "Contributions should be made against the develop branch, see README.rst."
18+
exit 1
19+
20+
validate_conditional_checks:
21+
name: Validate conditional checks
22+
timeout-minutes: 1440
23+
runs-on: gpuagrohr-02
24+
container:
25+
image: ubuntu:22.04
26+
27+
steps:
28+
29+
- name: Install dependencies
30+
run: |
31+
apt-get update
32+
apt-get install -y ca-certificates
33+
34+
- uses: blend/[email protected]
35+
with:
36+
interval: 20s
37+
timeout: 1440m
38+
checks-yaml: |
39+
- job: 'Test: U2OS'
40+
paths:
41+
- .github/workflows/regressiontests.yml
42+
- superdsm/**
43+
- examples/**
44+
- superdsm.yml
45+
- tests/regression
46+
- job: 'Test: NIH3T3'
47+
paths:
48+
- .github/workflows/regressiontests.yml
49+
- superdsm/**
50+
- examples/**
51+
- superdsm.yml
52+
- tests/regression
53+
- job: 'Test: GOWT1-1'
54+
paths:
55+
- .github/workflows/regressiontests.yml
56+
- superdsm/**
57+
- examples/**
58+
- superdsm.yml
59+
- tests/regression
60+
- job: 'Test: GOWT1-2'
61+
paths:
62+
- .github/workflows/regressiontests.yml
63+
- superdsm/**
64+
- examples/**
65+
- superdsm.yml
66+
- tests/regression
67+
- job: 'Test suite'
68+
paths:
69+
- .github/workflows/testsuite.yml
70+
- superdsm/**
71+
- examples/**
72+
- superdsm.yml
73+
- tests/**

‎.readthedocs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: 2
33
build:
44
os: "ubuntu-20.04"
55
tools:
6-
python: "3.7"
6+
python: "3.9"
77

88
sphinx:
99
fail_on_warning: true

‎README.rst

+7-2
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ Dependency Version Considerations:
2727

2828
The file *superdsm.yml* specifies the Conda environment required to accurately reproduce the results from our publications. For most of the dependencies (maybe even all), newer versions are also known to work, however, it has been observed that using different versions might yield slightly different results. To enhance consistency, reproducibility, and `FAIRness <https://www.nature.com/articles/s41597-022-01710-x>`_, most dependency versions are thus pinned.
2929

30-
This also concerns BLAS, which is pinned to ``blas==1.0``. As an alternative to using Conda, *requirements.txt* specifies the required *pip* dependencies with pinned versions. However, to the best of our knowledge, it is not possible to request a specific BLAS version in *pip*, meaning that using *pip* instead of Conda is discouraged.
30+
This also concerns BLAS, which is pinned to ``blas==1.0``. As an alternative to using Conda, *requirements.txt* specifies a minimal set of required *pip* dependencies with pinned versions. However, to the best of our knowledge, it is not possible to request a specific BLAS version in *pip*, meaning that using *pip* instead of Conda is discouraged.
3131

3232
Note that our Conda package from Bioconda allows different dependency versions, because otherwise it would not be possible to use the package with newer versions of Python. Thus, when using our Conda package, keep in mind that sticking to the versions of the dependencies specified in *superdsm.yml* is recommended.
3333

@@ -36,10 +36,15 @@ Performance Considerations:
3636

3737
For full performance on both Intel and AMD CPUs, NumPy with MKL support must be used (instead of OpenBLAS which is often the default, see `details <https://stackoverflow.com/questions/62783262/why-is-numpy-with-ryzen-threadripper-so-much-slower-than-xeon>`_). When using Conda, this can be ensured by adding the dependency ``blas=1.0=mkl`` to the Conda environment (or ``blas=*=mkl`` to allow different versions).
3838

39-
To take advantage of the acceleration provided by MKL on AMD CPUs, the MKL version must be pinned to ``2020.0``. Both specifications are included in the Conda environment specified in *superdsm.yml*. In addition, the environment variable ``MKL_DEBUG_CPU_TYPE=5`` must be set.
39+
To take advantage of the acceleration provided by MKL on AMD CPUs, the MKL version must be pinned to ``2020.0``. Both specifications are included in the Conda environment specified in *superdsm.yml*. In addition, the environment variable ``MKL_DEBUG_CPU_TYPE=5`` must be set. This happens automatically if the CPU is recognized as an AMD CPU. If automatic recognition of the CPU vendor fails, a warning is displayed when the module is loaded, in which case ``MKL_DEBUG_CPU_TYPE=5`` should be set automatically for AMD GPUs.
4040

4141
Later versions of MKL do not support ``MKL_DEBUG_CPU_TYPE=5``, and previous versions do not offer the required APIs. Unfortunately, it looks like this particular version of MKL has been removed from PyPI (see `available versions <https://pypi.org/project/mkl/#history>`_), so it is not possible to gain the full performance on AMD CPUs using *pip* instead of Conda, and thus the version of MKL is not pinned in *requirements.txt* by default.
4242

43+
Contributions:
44+
""""""""""""""
45+
46+
Contributions should be made against the ``develop`` branch, so that the documentation build on readthedocs.io is triggered, the documentation is built and reviewed (see `here <https://superdsm.readthedocs.io/en/develop/>`_), before ``develop`` is merged into ``master``. This ensures that the ``master`` branch always has an up-to-date documentation.
47+
4348
Publications:
4449
"""""""""""""
4550

‎docs/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
sphinx-autorun>=1.1.1
22
sphinx-rtd-theme
3+
pillow==10.2 # newer versions seem to require numpy>=1.21 which conflicts with /requirements.txt

‎docs/source/conf.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
# -- Project information
44

55
project = 'SuperDSM'
6-
copyright = '2017-2023 Leonid Kostrykin, Biomedical Computer Vision Group, Heidelberg University'
6+
copyright = '2017-2024 Leonid Kostrykin, Biomedical Computer Vision Group, Heidelberg University'
77
author = 'Leonid Kostrykin'
88

99
# -- General configuration

‎docs/source/usage.rst

+17
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,23 @@ The pipeline data object ``data`` is a dictionary containing all the intermediat
6868
6969
The ``seg`` object returned by the :py:meth:`~superdsm.render.render_result_over_image` function is an RGB image (represented by a ``numpy.ndarray`` object) which can be visualized directly (e.g., using matplotlib) or saved for later use (e.g., using :py:meth:`~superdsm.io.imwrite`). Use :py:meth:`~superdsm.render.rasterize_labels` to obtain segmentation masks from the pipeline data object.
7070

71+
.. _env_variables:
72+
73+
Environment variables
74+
*********************
75+
76+
MKL_DEBUG_CPU_TYPE
77+
78+
To take advantage of the acceleration provided by MKL on AMD CPUs, the environment variable ``MKL_DEBUG_CPU_TYPE=5`` should be set when using an AMD CPU. This usually happens automatically, unless automatic recognition of the CPU vendor fails (and a warning is shown).
79+
80+
SUPERDSM_INTERMEDIATE_OUTPUT
81+
82+
Set ``SUPERDSM_INTERMEDIATE_OUTPUT=0`` to mute the intermediate console output.
83+
84+
SUPERDSM_NUM_CPUS
85+
86+
Set ``SUPERDSM_NUM_CPUS=8`` to use 8 CPU cores in batch processing. Defaults to 2. Ignored when used interactively.
87+
7188
.. _references:
7289

7390
References

‎examples/BBBC039/task.json

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"runnable": true,
3-
"num_cpus": 16,
43
"environ": {
54
"MKL_NUM_THREADS": 2,
65
"OPENBLAS_NUM_THREADS": 2

‎examples/GOWT1-1/task.json

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"runnable": false,
3-
"num_cpus": 16,
43
"environ": {
54
"MKL_NUM_THREADS": 2,
65
"OPENBLAS_NUM_THREADS": 2

‎examples/GOWT1-2/task.json

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"runnable": false,
3-
"num_cpus": 16,
43
"environ": {
54
"MKL_NUM_THREADS": 2,
65
"OPENBLAS_NUM_THREADS": 2

‎examples/NIH3T3/task.json

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"runnable": false,
3-
"num_cpus": 16,
43
"environ": {
54
"MKL_NUM_THREADS": 2,
65
"OPENBLAS_NUM_THREADS": 2

‎examples/U2OS/task.json

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"runnable": false,
3-
"num_cpus": 16,
43
"environ": {
54
"MKL_NUM_THREADS": 2,
65
"OPENBLAS_NUM_THREADS": 2

‎examples/fibroblast/task.json

-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
{
22
"runnable": false,
3-
"num_cpus": 16,
43
"environ": {
54
"MKL_NUM_THREADS": 2,
65
"OPENBLAS_NUM_THREADS": 2

‎requirements.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ scikit-image==0.18.1
44
ipython>=7.31.1
55
dill==0.3.2
66
ray>=0.8.7
7-
cvxopt==1.2.6
7+
cvxopt==1.2.7
88
cvxpy==1.1.13
99
matplotlib>=3.0
1010
mkl>=2020.0
11+
imagecodecs==2022.9.26

0 commit comments

Comments
 (0)