diff --git a/examples/submission/README.md b/examples/submission/README.md index dc77bb930..eccb499c8 100644 --- a/examples/submission/README.md +++ b/examples/submission/README.md @@ -111,7 +111,9 @@ docker run --rm --runtime=nvidia --ipc=host -v LOCAL_PATH_INPUT:/input:ro -v LOC `-v` flag mounts the directories between your local host and the container. `:ro` specifies that the folder mounted with `-v` has read-only permissions. Make sure that `LOCAL_PATH_INPUT` contains your test samples, -and `LOCAL_PATH_OUTPUT` is an output folder for saving the predictions. During test set submission this command will +and `LOCAL_PATH_OUTPUT` is an output folder for saving the predictions. +IMPORTANT: `LOCAL_PATH_INPUT` and `LOCAL_PATH_OUTPUT` must be full paths! Relative paths do not work. +During test set submission this command will be run on a private server managed by the organizers with mounting to the folders with final test data. Please test the docker on your local computer using the command above before uploading! diff --git a/examples/submission/nnUNet_submission/Dockerfile b/examples/submission/nnUNet_submission/Dockerfile index 01ad1c3a3..5d9a235a1 100644 --- a/examples/submission/nnUNet_submission/Dockerfile +++ b/examples/submission/nnUNet_submission/Dockerfile @@ -1,4 +1,4 @@ -FROM nvcr.io/nvidia/pytorch:20.08-py3 +FROM nvcr.io/nvidia/pytorch:21.07-py3 # Install some basic utilities and python RUN apt-get update \ @@ -16,4 +16,6 @@ ADD run_inference.py ./ # for ensemble model inference # ADD parameters_ensembling /parameters_ensembling/ -# ADD run_inference_ensembling.py ./ \ No newline at end of file +# ADD run_inference_ensembling.py ./run_inference.py + +ENV OMP_NUM_THREADS=1 diff --git a/examples/submission/nnUNet_submission/Dockerfile_cascade b/examples/submission/nnUNet_submission/Dockerfile_cascade new file mode 100644 index 000000000..008395224 --- /dev/null +++ b/examples/submission/nnUNet_submission/Dockerfile_cascade @@ -0,0 +1,20 @@ +FROM nvcr.io/nvidia/pytorch:21.07-py3 + +# Install some basic utilities and python +RUN apt-get update \ + && apt-get install -y python3-pip python3-dev \ + && cd /usr/local/bin \ + && ln -s /usr/bin/python3 python \ + && pip3 install --upgrade pip + +# for single model inference +ADD run_inference_cascade.py ./run_inference.py +ADD parameters /parameters/ + +# install nnunet +RUN pip install git+https://github.com/MIC-DKFZ/nnUNet.git + +# needed for the cascade trainer, otherwise it will crash. Dumb coding on Fabians part +RUN mkdir /results +ENV RESULTS_FOLDER="/results" +ENV OMP_NUM_THREADS=1 diff --git a/examples/submission/nnUNet_submission/run_inference_cascade.py b/examples/submission/nnUNet_submission/run_inference_cascade.py new file mode 100644 index 000000000..43adcfd1d --- /dev/null +++ b/examples/submission/nnUNet_submission/run_inference_cascade.py @@ -0,0 +1,62 @@ +import shutil + +if __name__ == '__main__': + # this will be changed to /input for the docker + input_folder = '/input' + + # this will be changed to /output for the docker + output_folder = '/output' + + # this will be changed to /parameters/X for the docker + parameter_folder_cascade_fullres = '/parameters/3d_cascade_fullres' + parameter_folder_lowres = '/parameters/3d_lowres' + + from nnunet.inference.predict import predict_cases + from batchgenerators.utilities.file_and_folder_operations import subfiles, join, maybe_mkdir_p + + input_files = subfiles(input_folder, suffix='.nii.gz', join=False) + + # in the parameters folder are five models (fold_X) traines as a cross-validation. We use them as an ensemble for + # prediction + folds_cascade_fullres = (0, 1, 2, 3, 4) + folds_lowres = (0, 1, 2, 3, 4) + + # setting this to True will make nnU-Net use test time augmentation in the form of mirroring along all axes. This + # will increase inference time a lot at small gain, so we turn that off here (you do whatever you want) + do_tta = False + + # does inference with mixed precision. Same output, twice the speed on Turing and newer. It's free lunch! + mixed_precision = True + + # This will make nnU-Net save the softmax probabilities. We need them for ensembling the configurations. Note + # that ensembling the 5 folds of each configurationis done BEFORE saving the softmax probabilities + save_npz = False # no ensembling here + + # predict with 3d_lowres + output_folder_lowres = join(output_folder, '3d_lowres') + maybe_mkdir_p(output_folder_lowres) + output_files_lowres = [join(output_folder_lowres, i) for i in input_files] + + predict_cases(parameter_folder_lowres, [[join(input_folder, i)] for i in input_files], output_files_lowres, folds_lowres, + save_npz=save_npz, num_threads_preprocessing=2, num_threads_nifti_save=2, segs_from_prev_stage=None, + do_tta=do_tta, mixed_precision=mixed_precision, overwrite_existing=True, all_in_gpu=False, + step_size=0.5) + + # predict with 3d_fullres + output_folder_cascade_fullres = output_folder + maybe_mkdir_p(output_folder_cascade_fullres) + output_files_cascade_fullres = [join(output_folder_cascade_fullres, i) for i in input_files] + + # CAREFUL! I set all_in_gpu=True and step_size=0.75. These are not the defaults! + predict_cases(parameter_folder_cascade_fullres, [[join(input_folder, i)] for i in input_files], + output_files_cascade_fullres, folds_cascade_fullres, + save_npz=save_npz, num_threads_preprocessing=2, num_threads_nifti_save=2, + segs_from_prev_stage=[join(output_folder_lowres, i) for i in input_files], + do_tta=do_tta, mixed_precision=mixed_precision, overwrite_existing=True, all_in_gpu=True, + step_size=0.75) + + # cleanup + shutil.rmtree(output_folder_lowres) + + # done! + diff --git a/kits21/evaluation/metrics.py b/kits21/evaluation/metrics.py index 9ebdba770..09420b980 100644 --- a/kits21/evaluation/metrics.py +++ b/kits21/evaluation/metrics.py @@ -144,13 +144,13 @@ def evaluate_predictions(folder_with_predictions: str, num_processes: int = 8, w with open(join(folder_with_predictions, 'evaluation.csv'), "w") as f: f.write("caseID,Dice_kidney,Dice_masses,Dice_tumor,SD_kidney,SD_masses,SD_tumor\n") for i, c in enumerate(caseids): - f.write("%s,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f\n" % ( + f.write("%s,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f\n" % ( c, metrics[i, 0, 0], metrics[i, 1, 0], metrics[i, 2, 0], metrics[i, 0, 1], metrics[i, 1, 1], metrics[i, 2, 1], )) mean_metrics = metrics.mean(0) - f.write("average,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f" % ( + f.write("average,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f" % ( mean_metrics[0, 0], mean_metrics[1, 0], mean_metrics[2, 0], mean_metrics[0, 1], mean_metrics[1, 1], mean_metrics[2, 1], )) @@ -178,13 +178,13 @@ def evaluate_predictions_on_samples(folder_with_predictions: str, num_processes: with open(join(folder_with_predictions, 'evaluation_samples.csv'), "w") as f: f.write("caseID,Dice_kidney,Dice_masses,Dice_tumor,SD_kidney,SD_masses,SD_tumor\n") for i, c in enumerate(caseids): - f.write("%s,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f\n" % ( + f.write("%s,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f\n" % ( c, metrics[i, 0, 0], metrics[i, 1, 0], metrics[i, 2, 0], metrics[i, 0, 1], metrics[i, 1, 1], metrics[i, 2, 1], )) mean_metrics = metrics.mean(0) - f.write("average,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f" % ( + f.write("average,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f" % ( mean_metrics[0, 0], mean_metrics[1, 0], mean_metrics[2, 0], mean_metrics[0, 1], mean_metrics[1, 1], mean_metrics[2, 1], )) diff --git a/kits21/evaluation/nnUNet_summary.csv b/kits21/evaluation/nnUNet_summary.csv new file mode 100644 index 000000000..2bdb5f836 --- /dev/null +++ b/kits21/evaluation/nnUNet_summary.csv @@ -0,0 +1,7 @@ +,Dice_kidney,Dice_masses,Dice_tumor,SD_kidney,SD_masses,SD_tumor +nnU-Net_3d_lowres,0.9683,0.8702,0.8508,0.9272,0.7507,0.7347 +nnU-Net_3d_fullres,0.9666,0.8618,0.8493,0.9336,0.7532,0.7371 +nnU-Net_3d_cascade_fullres,0.9747,0.8799,0.8491,0.9453,0.7714,0.7393 +dummy,1,1,0,0,1,1 +dummy2,0,1,1,1,1,0 +dummy3,0,1,1,1,1,0 diff --git a/kits21/evaluation/ranking.py b/kits21/evaluation/ranking.py new file mode 100644 index 000000000..935a82ec8 --- /dev/null +++ b/kits21/evaluation/ranking.py @@ -0,0 +1,69 @@ +from typing import Callable +import numpy as np +import scipy.stats as ss + + +def rank_then_aggregate(data: np.ndarray, aggr_fn: Callable = np.mean): + """ + data must be (algos x metrics). Higher values must mean better result (so Dice is OK, but not HD95!). + If you want to use this code with HD95 you need to invert it as a preprocessing step + :param data: + :param aggr_fn: + :return: + """ + ranked = np.apply_along_axis(ss.rankdata, 0, -data, 'min') + aggregated = aggr_fn(ranked, axis=-1) + final_rank = ss.rankdata(aggregated, 'min') + return final_rank, aggregated + + +def rank_participants(summary_csv: str, output_csv_file: str) -> None: + results = np.loadtxt(summary_csv, dtype=str, delimiter=',', skiprows=1) + teams = results[:, 0] + + assert len(np.unique(teams)) == len(teams), 'Some teams have identical names, please fix' + metrics = results[:, 1:].astype(float) + + assert metrics.shape[1] == 6, 'expected 6 metrics, got %d' % metrics.shape[1] + + mean_dice = np.mean(metrics[:, :3], axis=1, keepdims=True) + mean_sd = np.mean(metrics[:, 3:], axis=1, keepdims=True) + + mean_metrics = np.concatenate((mean_dice, mean_sd), axis=1) + ranks, aggregated = rank_then_aggregate(mean_metrics) + + # now clean up ties. This is not the cleanest implementation, but eh + rank = 1 + while rank < (len(teams) + 1): + num_teams_on_that_rank = sum(ranks == rank) + if num_teams_on_that_rank <= 1: + rank += 1 + continue + else: + # tumor dice is tie breaker + teams_mask = ranks == rank + tumor_dice_scores = metrics[teams_mask, 2:3] + new_ranks = rank_then_aggregate(tumor_dice_scores)[0] - 1 + rank + if len(np.unique(new_ranks)) == 1: + print("WARNING: Cannot untie ranks of these teams... tumor_dice_scores and ranks are identical:") + print('team names:', teams[teams_mask]) + rank += 1 + if len(np.unique(new_ranks)) != sum(teams_mask): + ranks[teams_mask] = new_ranks + continue + ranks[teams_mask] = new_ranks + rank += 1 + + # print ranking + sorting = np.argsort(ranks) + with open(output_csv_file, 'w') as f: + f.write('team_name,final_rank,mean_rank,mean_dice,mean_sd,tumor_dice\n') + for i in sorting: + f.write('%s,%d,%.4f,%.8f,%.8f,%.8f\n' % (teams[i], ranks[i], aggregated[i], mean_dice[i], mean_sd[i], + metrics[i, 2])) + + +if __name__ == '__main__': + summary_file = 'nnUNet_summary.csv' # follow the example csv provided in this folder! + output_file = 'kits2021_ranking.csv' + rank_participants(summary_file, output_file)