increased precision of metrics in csv files, added code for ranking

examples/submission/README.md

@@ -111,7 +111,9 @@ docker run --rm --runtime=nvidia --ipc=host -v LOCAL_PATH_INPUT:/input:ro -v LOC
 
 `-v` flag mounts the directories between your local host and the container. `:ro` specifies that the folder mounted
 with `-v` has read-only permissions. Make sure that `LOCAL_PATH_INPUT` contains your test samples,
-and `LOCAL_PATH_OUTPUT` is an output folder for saving the predictions. During test set submission this command will
+and `LOCAL_PATH_OUTPUT` is an output folder for saving the predictions. 
+IMPORTANT: `LOCAL_PATH_INPUT` and `LOCAL_PATH_OUTPUT` must be full paths! Relative paths do not work.
+During test set submission this command will
 be run on a private server managed by the organizers with mounting to the folders with final test data. Please test
 the docker on your local computer using the command above before uploading!
 

examples/submission/nnUNet_submission/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/pytorch:20.08-py3
+FROM nvcr.io/nvidia/pytorch:21.07-py3
 
 # Install some basic utilities and python
 RUN apt-get update \
@@ -16,4 +16,6 @@ ADD run_inference.py ./
 
 # for ensemble model inference
 # ADD parameters_ensembling /parameters_ensembling/
-# ADD run_inference_ensembling.py ./
+# ADD run_inference_ensembling.py ./run_inference.py
+
+ENV OMP_NUM_THREADS=1

examples/submission/nnUNet_submission/Dockerfile_cascade

@@ -0,0 +1,20 @@
+FROM nvcr.io/nvidia/pytorch:21.07-py3
+
+# Install some basic utilities and python
+RUN apt-get update \
+  && apt-get install -y python3-pip python3-dev \
+  && cd /usr/local/bin \
+  && ln -s /usr/bin/python3 python \
+  && pip3 install --upgrade pip
+
+# for single model inference
+ADD run_inference_cascade.py ./run_inference.py
+ADD parameters /parameters/
+
+# install nnunet
+RUN pip install git+https://github.com/MIC-DKFZ/nnUNet.git
+
+# needed for the cascade trainer, otherwise it will crash. Dumb coding on Fabians part
+RUN mkdir /results
+ENV RESULTS_FOLDER="/results"
+ENV OMP_NUM_THREADS=1

examples/submission/nnUNet_submission/run_inference_cascade.py

@@ -0,0 +1,62 @@
+import shutil
+
+if __name__ == '__main__':
+    # this will be changed to /input for the docker
+    input_folder = '/input'
+
+    # this will be changed to /output for the docker
+    output_folder = '/output'
+
+    # this will be changed to /parameters/X for the docker
+    parameter_folder_cascade_fullres = '/parameters/3d_cascade_fullres'
+    parameter_folder_lowres = '/parameters/3d_lowres'
+
+    from nnunet.inference.predict import predict_cases
+    from batchgenerators.utilities.file_and_folder_operations import subfiles, join, maybe_mkdir_p
+
+    input_files = subfiles(input_folder, suffix='.nii.gz', join=False)
+
+    # in the parameters folder are five models (fold_X) traines as a cross-validation. We use them as an ensemble for
+    # prediction
+    folds_cascade_fullres = (0, 1, 2, 3, 4)
+    folds_lowres = (0, 1, 2, 3, 4)
+
+    # setting this to True will make nnU-Net use test time augmentation in the form of mirroring along all axes. This
+    # will increase inference time a lot at small gain, so we turn that off here (you do whatever you want)
+    do_tta = False
+
+    # does inference with mixed precision. Same output, twice the speed on Turing and newer. It's free lunch!
+    mixed_precision = True
+
+    # This will make nnU-Net save the softmax probabilities. We need them for ensembling the configurations. Note
+    # that ensembling the 5 folds of each configurationis done BEFORE saving the softmax probabilities
+    save_npz = False  # no ensembling here
+
+    # predict with 3d_lowres
+    output_folder_lowres = join(output_folder, '3d_lowres')
+    maybe_mkdir_p(output_folder_lowres)
+    output_files_lowres = [join(output_folder_lowres, i) for i in input_files]
+
+    predict_cases(parameter_folder_lowres, [[join(input_folder, i)] for i in input_files], output_files_lowres, folds_lowres,
+                  save_npz=save_npz, num_threads_preprocessing=2, num_threads_nifti_save=2, segs_from_prev_stage=None,
+                  do_tta=do_tta, mixed_precision=mixed_precision, overwrite_existing=True, all_in_gpu=False,
+                  step_size=0.5)
+
+    # predict with 3d_fullres
+    output_folder_cascade_fullres = output_folder
+    maybe_mkdir_p(output_folder_cascade_fullres)
+    output_files_cascade_fullres = [join(output_folder_cascade_fullres, i) for i in input_files]
+
+    # CAREFUL! I set all_in_gpu=True and step_size=0.75. These are not the defaults!
+    predict_cases(parameter_folder_cascade_fullres, [[join(input_folder, i)] for i in input_files],
+                  output_files_cascade_fullres, folds_cascade_fullres,
+                  save_npz=save_npz, num_threads_preprocessing=2, num_threads_nifti_save=2,
+                  segs_from_prev_stage=[join(output_folder_lowres, i) for i in input_files],
+                  do_tta=do_tta, mixed_precision=mixed_precision, overwrite_existing=True, all_in_gpu=True,
+                  step_size=0.75)
+
+    # cleanup
+    shutil.rmtree(output_folder_lowres)
+
+    # done!
+

kits21/evaluation/metrics.py

@@ -144,13 +144,13 @@ def evaluate_predictions(folder_with_predictions: str, num_processes: int = 8, w
         with open(join(folder_with_predictions, 'evaluation.csv'), "w") as f:
             f.write("caseID,Dice_kidney,Dice_masses,Dice_tumor,SD_kidney,SD_masses,SD_tumor\n")
             for i, c in enumerate(caseids):
-                f.write("%s,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f\n" % (
+                f.write("%s,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f\n" % (
                     c,
                     metrics[i, 0, 0], metrics[i, 1, 0], metrics[i, 2, 0],
                     metrics[i, 0, 1], metrics[i, 1, 1], metrics[i, 2, 1],
                 ))
             mean_metrics = metrics.mean(0)
-            f.write("average,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f" % (
+            f.write("average,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f" % (
                 mean_metrics[0, 0], mean_metrics[1, 0], mean_metrics[2, 0],
                 mean_metrics[0, 1], mean_metrics[1, 1], mean_metrics[2, 1],
             ))
@@ -178,13 +178,13 @@ def evaluate_predictions_on_samples(folder_with_predictions: str, num_processes:
         with open(join(folder_with_predictions, 'evaluation_samples.csv'), "w") as f:
             f.write("caseID,Dice_kidney,Dice_masses,Dice_tumor,SD_kidney,SD_masses,SD_tumor\n")
             for i, c in enumerate(caseids):
-                f.write("%s,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f\n" % (
+                f.write("%s,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f\n" % (
                     c,
                     metrics[i, 0, 0], metrics[i, 1, 0], metrics[i, 2, 0],
                     metrics[i, 0, 1], metrics[i, 1, 1], metrics[i, 2, 1],
                 ))
             mean_metrics = metrics.mean(0)
-            f.write("average,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f,%0.4f" % (
+            f.write("average,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f,%0.8f" % (
                 mean_metrics[0, 0], mean_metrics[1, 0], mean_metrics[2, 0],
                 mean_metrics[0, 1], mean_metrics[1, 1], mean_metrics[2, 1],
             ))

kits21/evaluation/nnUNet_summary.csv

@@ -0,0 +1,7 @@
+,Dice_kidney,Dice_masses,Dice_tumor,SD_kidney,SD_masses,SD_tumor
+nnU-Net_3d_lowres,0.9683,0.8702,0.8508,0.9272,0.7507,0.7347
+nnU-Net_3d_fullres,0.9666,0.8618,0.8493,0.9336,0.7532,0.7371
+nnU-Net_3d_cascade_fullres,0.9747,0.8799,0.8491,0.9453,0.7714,0.7393
+dummy,1,1,0,0,1,1
+dummy2,0,1,1,1,1,0
+dummy3,0,1,1,1,1,0

kits21/evaluation/ranking.py

@@ -0,0 +1,69 @@
+from typing import Callable
+import numpy as np
+import scipy.stats as ss
+
+
+def rank_then_aggregate(data: np.ndarray, aggr_fn: Callable = np.mean):
+    """
+    data must be (algos x metrics). Higher values must mean better result (so Dice is OK, but not HD95!).
+    If you want to use this code with HD95 you need to invert it as a preprocessing step
+    :param data:
+    :param aggr_fn:
+    :return:
+    """
+    ranked = np.apply_along_axis(ss.rankdata, 0, -data, 'min')
+    aggregated = aggr_fn(ranked, axis=-1)
+    final_rank = ss.rankdata(aggregated, 'min')
+    return final_rank, aggregated
+
+
+def rank_participants(summary_csv: str, output_csv_file: str) -> None:
+    results = np.loadtxt(summary_csv, dtype=str, delimiter=',', skiprows=1)
+    teams = results[:, 0]
+
+    assert len(np.unique(teams)) == len(teams), 'Some teams have identical names, please fix'
+    metrics = results[:, 1:].astype(float)
+
+    assert metrics.shape[1] == 6, 'expected 6 metrics, got %d' % metrics.shape[1]
+
+    mean_dice = np.mean(metrics[:, :3], axis=1, keepdims=True)
+    mean_sd = np.mean(metrics[:, 3:], axis=1, keepdims=True)
+
+    mean_metrics = np.concatenate((mean_dice, mean_sd), axis=1)
+    ranks, aggregated = rank_then_aggregate(mean_metrics)
+
+    # now clean up ties. This is not the cleanest implementation, but eh
+    rank = 1
+    while rank < (len(teams) + 1):
+        num_teams_on_that_rank = sum(ranks == rank)
+        if num_teams_on_that_rank <= 1:
+            rank += 1
+            continue
+        else:
+            # tumor dice is tie breaker
+            teams_mask = ranks == rank
+            tumor_dice_scores = metrics[teams_mask, 2:3]
+            new_ranks = rank_then_aggregate(tumor_dice_scores)[0] - 1 + rank
+            if len(np.unique(new_ranks)) == 1:
+                print("WARNING: Cannot untie ranks of these teams... tumor_dice_scores and ranks are identical:")
+                print('team names:', teams[teams_mask])
+                rank += 1
+            if len(np.unique(new_ranks)) != sum(teams_mask):
+                ranks[teams_mask] = new_ranks
+                continue
+            ranks[teams_mask] = new_ranks
+            rank += 1
+
+    # print ranking
+    sorting = np.argsort(ranks)
+    with open(output_csv_file, 'w') as f:
+        f.write('team_name,final_rank,mean_rank,mean_dice,mean_sd,tumor_dice\n')
+        for i in sorting:
+            f.write('%s,%d,%.4f,%.8f,%.8f,%.8f\n' % (teams[i], ranks[i], aggregated[i], mean_dice[i], mean_sd[i],
+                                                     metrics[i, 2]))
+
+
+if __name__ == '__main__':
+    summary_file = 'nnUNet_summary.csv'  # follow the example csv provided in this folder!
+    output_file = 'kits2021_ranking.csv'
+    rank_participants(summary_file, output_file)