wip add unittests

Author: maisli

evaluateInstanceSegmentation/__init__.py

@@ -1,2 +1,2 @@
-from .evaluate import evaluate_file
+from .evaluate import evaluate_file, evaluate_volume
 from .summarize import summarize_metric_dict

evaluateInstanceSegmentation/evaluate.py

@@ -37,14 +37,19 @@ class Metrics:
     def __init__(self, fn):
         self.metricsDict = {}
         self.metricsArray = []
-        self.fn = fn
-        self.outFl = open(self.fn+".txt", 'w')
+        if fn is not None:
+            self.fn = fn
+            self.outFl = open(self.fn + '.txt', 'w')
+        else:
+            self.fn = None
+            self.outFl = None
 
     def save(self):
-        self.outFl.close()
-        logger.info("saving %s", self.fn)
-        tomlFl = open(self.fn+".toml", 'w')
-        toml.dump(self.metricsDict, tomlFl)
+        if self.outFl is not None:
+            self.outFl.close()
+            logger.info("saving %s", self.fn)
+            tomlFl = open(self.fn+".toml", 'w')
+            toml.dump(self.metricsDict, tomlFl)
 
     def addTable(self, name, dct=None):
         levels = name.split(".")
@@ -67,8 +72,9 @@ def getTable(self, name, dct=None):
             return self.getTable(name, dct=dct[levels[0]])
 
     def addMetric(self, table, name, value):
-        as_str = "{}: {}".format(name, value)
-        self.outFl.write(as_str+"\n")
+        if self.outFl is not None:
+            as_str = "{}: {}".format(name, value)
+            self.outFl.write(as_str+"\n")
         self.metricsArray.append(value)
         tbl = self.getTable(table)
         tbl[name] = value
@@ -642,7 +648,7 @@ def evaluate_volume(gt_labels, pred_labels, outFn,
         ap = precision * recall
         aps.append(ap)
         if (precision + recall) > 0:
-            fscore = (2. * precision * recall) / max(1, precision + recall)
+            fscore = (2. * precision * recall) / (precision + recall)
         else:
             fscore = 0.0
         fscores.append(fscore)
@@ -696,7 +702,7 @@ def evaluate_volume(gt_labels, pred_labels, outFn,
                 for gt_i in np.arange(1, num_gt_labels + 1):
                     if gt_i in max_gt_ind_unique:
                         pred_union = np.zeros(
-                                pred_labels_rel.shape[1:], 
+                                pred_labels_rel.shape[1:],
                                 dtype=pred_labels_rel.dtype)
                         for pred_i in np.arange(num_pred_labels + 1)[max_gt_ind == gt_i]:
                             mask = pred_labels_rel[pred_i - 1] > 0
@@ -708,13 +714,11 @@ def evaluate_volume(gt_labels, pred_labels, outFn,
             else:
                 # if gt has overlapping instances, but not prediction
                 if len(gt_labels_rel.shape) > len(pred_labels_rel.shape):
-                    print("gt cov for overlapping gt, but not pred")
                     for i in range(1, recallMat.shape[0]):
                         gt_cov.append(np.sum(recallMat[i, max_gt_ind==i]))
                 # if none has overlapping instances
                 else:
                     gt_cov = np.sum(recallMat[1:, 1:], axis=1)
-            print("gt cov: ", gt_cov)
             gt_skel_coverage = np.mean(gt_cov)
             metrics.addMetric(tblNameGen, "gt_skel_coverage", gt_cov)
             metrics.addMetric(tblNameGen, "avg_gt_skel_coverage", gt_skel_coverage)

tests/test_metrics.py

@@ -0,0 +1,178 @@
+import unittest
+import os
+import numpy as np
+from skimage.morphology import skeletonize_3d
+
+from evaluateInstanceSegmentation import evaluate_file, evaluate_volume
+
+# work in progress
+class TestMetrics(unittest.TestCase):
+    # how to name stuff, convex and tubular instead of nuclei and neuron?
+    def set_expected(self, num_gt, num_pred, avg_gt_cov, avg_tp_cov, avg_f1_cov, gt_cov, tp_cov,
+            avAP59, avAP19, avFscore59, avFscore19,
+            tp_0_5, fp_0_5, fn_0_5, fs_0_5, fm_0_5):
+        # define metric dictionary structure for expected values
+        expected = {
+                "general": {}, 
+                "confusion_matrix": {"th_0_5":{}}
+                }
+        # set values
+        expected["general"]["Num GT"] = num_gt
+        expected["general"]["Num Pred"] = num_pred
+        expected["general"]["avg_gt_skel_coverage"] = avg_gt_cov
+        expected["general"]["avg_tp_skel_coverage"] = avg_tp_cov
+        expected["general"]["avg_f1_cov_score"] = avg_f1_cov
+        expected["general"]["gt_skel_coverage"] = gt_cov
+        expected["general"]["tp_skel_coverage"] = tp_cov
+        expected["confusion_matrix"]["avAP59"] = avAP59
+        expected["confusion_matrix"]["avAP19"] = avAP19
+        expected["confusion_matrix"]["avFscore59"] = avFscore59
+        expected["confusion_matrix"]["avFscore19"] = avFscore19
+        expected["confusion_matrix"]["th_0_5"]["AP_TP"] = tp_0_5
+        expected["confusion_matrix"]["th_0_5"]["AP_FP"] = fp_0_5
+        expected["confusion_matrix"]["th_0_5"]["AP_FN"] = fn_0_5
+        expected["confusion_matrix"]["th_0_5"]["false_split"] = fs_0_5
+        expected["confusion_matrix"]["th_0_5"]["false_merge"] = fm_0_5
+        
+        return expected
+
+
+    def check_results(self, results, expected):
+        # check general
+        res = results["general"]
+        exp = expected["general"]
+        self.assertEqual(res["Num GT"], exp["Num GT"])
+        self.assertEqual(res["Num Pred"], exp["Num Pred"])
+        self.assertEqual(round(res["avg_gt_skel_coverage"], 4),
+                round(exp["avg_gt_skel_coverage"], 4))
+        self.assertEqual(round(res["avg_tp_skel_coverage"], 4),
+                round(exp["avg_tp_skel_coverage"], 4))
+        self.assertEqual(res["avg_f1_cov_score"], exp["avg_f1_cov_score"])
+        self.assertListEqual(list(np.round(res["gt_skel_coverage"], 4)), 
+                list(np.round(exp["gt_skel_coverage"], 4)))
+        self.assertListEqual(list(res["tp_skel_coverage"]), list(exp["tp_skel_coverage"]))
+        # check confusion table
+        res = results["confusion_matrix"]
+        exp = expected["confusion_matrix"]
+        self.assertEqual(res["avAP59"], exp["avAP59"])
+        self.assertEqual(res["avAP19"], exp["avAP19"])
+        self.assertEqual(res["avFscore59"], exp["avFscore59"])
+        self.assertEqual(res["avFscore19"], exp["avFscore19"])
+        # check error quantities for confusion table at threshold 0.5
+        res = results["confusion_matrix"]["th_0_5"]
+        exp = expected["confusion_matrix"]["th_0_5"]
+        self.assertEqual(res["AP_TP"], exp["AP_TP"])
+        self.assertEqual(res["AP_FP"], exp["AP_FP"])
+        self.assertEqual(res["AP_FN"], exp["AP_FN"])
+        self.assertEqual(res["false_split"], exp["false_split"])
+        self.assertEqual(res["false_merge"], exp["false_merge"])
+
+
+    def run_test_case(self, config, gt, pred, expected):
+        
+        result_dict = evaluate_volume(gt, pred, config["outFn"],
+                config["localization_criterion"], config["assignment_strategy"],
+                config["evaluate_false_labels"], config["unique_false_labels"],
+                config["add_general_metrics"],
+                config["visualize"], config["visualize_type"], 
+                config["overlapping_inst"], config["partly"])
+        print(result_dict)
+        self.check_results(result_dict, expected)
+    
+    def test_2d_nuclei(self):
+        print("todo: test 2d nuclei")
+
+
+    def test_3d_nuclei(self):
+        print("todo: test 2d nuclei")
+
+
+    def test_3d_neuron(self):
+        gt = np.zeros((2, 30, 30, 30), dtype=np.int32)
+        gt[0, 14:17, 14:17, 5:25] = 1
+        gt[1, 14:17, 5:25, 14:17] = 2
+
+        print(np.sum(gt==1), np.sum(gt==2), np.sum(np.sum(gt>0, axis=0) > 1))
+        
+        # set parameters
+        config = {
+                "outFn": None,
+                "localization_criterion": "cldice",
+                "assignment_strategy": "greedy",
+                "add_general_metrics": ["avg_gt_skel_coverage", 
+                    "avg_f1_cov_score", "avg_tp_skel_coverage"],
+                "evaluate_false_labels": True,
+                "unique_false_labels": False,
+                "visualize": False,
+                "visualize_type": None,
+                "overlapping_inst": True,
+                "partly": False
+                }
+
+        # test case 1: perfect segmentation
+        # (1.1) pred + gt overlaps
+        pred = gt.copy()
+        self.run_test_case(config, gt, pred,
+                self.set_expected(2, 2, 1.0, 1.0, 1.0, [1.0, 1.0], [1.0, 1.0], 
+                    1.0, 1.0, 1.0, 1.0,
+                    2, 0, 0, 0, 0)
+                )
+
+        # (1.2) gt overlaps
+        pred = np.max(pred, axis=0)
+        config["overlapping_inst"] = False
+        # set expected values
+        gt_cov = np.array([15/18.0, 1.0], dtype=np.float32)
+        avg_gt_cov = np.mean(gt_cov)
+        avg_f1_cov = np.mean([avg_gt_cov, 1.0])
+        self.run_test_case(config, gt, pred,
+                self.set_expected(2, 2, avg_gt_cov, avg_gt_cov, avg_f1_cov, 
+                    gt_cov, gt_cov,
+                    0.925, 1.0, 0.95, 1.0,
+                    2, 0, 0, 0, 0)
+                )
+        
+        # (1.3) no overlaps
+        gt = np.max(gt, axis=0)
+        self.run_test_case(config, gt, pred,
+                self.set_expected(2, 2, 1.0, 1.0, 1.0, [1.0, 1.0], [1.0, 1.0], 
+                    1.0, 1.0, 1.0, 1.0,
+                    2, 0, 0, 0, 0)
+                )
+
+        # test case 2: erroneous segmentation
+        gt = np.zeros((3, 30, 30, 30), dtype=np.int32)
+        gt[0, 14:17, 14:17, 5:25] = 1
+        gt[1, 14:17, 5:25, 14:17] = 2
+        gt[2, 5:8, 5:25, 20:23] = 3
+
+        pred = np.zeros((5, 30, 30, 30), dtype=np.int32)
+        pred[0, 14:17, 14:17, 5:25] = 1
+        pred[0, 14:17, 5:20, 14:17] = 1
+        pred[1, 25:30, 25:30, 25:30] = 2
+        pred[2, 5:8, 5:11, 20:23] = 3
+        pred[3, 5:8, 19:22, 20:23] = 4
+        pred[4, 1:5, 1:5, 1:5] = 5
+
+        # (2.1) pred + gt overlaps
+        config["overlapping_inst"] = False
+        gt_cov = np.array([1.0, 14/18.0, 8/18.0], dtype=np.float32)
+        avg_gt_cov = np.mean(gt_cov)
+        tp_cov = np.array([1.0, 14/18.0], dtype=np.float32)
+        avg_tp_cov = np.mean(tp_cov)
+        ap19 = np.mean([4/15.0,] * 4 + [1/15.0,] * 3 + [0.0, 0.0])
+        ap59 = np.mean([1/15.0,] * 6 + [0.0,] * 4)
+        fscore19 = np.mean([0.5,] * 4 + [0.25,] * 3 + [0.0, 0.0])
+        fscore59 = np.mean([0.25,] * 6 + [0.0,] * 4)
+        avg_f1_cov = np.mean([avg_gt_cov, fscore19])
+        self.run_test_case(config, gt, pred,
+                self.set_expected(3, 5, avg_gt_cov, avg_tp_cov, avg_f1_cov,
+                    gt_cov, tp_cov,
+                    ap59, ap19, fscore59, fscore19,
+                    1, 4, 2, 2, 1)
+                )
+
+
+if __name__ == '__main__':
+    unittest.main()
+