Release v0.10.1

promort/VERSION

@@ -1 +1 @@
-0.10.0
+0.10.1

promort/reviews_manager/management/commands/build_clinical_reviews_worklist.py

@@ -23,6 +23,7 @@
 from slides_manager.models import SlideEvaluation
 from reviews_manager.models import ROIsAnnotation, ClinicalAnnotation, ClinicalAnnotationStep, ReviewsComparison
 
+from csv import DictReader, DictWriter
 from uuid import uuid4
 import logging, random
 from datetime import datetime
@@ -34,16 +35,21 @@ class Command(BaseCommand):
     help = 'build second reviewers worklist based on existing ROIs annotations'
 
     def add_arguments(self, parser):
+        parser.add_argument('--worklist-file', dest='worklist', type=str, default=None,
+                            help='a CSV file containing the list of the ROIs annotations that will be processed')
         parser.add_argument('--reviewers-count', dest='reviewers_count', type=int, default=1,
                             help='the number of clinical reviews created for each ROIs annotation')
 
     def _get_clinical_manager_users(self):
         clinical_managers_group = Group.objects.get(name=DEFAULT_GROUPS['clinical_manager']['name'])
         return clinical_managers_group.user_set.all()
 
-    def _get_rois_annotations_list(self):
+    def _get_rois_annotations_list(self, rois_annotations_list=None):
         linked_annotations = [ca.rois_review.label for ca in ClinicalAnnotation.objects.all()]
-        return ROIsAnnotation.objects.exclude(label__in=linked_annotations)
+        if rois_annotations_list is None:
+            return ROIsAnnotation.objects.exclude(label__in=linked_annotations)
+        else:
+            return ROIsAnnotation.objects.filter(label__in=rois_annotations_list).exclude(label__in=linked_annotations)
 
     def _select_clinical_reviewers(self, rois_annotation, clinical_managers, reviewers_count):
         if reviewers_count >= len(clinical_managers):
@@ -116,14 +122,25 @@ def _create_clinical_annotation_step(self, clinical_annotation_obj, rois_annotat
     #     logger.info('Create Reviews Comparison for Clinical Annotation Steps %s and %s',
     #                 review_step_1_obj.label, review_step_2_obj.label)
     #     return reviews_comparison_obj
+
+    def load_roi_annotations_list(self, worklist_file):
+        with open(worklist_file) as f:
+            reader = DictReader(f)
+            return [row['annotation_label'] for row in reader]
+
 
     def handle(self, *args, **opts):
         logger.info('=== Starting clinical annotations worklist creation ===')
         reviewers_count = opts['reviewers_count']
+        worklist_file = opts['worklist']
         clinical_annotations_manager = self._get_clinical_manager_users()
         if len(clinical_annotations_manager) == 0:
             raise CommandError('There must be at least 1 user with Clinical Annotations Manager role')
-        rois_annotations = self._get_rois_annotations_list()
+        if worklist_file:
+            rois_annotations_list = self.load_roi_annotations_list(worklist_file)
+        else:
+            rois_annotations_list = None
+        rois_annotations = self._get_rois_annotations_list(rois_annotations_list)
         if len(rois_annotations) == 0:
             logger.info('There are no ROIs Annotations to process')
         for r_ann in rois_annotations:

promort/reviews_manager/management/commands/get_rois_annotation_steps_data.py

@@ -0,0 +1,88 @@
+#  Copyright (c) 2022, CRS4
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy of
+#  this software and associated documentation files (the "Software"), to deal in
+#  the Software without restriction, including without limitation the rights to
+#  use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+#  the Software, and to permit persons to whom the Software is furnished to do so,
+#  subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+#  FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+#  COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+#  IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+#  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from django.core.management.base import BaseCommand
+from django.core.paginator import Paginator
+from reviews_manager.models import ROIsAnnotationStep
+
+from csv import DictWriter
+
+import logging
+
+logger = logging.getLogger('promort_commands')
+
+
+class Command(BaseCommand):
+    help = """
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument('--output_file', dest='output', type=str, required=True,
+                            help='path of the output CSV file')
+        parser.add_argument('--page_size', dest='page_size', type=int, default=0,
+                            help='the number of records retrieved for each page (this will enable pagination)')
+
+    def _get_formatted_time(self, timestamp):
+        try:
+            return timestamp.strftime('%Y-%m-%d %H:%M:%S')
+        except AttributeError:
+            return None
+
+    def _dump_row(self, step, csv_writer):
+        csv_writer.writerow({
+            'case_id': step.rois_annotation.case.id,
+            'slide_id': step.slide.id,
+            'roi_review_step_id': step.label,
+            'creation_date': self._get_formatted_time(step.creation_date),
+            'start_date': self._get_formatted_time(step.start_date),
+            'completion_date': self._get_formatted_time(step.completion_date),
+            'reviewer': step.rois_annotation.reviewer.username,
+            'slices_count': step.slices.count(),
+            'cores_count': len(step.cores),
+            'focus_regions_count': len(step.focus_regions)
+        })
+
+    def _dump_data(self, page_size, csv_writer):
+        if page_size > 0:
+            logger.info('Pagination enabled (%d records for page)', page_size)
+            ras_qs = ROIsAnnotationStep.objects.get_queryset().order_by('creation_date')
+            paginator = Paginator(ras_qs, page_size)
+            for x in paginator.page_range:
+                logger.info('-- page %d --', x)
+                page = paginator.page(x)
+                for s in page.object_list:
+                    self._dump_row(s, csv_writer)
+        else:
+            logger.info('Loading full batch')
+            steps = ROIsAnnotationStep.objects.all()
+            for s in steps:
+                self._dump_row(s, csv_writer)
+
+    def _export_data(self, out_file, page_size):
+        header = ['case_id', 'slide_id', 'roi_review_step_id', 'creation_date', 'start_date', 'completion_date',
+                  'reviewer', 'slices_count', 'cores_count', 'focus_regions_count']
+        with open(out_file, 'w') as ofile:
+            writer = DictWriter(ofile, delimiter=',', fieldnames=header)
+            writer.writeheader()
+            self._dump_data(page_size, writer)
+
+    def handle(self, *args, **opts):
+        logger.info('=== Starting export job ===')
+        self._export_data(opts['output'], opts['page_size'])
+        logger.info('=== Data saved to %s ===', opts['output'])

promort/rois_manager/management/commands/extract_cores.py

@@ -31,7 +31,7 @@
 
 class Command(BaseCommand):
     help = """
-    Extract focus regions as JSON objects
+    Extract cores as JSON objects
     """
 
     def add_arguments(self, parser):
@@ -89,21 +89,32 @@ def _dump_core(self, core, slide_id, slide_bounds, out_folder):
         bbox = self._extract_bounding_box(points)
         with open(file_path, 'w') as ofile:
             json.dump(points, ofile)
-        return {
+        core_data = {
             'slide_id': slide_id,
             'slice_id': core.slice.id,
             'core_id': core.id,
             'author': core.author.username,
             'core_label': core.label,
             'file_name': 'c_%d.json' % core.id,
             'bbox': bbox,
-            'focus_regions_count': core.focus_regions.count()
+            'focus_regions_count': core.focus_regions.count(),
+            'positive': core.is_positive()
         }
+        if core.is_positive():
+            if core.clinical_annotations.count() == 1:
+                core_data.update({
+                    'primary_gleason': core.clinical_annotations.first().primary_gleason,
+                    'secondary_gleason': core.clinical_annotations.first().secondary_gleason
+                })
+            else:
+                logger.warning("Multiple clinical annotations for core {0}, Gleason extraction failed".format(core.id))
+        return core_data
 
     def _dump_details(self, details, out_folder):
         with open(os.path.join(out_folder, 'cores.csv'), 'w') as ofile:
             writer = DictWriter(ofile, ['slide_id', 'slice_id', 'core_id', 'author', 'core_label',
-                                        'focus_regions_count', 'bbox', 'file_name'])
+                                        'focus_regions_count', 'bbox', 'positive',
+                                        'primary_gleason', 'secondary_gleason', 'file_name'])
             writer.writeheader()
             writer.writerows(details)
 

promort/rois_manager/management/commands/extract_slices.py

@@ -0,0 +1,163 @@
+#  Copyright (c) 2019, CRS4
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy of
+#  this software and associated documentation files (the "Software"), to deal in
+#  the Software without restriction, including without limitation the rights to
+#  use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+#  the Software, and to permit persons to whom the Software is furnished to do so,
+#  subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+#  FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+#  COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+#  IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+#  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from django.core.management.base import BaseCommand
+from reviews_manager.models import ROIsAnnotationStep
+from promort.settings import OME_SEADRAGON_BASE_URL
+
+import logging, os, requests, json
+from csv import DictWriter
+from urllib.parse import urljoin
+from shapely.geometry import Polygon
+
+logger = logging.getLogger("promort_commands")
+
+
+class Command(BaseCommand):
+    help = """
+    Extract slices as JSON objects
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--output_folder",
+            dest="out_folder",
+            type=str,
+            required=True,
+            help="path of the output folder for the extracted JSON objects",
+        )
+        parser.add_argument(
+            "--exclude_rejected",
+            dest="exclude_rejected",
+            action="store_true",
+            help="exclude slices from review steps rejected by the user",
+        )
+        parser.add_argument(
+            "--limit-bounds",
+            dest="limit_bounds",
+            action="store_true",
+            help="extract ROIs considering only the non-empty slide region",
+        )
+
+    def _load_rois_annotation_steps(self, exclude_rejected):
+        steps = ROIsAnnotationStep.objects.filter(completion_date__isnull=False)
+        if exclude_rejected:
+            steps = [s for s in steps if s.slide_evaluation.adequate_slide]
+        return steps
+
+    def _get_slide_bounds(self, slide):
+        if slide.image_type == "OMERO_IMG":
+            url = urljoin(
+                OME_SEADRAGON_BASE_URL, "deepzoom/slide_bounds/%d.dzi" % slide.omero_id
+            )
+        elif slide.image_type == "MIRAX":
+            url = urljoin(
+                OME_SEADRAGON_BASE_URL, "mirax/deepzoom/slide_bounds/%s.dzi" % slide.id
+            )
+        else:
+            logger.error(
+                "Unknown image type %s for slide %s", slide.image_type, slide.id
+            )
+            return None
+        response = requests.get(url)
+        if response.status_code == requests.codes.OK:
+            return response.json()
+        else:
+            logger.error("Error while loading slide bounds %s", slide.id)
+            return None
+
+    def _extract_points(self, roi_json, slide_bounds):
+        points = list()
+        shape = json.loads(roi_json)
+        segments = shape["segments"]
+        for x in segments:
+            points.append(
+                (
+                    x["point"]["x"] + int(slide_bounds["bounds_x"]),
+                    x["point"]["y"] + int(slide_bounds["bounds_y"]),
+                )
+            )
+        return points
+
+    def _dump_slice(self, slice, slide_id, slide_bounds, out_folder):
+        file_path = os.path.join(out_folder, "s_%d.json" % slice.id)
+        points = self._extract_points(slice.roi_json, slide_bounds)
+        with open(file_path, "w") as ofile:
+            json.dump(points, ofile)
+        slice_data = {
+            "slide_id": slide_id,
+            "slice_id": slice.id,
+            "author": slice.author.username,
+            "slice_label": slice.label,
+            "file_name": "s_%d.json" % slice.id,
+            "cores_count": slice.cores.count(),
+        }
+        return slice_data
+
+    def _dump_details(self, details, out_folder):
+        with open(os.path.join(out_folder, "slices.csv"), "w") as ofile:
+            writer = DictWriter(
+                ofile,
+                [
+                    "slide_id",
+                    "slice_id",
+                    "author",
+                    "slice_label",
+                    "cores_count",
+                    "file_name",
+                ],
+            )
+            writer.writeheader()
+            writer.writerows(details)
+
+    def _dump_slices(self, step, out_folder, limit_bounds):
+        slices = step.slices
+        slide = step.slide
+        logger.info("Loading info for slide %s", slide.id)
+        if not limit_bounds:
+            slide_bounds = self._get_slide_bounds(slide)
+        else:
+            slide_bounds = {"bounds_x": 0, "bounds_y": 0}
+        if slide_bounds:
+            logger.info("Dumping %d slices for step %s", slices.count(), step.label)
+            if slices.count() > 0:
+                out_path = os.path.join(out_folder, step.slide.id, step.label)
+                try:
+                    os.makedirs(out_path)
+                except OSError:
+                    pass
+                slices_details = list()
+                for s in slices.all():
+                    slices_details.append(
+                        self._dump_slice(s, step.slide.id, slide_bounds, out_path)
+                    )
+                self._dump_details(slices_details, out_path)
+
+    def _export_data(self, out_folder, exclude_rejected=False, limit_bounds=False):
+        steps = self._load_rois_annotation_steps(exclude_rejected)
+        logger.info("Loaded %d ROIs Annotation Steps", len(steps))
+        for s in steps:
+            self._dump_slices(s, out_folder, limit_bounds)
+
+    def handle(self, *args, **opts):
+        logger.info("=== Starting export job ===")
+        self._export_data(
+            opts["out_folder"], opts["exclude_rejected"], opts["limit_bounds"]
+        )
+        logger.info("=== Export completed ===")

promort/src/js/ome_seadragon_viewer/viewer.controllers.js

@@ -328,7 +328,7 @@
                         vm.dzi_url = vm.ome_base_url + 'deepzoom/get/' + vm.slide_details.omero_id + '.dzi';
                     }
 
-                    if (typeof(vm.prediction_id) !== 'undefined') {
+                    if (CurrentPredictionDetailsService.predictionAvailable()) {
                         HeatmapViewerService.getPredictionInfo(vm.prediction_id)
                             .then(PredictionInfoSuccessFn, PredictionInfoErrorFn);
 

promort/src/js/predictions_manager/predictions_manager.services.js

@@ -38,6 +38,7 @@
             getPredictionByReviewStep: getPredictionByReviewStep,
             getLatestPredictionBySlide: getLatestPredictionBySlide,
             registerCurrentPrediction: registerCurrentPrediction,
+            predictionAvailable: predictionAvailable,
             getPredictionId: getPredictionId,
             getSlideId: getSlideId,
             getCaseId: getCaseId
@@ -68,6 +69,10 @@
             caseID = case_id;
         }
 
+        function predictionAvailable() {
+            return typeof(predictionID)!=='undefined';
+        }
+
         function getPredictionId() {
             return predictionID;
         }