diff --git a/promort/VERSION b/promort/VERSION index 78bc1ab..5712157 100644 --- a/promort/VERSION +++ b/promort/VERSION @@ -1 +1 @@ -0.10.0 +0.10.1 diff --git a/promort/reviews_manager/management/commands/build_clinical_reviews_worklist.py b/promort/reviews_manager/management/commands/build_clinical_reviews_worklist.py index 670f8f0..80b8bd6 100644 --- a/promort/reviews_manager/management/commands/build_clinical_reviews_worklist.py +++ b/promort/reviews_manager/management/commands/build_clinical_reviews_worklist.py @@ -23,6 +23,7 @@ from slides_manager.models import SlideEvaluation from reviews_manager.models import ROIsAnnotation, ClinicalAnnotation, ClinicalAnnotationStep, ReviewsComparison +from csv import DictReader, DictWriter from uuid import uuid4 import logging, random from datetime import datetime @@ -34,6 +35,8 @@ class Command(BaseCommand): help = 'build second reviewers worklist based on existing ROIs annotations' def add_arguments(self, parser): + parser.add_argument('--worklist-file', dest='worklist', type=str, default=None, + help='a CSV file containing the list of the ROIs annotations that will be processed') parser.add_argument('--reviewers-count', dest='reviewers_count', type=int, default=1, help='the number of clinical reviews created for each ROIs annotation') @@ -41,9 +44,12 @@ def _get_clinical_manager_users(self): clinical_managers_group = Group.objects.get(name=DEFAULT_GROUPS['clinical_manager']['name']) return clinical_managers_group.user_set.all() - def _get_rois_annotations_list(self): + def _get_rois_annotations_list(self, rois_annotations_list=None): linked_annotations = [ca.rois_review.label for ca in ClinicalAnnotation.objects.all()] - return ROIsAnnotation.objects.exclude(label__in=linked_annotations) + if rois_annotations_list is None: + return ROIsAnnotation.objects.exclude(label__in=linked_annotations) + else: + return ROIsAnnotation.objects.filter(label__in=rois_annotations_list).exclude(label__in=linked_annotations) def _select_clinical_reviewers(self, rois_annotation, clinical_managers, reviewers_count): if reviewers_count >= len(clinical_managers): @@ -116,14 +122,25 @@ def _create_clinical_annotation_step(self, clinical_annotation_obj, rois_annotat # logger.info('Create Reviews Comparison for Clinical Annotation Steps %s and %s', # review_step_1_obj.label, review_step_2_obj.label) # return reviews_comparison_obj + + def load_roi_annotations_list(self, worklist_file): + with open(worklist_file) as f: + reader = DictReader(f) + return [row['annotation_label'] for row in reader] + def handle(self, *args, **opts): logger.info('=== Starting clinical annotations worklist creation ===') reviewers_count = opts['reviewers_count'] + worklist_file = opts['worklist'] clinical_annotations_manager = self._get_clinical_manager_users() if len(clinical_annotations_manager) == 0: raise CommandError('There must be at least 1 user with Clinical Annotations Manager role') - rois_annotations = self._get_rois_annotations_list() + if worklist_file: + rois_annotations_list = self.load_roi_annotations_list(worklist_file) + else: + rois_annotations_list = None + rois_annotations = self._get_rois_annotations_list(rois_annotations_list) if len(rois_annotations) == 0: logger.info('There are no ROIs Annotations to process') for r_ann in rois_annotations: diff --git a/promort/reviews_manager/management/commands/get_rois_annotation_steps_data.py b/promort/reviews_manager/management/commands/get_rois_annotation_steps_data.py new file mode 100644 index 0000000..0aad577 --- /dev/null +++ b/promort/reviews_manager/management/commands/get_rois_annotation_steps_data.py @@ -0,0 +1,88 @@ +# Copyright (c) 2022, CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from django.core.management.base import BaseCommand +from django.core.paginator import Paginator +from reviews_manager.models import ROIsAnnotationStep + +from csv import DictWriter + +import logging + +logger = logging.getLogger('promort_commands') + + +class Command(BaseCommand): + help = """ + """ + + def add_arguments(self, parser): + parser.add_argument('--output_file', dest='output', type=str, required=True, + help='path of the output CSV file') + parser.add_argument('--page_size', dest='page_size', type=int, default=0, + help='the number of records retrieved for each page (this will enable pagination)') + + def _get_formatted_time(self, timestamp): + try: + return timestamp.strftime('%Y-%m-%d %H:%M:%S') + except AttributeError: + return None + + def _dump_row(self, step, csv_writer): + csv_writer.writerow({ + 'case_id': step.rois_annotation.case.id, + 'slide_id': step.slide.id, + 'roi_review_step_id': step.label, + 'creation_date': self._get_formatted_time(step.creation_date), + 'start_date': self._get_formatted_time(step.start_date), + 'completion_date': self._get_formatted_time(step.completion_date), + 'reviewer': step.rois_annotation.reviewer.username, + 'slices_count': step.slices.count(), + 'cores_count': len(step.cores), + 'focus_regions_count': len(step.focus_regions) + }) + + def _dump_data(self, page_size, csv_writer): + if page_size > 0: + logger.info('Pagination enabled (%d records for page)', page_size) + ras_qs = ROIsAnnotationStep.objects.get_queryset().order_by('creation_date') + paginator = Paginator(ras_qs, page_size) + for x in paginator.page_range: + logger.info('-- page %d --', x) + page = paginator.page(x) + for s in page.object_list: + self._dump_row(s, csv_writer) + else: + logger.info('Loading full batch') + steps = ROIsAnnotationStep.objects.all() + for s in steps: + self._dump_row(s, csv_writer) + + def _export_data(self, out_file, page_size): + header = ['case_id', 'slide_id', 'roi_review_step_id', 'creation_date', 'start_date', 'completion_date', + 'reviewer', 'slices_count', 'cores_count', 'focus_regions_count'] + with open(out_file, 'w') as ofile: + writer = DictWriter(ofile, delimiter=',', fieldnames=header) + writer.writeheader() + self._dump_data(page_size, writer) + + def handle(self, *args, **opts): + logger.info('=== Starting export job ===') + self._export_data(opts['output'], opts['page_size']) + logger.info('=== Data saved to %s ===', opts['output']) diff --git a/promort/rois_manager/management/commands/extract_cores.py b/promort/rois_manager/management/commands/extract_cores.py index b4c99b3..a06d8a1 100644 --- a/promort/rois_manager/management/commands/extract_cores.py +++ b/promort/rois_manager/management/commands/extract_cores.py @@ -31,7 +31,7 @@ class Command(BaseCommand): help = """ - Extract focus regions as JSON objects + Extract cores as JSON objects """ def add_arguments(self, parser): @@ -89,7 +89,7 @@ def _dump_core(self, core, slide_id, slide_bounds, out_folder): bbox = self._extract_bounding_box(points) with open(file_path, 'w') as ofile: json.dump(points, ofile) - return { + core_data = { 'slide_id': slide_id, 'slice_id': core.slice.id, 'core_id': core.id, @@ -97,13 +97,24 @@ def _dump_core(self, core, slide_id, slide_bounds, out_folder): 'core_label': core.label, 'file_name': 'c_%d.json' % core.id, 'bbox': bbox, - 'focus_regions_count': core.focus_regions.count() + 'focus_regions_count': core.focus_regions.count(), + 'positive': core.is_positive() } + if core.is_positive(): + if core.clinical_annotations.count() == 1: + core_data.update({ + 'primary_gleason': core.clinical_annotations.first().primary_gleason, + 'secondary_gleason': core.clinical_annotations.first().secondary_gleason + }) + else: + logger.warning("Multiple clinical annotations for core {0}, Gleason extraction failed".format(core.id)) + return core_data def _dump_details(self, details, out_folder): with open(os.path.join(out_folder, 'cores.csv'), 'w') as ofile: writer = DictWriter(ofile, ['slide_id', 'slice_id', 'core_id', 'author', 'core_label', - 'focus_regions_count', 'bbox', 'file_name']) + 'focus_regions_count', 'bbox', 'positive', + 'primary_gleason', 'secondary_gleason', 'file_name']) writer.writeheader() writer.writerows(details) diff --git a/promort/rois_manager/management/commands/extract_slices.py b/promort/rois_manager/management/commands/extract_slices.py new file mode 100644 index 0000000..2333a40 --- /dev/null +++ b/promort/rois_manager/management/commands/extract_slices.py @@ -0,0 +1,163 @@ +# Copyright (c) 2019, CRS4 +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from django.core.management.base import BaseCommand +from reviews_manager.models import ROIsAnnotationStep +from promort.settings import OME_SEADRAGON_BASE_URL + +import logging, os, requests, json +from csv import DictWriter +from urllib.parse import urljoin +from shapely.geometry import Polygon + +logger = logging.getLogger("promort_commands") + + +class Command(BaseCommand): + help = """ + Extract slices as JSON objects + """ + + def add_arguments(self, parser): + parser.add_argument( + "--output_folder", + dest="out_folder", + type=str, + required=True, + help="path of the output folder for the extracted JSON objects", + ) + parser.add_argument( + "--exclude_rejected", + dest="exclude_rejected", + action="store_true", + help="exclude slices from review steps rejected by the user", + ) + parser.add_argument( + "--limit-bounds", + dest="limit_bounds", + action="store_true", + help="extract ROIs considering only the non-empty slide region", + ) + + def _load_rois_annotation_steps(self, exclude_rejected): + steps = ROIsAnnotationStep.objects.filter(completion_date__isnull=False) + if exclude_rejected: + steps = [s for s in steps if s.slide_evaluation.adequate_slide] + return steps + + def _get_slide_bounds(self, slide): + if slide.image_type == "OMERO_IMG": + url = urljoin( + OME_SEADRAGON_BASE_URL, "deepzoom/slide_bounds/%d.dzi" % slide.omero_id + ) + elif slide.image_type == "MIRAX": + url = urljoin( + OME_SEADRAGON_BASE_URL, "mirax/deepzoom/slide_bounds/%s.dzi" % slide.id + ) + else: + logger.error( + "Unknown image type %s for slide %s", slide.image_type, slide.id + ) + return None + response = requests.get(url) + if response.status_code == requests.codes.OK: + return response.json() + else: + logger.error("Error while loading slide bounds %s", slide.id) + return None + + def _extract_points(self, roi_json, slide_bounds): + points = list() + shape = json.loads(roi_json) + segments = shape["segments"] + for x in segments: + points.append( + ( + x["point"]["x"] + int(slide_bounds["bounds_x"]), + x["point"]["y"] + int(slide_bounds["bounds_y"]), + ) + ) + return points + + def _dump_slice(self, slice, slide_id, slide_bounds, out_folder): + file_path = os.path.join(out_folder, "s_%d.json" % slice.id) + points = self._extract_points(slice.roi_json, slide_bounds) + with open(file_path, "w") as ofile: + json.dump(points, ofile) + slice_data = { + "slide_id": slide_id, + "slice_id": slice.id, + "author": slice.author.username, + "slice_label": slice.label, + "file_name": "s_%d.json" % slice.id, + "cores_count": slice.cores.count(), + } + return slice_data + + def _dump_details(self, details, out_folder): + with open(os.path.join(out_folder, "slices.csv"), "w") as ofile: + writer = DictWriter( + ofile, + [ + "slide_id", + "slice_id", + "author", + "slice_label", + "cores_count", + "file_name", + ], + ) + writer.writeheader() + writer.writerows(details) + + def _dump_slices(self, step, out_folder, limit_bounds): + slices = step.slices + slide = step.slide + logger.info("Loading info for slide %s", slide.id) + if not limit_bounds: + slide_bounds = self._get_slide_bounds(slide) + else: + slide_bounds = {"bounds_x": 0, "bounds_y": 0} + if slide_bounds: + logger.info("Dumping %d slices for step %s", slices.count(), step.label) + if slices.count() > 0: + out_path = os.path.join(out_folder, step.slide.id, step.label) + try: + os.makedirs(out_path) + except OSError: + pass + slices_details = list() + for s in slices.all(): + slices_details.append( + self._dump_slice(s, step.slide.id, slide_bounds, out_path) + ) + self._dump_details(slices_details, out_path) + + def _export_data(self, out_folder, exclude_rejected=False, limit_bounds=False): + steps = self._load_rois_annotation_steps(exclude_rejected) + logger.info("Loaded %d ROIs Annotation Steps", len(steps)) + for s in steps: + self._dump_slices(s, out_folder, limit_bounds) + + def handle(self, *args, **opts): + logger.info("=== Starting export job ===") + self._export_data( + opts["out_folder"], opts["exclude_rejected"], opts["limit_bounds"] + ) + logger.info("=== Export completed ===") diff --git a/promort/src/js/ome_seadragon_viewer/viewer.controllers.js b/promort/src/js/ome_seadragon_viewer/viewer.controllers.js index 8d6e72f..96e20e6 100644 --- a/promort/src/js/ome_seadragon_viewer/viewer.controllers.js +++ b/promort/src/js/ome_seadragon_viewer/viewer.controllers.js @@ -328,7 +328,7 @@ vm.dzi_url = vm.ome_base_url + 'deepzoom/get/' + vm.slide_details.omero_id + '.dzi'; } - if (typeof(vm.prediction_id) !== 'undefined') { + if (CurrentPredictionDetailsService.predictionAvailable()) { HeatmapViewerService.getPredictionInfo(vm.prediction_id) .then(PredictionInfoSuccessFn, PredictionInfoErrorFn); diff --git a/promort/src/js/predictions_manager/predictions_manager.services.js b/promort/src/js/predictions_manager/predictions_manager.services.js index 43f6e7d..9599287 100644 --- a/promort/src/js/predictions_manager/predictions_manager.services.js +++ b/promort/src/js/predictions_manager/predictions_manager.services.js @@ -38,6 +38,7 @@ getPredictionByReviewStep: getPredictionByReviewStep, getLatestPredictionBySlide: getLatestPredictionBySlide, registerCurrentPrediction: registerCurrentPrediction, + predictionAvailable: predictionAvailable, getPredictionId: getPredictionId, getSlideId: getSlideId, getCaseId: getCaseId @@ -68,6 +69,10 @@ caseID = case_id; } + function predictionAvailable() { + return typeof(predictionID)!=='undefined'; + } + function getPredictionId() { return predictionID; } diff --git a/promort/src/js/rois_manager/rois_manager.controllers.js b/promort/src/js/rois_manager/rois_manager.controllers.js index 84557ad..866deb9 100644 --- a/promort/src/js/rois_manager/rois_manager.controllers.js +++ b/promort/src/js/rois_manager/rois_manager.controllers.js @@ -98,6 +98,7 @@ vm._clearNavmap = _clearNavmap; vm._filterNavmapByShape = _filterNavmapByShape; vm._updateNavmap = _updateNavmap; + vm.predictionEnabled = predictionEnabled; vm.navmapDisplayEnabled = navmapDisplayEnabled; vm.switchNavmapDisplay = switchNavmapDisplay; vm.removeSliceNavmapFilter = removeSliceNavmapFilter; @@ -559,6 +560,10 @@ } } + function predictionEnabled() { + return CurrentPredictionDetailsService.predictionAvailable(); + } + function navmapDisplayEnabled() { return vm.displayNavmap; } diff --git a/promort/static_src/templates/rois_manager/manager.html b/promort/static_src/templates/rois_manager/manager.html index 2f2109f..bb7ea00 100644 --- a/promort/static_src/templates/rois_manager/manager.html +++ b/promort/static_src/templates/rois_manager/manager.html @@ -55,7 +55,8 @@

ROIs editing - Slide {{ rmc.slide_index }}

-
-
+

PALETTE