Merge pull request #122 from DigitalSlideArchive/girder-ids-in-annotations

Handle annotations with girderId references.

Author: manthey

README.rst

@@ -48,9 +48,29 @@ Development
 
 The most convenient way to develop on HistomicsUI is to use the `devops scripts from the Digital Slide Archive <https://github.com/DigitalSlideArchive/digital_slide_archive/tree/master/devops>`_.
 
+Annotations and Metadata from Jobs
+----------------------------------
+
+This handles ingesting annotations and metadata that are uploaded and associating them with existing large image items in the Girder database.  These annotations and metadata re commonly generated through jobs, such as HistomicTK tasks, but can also be added manually.
+
+If a file is uploaded to the Girder system that includes a ``reference`` record, and that ``reference`` record contains an ``identifier`` field and a ``fileId`` field, specific identifiers can be used to ingest the results.  If a ``userId`` is specified in the ``reference`` record, permissions for adding the annotation or metadata are associated with that user.
+
+Metadata
+========
+
+Identifiers ending in ``ItemMetadata`` are loaded and then set as metadata on the associated item that contains the specified file.  Conceptually, this is the same as calling the ``PUT`` ``item/{id}/metadata`` endpoint.
+
+Annotations
+===========
+
+Identifiers ending in ``AnnotationFile`` are loaded as annotations, associated with the item that contains the specified file.  Conceptually, this is the same as uploaded the file via the annotation endpoints for the item associated with the specified ``fileId``.
+
+If the annotation file contains any annotations with elements that contain ``girderId`` values, the ``girderId`` values can be ``identifier`` values from files that were uploaded with a ``reference`` record that contains a matching ``uuid`` field.  The ``uuid`` field is required for this, but is treated as an arbitrary string.
+
+
 Funding
 -------
-This work is funded in part by the NIH grant U24-CA194362-01_.
+This work was funded in part by the NIH grant U24-CA194362-01_.
 
 .. _HistomicsUI: https://github.com/DigitalSlideArchive/HistomicsUI
 .. _Docker: https://www.docker.com/

histomicsui/handlers.py

@@ -1,6 +1,7 @@
 import datetime
 import json
 
+import cachetools
 from girder import logger
 from girder.constants import AccessType
 from girder.exceptions import RestException
@@ -13,6 +14,8 @@
 
 from .constants import PluginSettings
 
+_recentIdentifiers = cachetools.TTLCache(maxsize=100, ttl=86400)
+
 
 def _itemFromEvent(event, identifierEnding, itemAccessLevel=AccessType.READ):
     """
@@ -34,6 +37,13 @@ def _itemFromEvent(event, identifierEnding, itemAccessLevel=AccessType.READ):
                 identifier = reference['identifier']
         except (ValueError, TypeError):
             logger.debug('Failed to parse data.process reference: %r', reference)
+    if identifier and 'uuid' in reference:
+        if reference['uuid'] not in _recentIdentifiers:
+            _recentIdentifiers[reference['uuid']] = {}
+        _recentIdentifiers[reference['uuid']][identifier] = info
+        reprocessFunc = _recentIdentifiers[reference['uuid']].pop('_reprocess', None)
+        if reprocessFunc:
+            reprocessFunc()
     if identifier is not None and identifier.endswith(identifierEnding):
         if 'userId' not in reference or 'itemId' not in reference or 'fileId' not in reference:
             logger.error('Reference does not contain required information.')
@@ -46,7 +56,50 @@ def _itemFromEvent(event, identifierEnding, itemAccessLevel=AccessType.READ):
         user = User().load(userId, force=True)
         image = File().load(imageId, level=AccessType.READ, user=user)
         item = Item().load(image['itemId'], level=itemAccessLevel, user=user)
-        return {'item': item, 'user': user, 'file': image}
+        return {'item': item, 'user': user, 'file': image, 'uuid': reference.get('uuid')}
+
+
+def resolveAnnotationGirderIds(event, results, data, possibleGirderIds):
+    """
+    If an annotation has references to girderIds, resolve them to actual ids.
+
+    :param event: a data.process event.
+    :param results: the results from _itemFromEvent,
+    :param data: annotation data.
+    :param possibleGirderIds: a list of annotation elements with girderIds
+        needing resolution.
+    :returns: True if all ids were processed.
+    """
+    # Exclude actual girderIds from resolution
+    girderIds = []
+    for element in possibleGirderIds:
+        # This will throw an exception if the girderId isn't well-formed as an
+        # actual id.
+        try:
+            if Item().load(element['girderId'], level=AccessType.READ, force=True) is None:
+                girderIds.append(element)
+        except Exception:
+            girderIds.append(element)
+    if not len(girderIds):
+        return True
+    idRecord = _recentIdentifiers.get(results.get('uuid'))
+    if idRecord and not all(element['girderId'] in idRecord for element in girderIds):
+        idRecord['_reprocess'] = lambda: process_annotations(event)
+        return False
+    for element in girderIds:
+        element['girderId'] = str(idRecord[element['girderId']]['file']['itemId'])
+        # Currently, all girderIds inside annotations are expected to be
+        # large images.  In this case, load them and ask if they can be so,
+        # in case they are small images
+        from girder_large_image.models.image_item import ImageItem
+
+        try:
+            item = ImageItem().load(element['girderId'], force=True)
+            ImageItem().createImageItem(
+                item, list(ImageItem().childFiles(item=item, limit=1))[0], createJob=False)
+        except Exception:
+            pass
+    return True
 
 
 def process_annotations(event):
@@ -73,6 +126,16 @@ def process_annotations(event):
 
     if not isinstance(data, list):
         data = [data]
+    # Check some of the early elements to see if there are any girderIds
+    # that need resolution.
+    if 'uuid' in results:
+        girderIds = [
+            element for annotation in data
+            for element in annotation.get('elements', [])[:100]
+            if 'girderId' in element]
+        if len(girderIds):
+            if not resolveAnnotationGirderIds(event, results, data, girderIds):
+                return
     for annotation in data:
         try:
             Annotation().createAnnotation(item, user, annotation)

setup.py

@@ -44,6 +44,7 @@ def prerelease_local_scheme(version):
     install_requires=[
         'girder-large-image-annotation',
         'girder-slicer-cli-web>=1.2.3',
+        'cachetools',
     ],
     extras_require={
         'analysis': [

tests/test_files/sample_girder_id.anot

@@ -0,0 +1,24 @@
+{
+  "name": "Sample",
+  "elements": [
+    {
+      "type": "image",
+      "girderId": "ImageRecord1"
+    },
+    {
+      "type": "point",
+      "center": [1033,363,0],
+      "lineColor": "rgb(0,0,0)",
+      "lineWidth": 2,
+      "fillColor": "rgba(0,0,0,0)"
+    },
+    {
+      "type": "polyline",
+      "points": [[1029,408,0], [1012,449,0], [1055,458,0], [1060,414,0]],
+      "fillColor": "rgba(0,0,0,0)",
+      "lineColor": "rgb(0,0,0)",
+      "lineWidth": 2,
+      "closed": true
+    }
+  ]
+}

tests/test_handlers.py

@@ -65,3 +65,57 @@ def testMetadataHandler(self, server, fsAssetstore, admin):
         item = Item().load(file['itemId'], user=admin)
         assert item['meta']['sample'] == 'value'
         assert item['meta']['complex']['key1'] == 'value1'
+
+    def testAnnotationWithGirderIdHandler(self, server, fsAssetstore, admin):
+        file = utilities.uploadExternalFile('Easy1.png', admin, fsAssetstore)
+        item = Item().load(file['itemId'], user=admin)
+        utilities.uploadExternalFile(
+            'Easy1.png', admin, fsAssetstore, reference=json.dumps({
+                'identifier': 'ImageRecord1',
+                'uuid': '12345',
+                'userId': str(admin['_id']),
+                'itemId': str(item['_id']),
+                'fileId': str(file['_id']),
+            }))
+        assert Annotation().findOne({'itemId': item['_id']}) is None
+        utilities.uploadTestFile(
+            'sample_girder_id.anot', admin, fsAssetstore, reference=json.dumps({
+                'identifier': 'IsAnAnnotationFile',
+                'uuid': '12345',
+                'userId': str(admin['_id']),
+                'itemId': str(item['_id']),
+                'fileId': str(file['_id']),
+            }))
+        starttime = time.time()
+        while time.time() < starttime + 10:
+            if Annotation().findOne({'itemId': item['_id']}) is not None:
+                break
+            time.sleep(0.1)
+        assert Annotation().findOne({'itemId': item['_id']}) is not None
+
+    def testAnnotationWithGirderIdHandlerAltOrder(self, server, fsAssetstore, admin):
+        file = utilities.uploadExternalFile('Easy1.png', admin, fsAssetstore)
+        item = Item().load(file['itemId'], user=admin)
+        utilities.uploadTestFile(
+            'sample_girder_id.anot', admin, fsAssetstore, reference=json.dumps({
+                'identifier': 'IsAnAnnotationFile',
+                'uuid': '12346',
+                'userId': str(admin['_id']),
+                'itemId': str(item['_id']),
+                'fileId': str(file['_id']),
+            }))
+        assert Annotation().findOne({'itemId': item['_id']}) is None
+        utilities.uploadExternalFile(
+            'Easy1.png', admin, fsAssetstore, reference=json.dumps({
+                'identifier': 'ImageRecord1',
+                'uuid': '12346',
+                'userId': str(admin['_id']),
+                'itemId': str(item['_id']),
+                'fileId': str(file['_id']),
+            }))
+        starttime = time.time()
+        while time.time() < starttime + 10:
+            if Annotation().findOne({'itemId': item['_id']}) is not None:
+                break
+            time.sleep(0.1)
+        assert Annotation().findOne({'itemId': item['_id']}) is not None