🆕 Add Reading Slide-level info & Simple Plugins for Visualization Tool

docs/visualization.rst

@@ -218,6 +218,10 @@ Additional features can be added to nodes by adding extra keys to the dictionary
 
 It will be possible to color the nodes by these features in the interface, and the top 10 will appear in a tooltip when hovering over a node (you will have to turn on the hovertool in the small toolbar to the right of the main window to enable this, it is disabled by default.)
 
+Slide Level Information
+^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have slide-level predictions, ground truth labels, or other metadata you wish to be able to see associated with slides in the interface, this can be provided as a .csv formatted table placed in the slides folder, with "Image File" as the first column. The other columns can be anything you like. When loading a slide in the UI, if the slide name appears in the "Image File" column of the provided .csv, any other entries in that row will be displayed in the interface below the main view window when the slide is selected.
 
 .. _examples:
 
@@ -422,7 +426,7 @@ and the ability to toggle on or off specific UI elements:
 
 ::
 
-    "UI_elements_1": {              # controls which UI elements are visible
+    "ui_elements_1": {              # controls which UI elements are visible
         "slide_select": 1,          # slide select box
         "layer_drop": 1,            # overlay select drop down
         "slide_row": 1,             # slide alpha toggle and slider
@@ -437,7 +441,7 @@ and the ability to toggle on or off specific UI elements:
 
 ::
 
-    "UI_elements_2": {              # controls visible UI elements on second tab in UI
+    "ui_elements_2": {              # controls visible UI elements on second tab in UI
         "opt_buttons": 1,           # UI elements providing a few options including if annotations should be filled/outline only
         "pt_size_spinner": 1,       # control for point size and graph node size
         "edge_size_spinner": 1,     # control for edge thickness

tests/test_app_bokeh.py

@@ -101,6 +101,10 @@ def annotation_path(data_path: dict[str, Path]) -> dict[str, object]:
         "patch-extraction-vf",
         data_path["base_path"] / "slides",
     )
+    data_path["meta"] = _fetch_remote_sample(
+        "test_meta",
+        data_path["base_path"] / "slides",
+    )
     data_path["annotations"] = _fetch_remote_sample(
         "annotation_store_svs_1",
         data_path["base_path"] / "overlays",
@@ -181,8 +185,8 @@ def test_get_level_by_extent() -> None:
 
 def test_roots(doc: Document) -> None:
     """Test that the document has the correct number of roots."""
-    # should be 4 roots: main window, controls, slide_info, popup table
-    assert len(doc.roots) == 4
+    # should be 5 roots: main window, controls, slide_info, popup, extra_layout
+    assert len(doc.roots) == 5
 
 
 def test_config_loaded(data_path: pytest.TempPathFactory) -> None:
@@ -210,6 +214,11 @@ def test_slide_select(doc: Document, data_path: pytest.TempPathFactory) -> None:
     slide_select.value = ["CMU-1.ndpi"]
     assert main.UI["vstate"].slide_path == data_path["slide2"]
 
+    # check the slide metadata is loaded from csv
+    desc = doc.get_model_by_name("description")
+    assert "valA" in desc.text
+    assert "valB" not in desc.text
+
     # check selecting nothing has no effect
     slide_select.value = []
     assert main.UI["vstate"].slide_path == data_path["slide2"]
@@ -618,7 +627,7 @@ def test_pt_size_spinner(doc: Document) -> None:
     pt_size_spinner.value = 10
     # check that the point size has been set correctly
     assert (
-        main.UI["p"].renderers[main.UI["vstate"].layer_dict["nodes"]].glyph.size
+        main.UI["p"].renderers[main.UI["vstate"].layer_dict["nodes"]].glyph.radius
         == 2 * 10
     )
 

tests/test_json_config_bokeh.py

@@ -7,6 +7,7 @@
 from threading import Thread
 from typing import TYPE_CHECKING
 
+import pandas as pd
 import pytest
 import requests
 from bokeh.client.session import ClientSession, pull_session
@@ -29,6 +30,15 @@ def annotation_path(data_path: dict[str, Path]) -> dict[str, Path]:
         "ndpi-1",
         data_path["base_path"] / "slides",
     )
+    data_path["meta"] = _fetch_remote_sample(
+        "test_meta",
+        data_path["base_path"] / "slides",
+    )
+    meta_df = pd.read_csv(data_path["meta"])
+    # change 'Image File' column name to 'Wrong Name'
+    meta_df = meta_df.rename(columns={"Image File": "Wrong Name"})
+    # save so we can test behaviour if required column isn't there
+    meta_df.to_csv(data_path["meta"], index=False)
     data_path["annotations"] = _fetch_remote_sample(
         "annotation_store_svs_1",
         data_path["base_path"] / "overlays",
@@ -82,6 +92,10 @@ def test_slides_available(bk_session: ClientSession) -> None:
     slide_select.value = ["CMU-1.ndpi"]
     assert len(layer_drop.menu) == 2
 
+    # check the metadata wasnt found as the column name was wrong
+    desc = doc.get_model_by_name("description")
+    assert "Metadata:" not in desc.text
+
     bk_session.document.clear()
     assert len(bk_session.document.roots) == 0
     bk_session.close()

tiatoolbox/cli/visualize.py

@@ -77,6 +77,14 @@ def run_bokeh(img_input: list[str], port: int, *, noshow: bool) -> None:
     This option must be used in conjunction with --slides.
     The --base-path option should not be used in this case.""",
 )
+@click.option(
+    "--plugin",
+    multiple=True,
+    help=r"""Path to a file to define an extra layout containing extra
+    resources such as graphs below each slide. Some pre-built plugins are
+    available in the tiatoolbox\visualization\templates folder. Can pass
+    multiple instances of this option to add multiple ui additions.""",
+)
 @click.option(
     "--port",
     type=int,
@@ -88,6 +96,7 @@ def visualize(
     base_path: str,
     slides: str,
     overlays: str,
+    plugin: list[str],
     port: int,
     *,
     noshow: bool,
@@ -101,6 +110,7 @@ def visualize(
         base_path (str): Path to base directory containing images to be displayed.
         slides (str): Path to directory containing slides to be displayed.
         overlays (str): Path to directory containing overlays to be displayed.
+        plugin (list): Paths to files containing ui plugins.
         port (int): Port to launch the visualization tool on.
         noshow (bool): Do not launch in browser (mainly intended for testing).
 
@@ -109,7 +119,7 @@ def visualize(
     if base_path is None and (slides is None or overlays is None):
         msg = "Must specify either base-path or both slides and overlays."
         raise ValueError(msg)
-    img_input = [base_path, slides, overlays]
+    img_input = [base_path, slides, overlays, *list(plugin)]
     img_input = [p for p in img_input if p is not None]
     # check that the input paths exist
     for input_path in img_input:

tiatoolbox/data/remote_samples.yaml

@@ -137,5 +137,7 @@ files:
     url: [ *testdata, "annotation/test1_config.json"]
   config_2:
     url: [ *testdata, "annotation/test2_config.json"]
+  test_meta:
+    url: [ *testdata, "annotation/test_meta.csv"]
   nuclick-output:
     url: [*modelroot, "predictions/nuclei_mask/nuclick-output.npy"]

tiatoolbox/visualization/bokeh_app/main.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import importlib
 import json
 import sys
 import tempfile
@@ -12,6 +13,7 @@
 from typing import TYPE_CHECKING, Any, Callable, SupportsFloat
 
 import numpy as np
+import pandas as pd
 import requests
 import torch
 from bokeh.events import ButtonClick, DoubleTap, MenuItemClick
@@ -70,7 +72,7 @@
 )
 from tiatoolbox.tools.pyramid import ZoomifyGenerator
 from tiatoolbox.utils.visualization import random_colors
-from tiatoolbox.visualization.ui_utils import get_level_by_extent
+from tiatoolbox.visualization.ui_utils import UIWrapper, get_level_by_extent
 from tiatoolbox.wsicore.wsireader import WSIReader
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -117,23 +119,27 @@
         self.item = val
 
 
-class UIWrapper:
-    """Wrapper class to access ui elements."""
-
-    def __init__(self: UIWrapper) -> None:
-        """Initialize the class."""
-        self.active = 0
-
-    def __getitem__(self: UIWrapper, key: str) -> Any:  # noqa: ANN401
-        """Gets ui element for the active window."""
-        return win_dicts[self.active][key]
+def to_camel(name: str) -> str:
+    """Convert a string to upper camel case."""
+    parts = name.split("_")
+    return "".join([p.capitalize() for p in parts])
 
 
 def format_info(info: dict[str, Any]) -> str:
     """Format the slide info for display."""
-    info_str = f"<b>Slide Name: {info.pop('file_path').name}</b><br>"
+    slide_name = info.pop("file_path").name
+    info_str = f"<b>Slide Name: {slide_name}</b><br>"
     for k, v in info.items():
         info_str += f"{k}: {v}<br>"
+    # if there is metadata, add it
+    if doc_config.metadata is not None:
+        try:
+            row_data = doc_config.metadata.loc[slide_name]
+            info_str += "<br><b>Metadata:</b><br>"
+            for k, v in row_data.items():
+                info_str += f"{k}: {v}<br>"
+        except KeyError:
+            info_str += "<br><b>No metadata found.</b><br>"
     return info_str
 
 
@@ -778,7 +784,7 @@
 
 def pt_size_cb(attr: str, old: float, new: float) -> None:  # noqa: ARG001
     """Callback to change the size of the points."""
-    UI["vstate"].graph_node.size = 2 * new
+    UI["vstate"].graph_node.radius = 2 * new
 
 
 def edge_size_cb(attr: str, old: float, new: float) -> None:  # noqa: ARG001
@@ -848,6 +854,13 @@
     if len(new) == 0:
         return
     slide_path = Path(doc_config["slide_folder"]) / Path(new[0])
+    if doc_config["extra_layout"] is not None:
+        # create the extra layout if we can
+        extras = []
+        for cl in doc_config["extra_layout"]:
+            extras.extend(cl.create_extra_layout(slide_path, extra_layout.children))
+            cl.add_to_ui()
+        extra_layout.children = extras
     # Reset the data sources for glyph overlays
     UI["pt_source"].data = {"x": [], "y": []}
     UI["box_source"].data = {"x": [], "y": [], "width": [], "height": []}
@@ -1272,6 +1285,11 @@
     height=200,
     sizing_mode="stretch_width",
 )
+extra_layout = column(
+    children=[],
+    name="extra_layout",
+    sizing_mode="stretch_both",
+)
 
 
 def gather_ui_elements(  # noqa: PLR0915
@@ -1771,7 +1789,14 @@
     box_source = ColumnDataSource({"x": [], "y": [], "width": [], "height": []})
     pt_source = ColumnDataSource({"x": [], "y": []})
     r = p.rect("x", "y", "width", "height", source=box_source, fill_alpha=0)
-    c = p.circle("x", "y", source=pt_source, color="red", size=5)
+    c = p.circle(
+        "x",
+        "y",
+        source=pt_source,
+        color="red",
+        radius=3,
+        radius_units="screen",
+    )
     p.add_tools(BoxEditTool(renderers=[r], num_objects=1))
     p.add_tools(PointDrawTool(renderers=[c]))
     p.add_tools(TapTool())
@@ -1783,7 +1808,13 @@
     # Add graph stuff
     node_source = ColumnDataSource({"x_": [], "y_": [], "node_color_": []})
     edge_source = ColumnDataSource({"x0_": [], "y0_": [], "x1_": [], "y1_": []})
-    vstate.graph_node = Circle(x="x_", y="y_", fill_color="node_color_", size=5)
+    vstate.graph_node = Circle(
+        x="x_",
+        y="y_",
+        fill_color="node_color_",
+        radius=3,
+        radius_units="screen",
+    )
     vstate.graph_edge = Segment(x0="x0_", y0="y0_", x1="x1_", y1="y1_")
     p.add_glyph(node_source, vstate.graph_node)
     node_source.selected.on_change("indices", node_select_cb)
@@ -1853,14 +1884,16 @@
         "hover": hover,
         "user": user,
         "color_bar": color_bar,
+        "ui_layout": ui_layout,
+        "extra_options": extra_options,
     }
 
 
 # Main ui containers
-UI = UIWrapper()
 windows = []
 controls = []
 win_dicts = []
+UI = UIWrapper(win_dicts)
 
 # Popup for annotation viewing on double click
 popup_div = Div(
@@ -2004,6 +2037,7 @@
             "overlay_folder": Path("/app_data").joinpath("overlays"),
         }
         self.sys_args = None
+        self.metadata = None
 
     def __getitem__(self: DocConfig, key: str) -> Any:  # noqa: ANN401
         """Get an item from the config."""
@@ -2033,6 +2067,23 @@
         base_folder = slide_folder.parent
         overlay_folder = Path(sys_args[2])
 
+        plugins = None
+        if len(sys_args) > 3:  # noqa: PLR2004
+            # also passed a path to a plugin file defining extra ui elements
+            plugins = []
+            for p in sys_args[3:]:
+                plugin_path = Path(p)
+
+                # should have class named (in camel case) after the filename, that
+                # handles the extra ui elements
+                spec = importlib.util.spec_from_file_location(
+                    "plugin_mod",
+                    plugin_path,
+                )
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)  # Execute the module
+                plugins.append(module.__getattribute__(to_camel(plugin_path.stem))(UI))
+
         # Load a color_dict and/or slide initial view windows from a json file
         config_file = list(overlay_folder.glob("*config.json"))
         config = self.config
@@ -2045,6 +2096,7 @@
         config["base_folder"] = base_folder
         config["slide_folder"] = slide_folder
         config["overlay_folder"] = overlay_folder
+        config["extra_layout"] = plugins
         config["demo_name"] = self.config["demo_name"]
         if "initial_views" not in config:
             config["initial_views"] = {}
@@ -2070,10 +2122,26 @@
 
         """
         self._get_config()
+        # see if there's a metadata .csv file in slides folder
+        metadata_file = list(doc_config["slide_folder"].glob("*.csv"))
+        if len(metadata_file) > 0:
+            metadata_file = metadata_file[0]
+            with metadata_file.open() as f:
+                metadata = pd.read_csv(f, encoding="utf-8")
+                # must have an 'Image File' column to associate with slides
+                if "Image File" in metadata.columns:
+                    metadata = metadata.set_index("Image File")
+                else:
+                    # can't use it so set to None
+                    metadata = None
+        else:
+            # no metadata file
+            metadata = None
+        self.metadata = metadata
 
         # Set initial slide to first one in base folder
         slide_list = []
-        for ext in ["*.svs", "*ndpi", "*.tiff", "*.mrxs", "*.png", "*.jpg"]:
+        for ext in ["*.svs", "*ndpi", "*.tiff", "*.mrxs", "*.png", "*.jpg", "*.tif"]:
             slide_list.extend(list(doc_config["slide_folder"].glob(ext)))
             slide_list.extend(
                 list(doc_config["slide_folder"].glob(str(Path("*") / ext))),
@@ -2102,6 +2170,7 @@
         base_doc.add_root(control_tabs)
         base_doc.add_root(popup_table)
         base_doc.add_root(slide_info)
+        base_doc.add_root(extra_layout)
         base_doc.title = "Tiatoolbox Visualization Tool"
         return slide_wins, control_tabs
 

tiatoolbox/visualization/bokeh_app/templates/__init__.py

@@ -0,0 +1 @@
+"""plugins and templates for bokeh visualization app."""