From d132636f183df134f4cb9602a43f6a1386eb8a95 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 2 Mar 2022 13:23:38 -0600 Subject: [PATCH 01/20] Add Implicit spec to loop over metadata-less "collections" --- ome_zarr/reader.py | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 9059e249..d6130d94 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -7,6 +7,7 @@ import dask.array as da import numpy as np +import zarr from dask import delayed from .axes import Axes @@ -45,21 +46,32 @@ def __init__( self.post_nodes: List[Node] = [] # TODO: this should be some form of plugin infra over subclasses + found: List[Spec] = [] if Labels.matches(zarr): - self.specs.append(Labels(self)) + found.append(Labels(self)) + self.specs.append(found[-1]) if Label.matches(zarr): - self.specs.append(Label(self)) + found.append(Label(self)) + self.specs.append(found[-1]) if Multiscales.matches(zarr): - self.specs.append(Multiscales(self)) + found.append(Multiscales(self)) + self.specs.append(found[-1]) if OMERO.matches(zarr): - self.specs.append(OMERO(self)) + found.append(OMERO(self)) + self.specs.append(found[-1]) if plate_labels: - self.specs.append(PlateLabels(self)) + found.append(PlateLabels(self)) + self.specs.append(found[-1]) elif Plate.matches(zarr): - self.specs.append(Plate(self)) + found.append(Plate(self)) + self.specs.append(found[-1]) # self.add(zarr, plate_labels=True) if Well.matches(zarr): - self.specs.append(Well(self)) + found.append(Well(self)) + self.specs.append(found[-1]) + + if not found: + self.specs.append(Implicit(self)) @overload def first(self, spectype: Type["Well"]) -> Optional["Well"]: @@ -178,6 +190,25 @@ def lookup(self, key: str, default: Any) -> Any: return self.zarr.root_attrs.get(key, default) +class Implicit(Spec): + """ + A spec-type which simply iterates over available zgroups. + """ + + @staticmethod + def matches(zarr: ZarrLocation) -> bool: + """Always return true""" + return True + + def __init__(self, node: Node) -> None: + super().__init__(node) + + for name in zarr.group(self.zarr.store).group_keys(): + child_zarr = self.zarr.create(name) + if child_zarr.exists(): + node.add(child_zarr) + + class Labels(Spec): """Relatively small specification for the well-known "labels" group which only contains the name of subgroups which should be loaded as labeled images.""" From 7470587c5f06ff50717451bdff324cd294655552 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 2 Mar 2022 18:55:12 -0600 Subject: [PATCH 02/20] Add Leaf & Root specs Pointing ome_zarr at a non-root node will now walk-up the hierarchy until the root node is discovered. --- ome_zarr/reader.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index d6130d94..f09807dd 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -73,6 +73,12 @@ def __init__( if not found: self.specs.append(Implicit(self)) + # Load up the hierarchy + if Leaf.matches(zarr): + self.specs.append(Leaf(self)) + else: + self.specs.append(Root(self)) + @overload def first(self, spectype: Type["Well"]) -> Optional["Well"]: ... @@ -209,6 +215,41 @@ def __init__(self, node: Node) -> None: node.add(child_zarr) +class Leaf(Spec): + """ + A non-root level of the Zarr hierarchy + """ + + @staticmethod + def matches(zarr: ZarrLocation) -> bool: + """Return if the parent directory is within the zarr fileset""" + + parent_zarr = zarr.create("..") + return bool(parent_zarr.exists() and (parent_zarr.zgroup or parent_zarr.zarray)) + + def __init__(self, node: Node) -> None: + super().__init__(node) + parent_zarr = node.zarr.create("..") + if parent_zarr.exists() and (parent_zarr.zgroup or parent_zarr.zarray): + node.add(parent_zarr) + + +class Root(Spec): + """ + Root of the Zarr fileset + """ + + @staticmethod + def matches(zarr: ZarrLocation) -> bool: + """Return if the parent directory is not within the zarr fileset""" + + parent_zarr = zarr.create("..") + return parent_zarr.exists() and not (parent_zarr.zgroup or parent_zarr.zarray) + + def __init__(self, node: Node) -> None: + super().__init__(node) + + class Labels(Spec): """Relatively small specification for the well-known "labels" group which only contains the name of subgroups which should be loaded as labeled images.""" From 8a4603eb58623bf97e6ca15ec58c7cfb398eb7b0 Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 3 Mar 2022 14:41:17 -0600 Subject: [PATCH 03/20] Add support for entrypoint `ome_zarr.spec` This allows the definition of node specs outside of this repository. Primary driver is for experimenting with specs which depend on ome_types, etc. These may eventually be pulled into the mainline. --- .isort.cfg | 2 +- ome_zarr/reader.py | 7 +++++++ setup.py | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.isort.cfg b/.isort.cfg index fec62009..cab00eab 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -known_third_party = dask,numcodecs,numpy,pytest,scipy,setuptools,skimage,zarr +known_third_party = dask,entrypoints,numcodecs,numpy,pytest,scipy,setuptools,skimage,zarr multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index f09807dd..8d7e2f74 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -6,6 +6,7 @@ from typing import Any, Dict, Iterator, List, Optional, Type, Union, cast, overload import dask.array as da +import entrypoints import numpy as np import zarr from dask import delayed @@ -70,6 +71,12 @@ def __init__( found.append(Well(self)) self.specs.append(found[-1]) + for key, value in entrypoints.get_group_named("ome_zarr.spec").items(): + cls = value.load() + if cls.matches(zarr): + found.append(cls(self)) + self.specs.append(found[-1]) + if not found: self.specs.append(Implicit(self)) diff --git a/setup.py b/setup.py index d946c87a..d2691bff 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ def read(fname): install_requires += (["requests"],) install_requires += (["scikit-image"],) install_requires += (["toolz"],) +install_requires += (["entrypoints"],) setup( From 02f88c9b287db58a98b03853eb21f80877cbbc0a Mon Sep 17 00:00:00 2001 From: jmoore Date: Thu, 3 Mar 2022 15:06:04 -0600 Subject: [PATCH 04/20] Also add entrypoints to requirements-dev.txt --- requirements/requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index d5a00601..5489aeb1 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,6 +1,7 @@ black cython >= 0.29.16 numpy >= 1.16.0 +entrypoints pre-commit tox wheel From 9824efef63b5cfe995eeb697de10e98a34be0bb0 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 13 Apr 2022 13:36:50 +0200 Subject: [PATCH 05/20] Minor doc improvements --- ome_zarr/reader.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 8d7e2f74..ec634ad8 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -1,4 +1,11 @@ -"""Reading logic for ome-zarr.""" +"""Reading logic for ome-zarr. + +The main class (Reader) is initialitzed with an [ome_zarr.io.ZarrLocation] +as returned by [ome_zarr.io.parse_url] and walks up and down the Zarr +hierarchy parsing each array or group into a [Node] which is aware of all +meta(data) specifications ([Spec] class) which are available in the current +runtime. +""" import logging import math @@ -71,12 +78,16 @@ def __init__( found.append(Well(self)) self.specs.append(found[-1]) + # Load all entrypoints and give them a chance + # to claim parse the current node. for key, value in entrypoints.get_group_named("ome_zarr.spec").items(): cls = value.load() if cls.matches(zarr): found.append(cls(self)) self.specs.append(found[-1]) + # Anything that has not received a type at this point + # can be considered an implicit group. if not found: self.specs.append(Implicit(self)) From 6ae6355475fe3b262bd836b80cebbfe29d9aa63f Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 13 Apr 2022 14:13:24 +0200 Subject: [PATCH 06/20] Disable Root/Leaf specifiations --- ome_zarr/reader.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ec634ad8..15d7d217 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -91,11 +91,12 @@ def __init__( if not found: self.specs.append(Implicit(self)) - # Load up the hierarchy - if Leaf.matches(zarr): - self.specs.append(Leaf(self)) - else: - self.specs.append(Root(self)) + if False: # Temporarily disable. See #174 + # Load up the hierarchy + if Leaf.matches(zarr): + self.specs.append(Leaf(self)) + else: + self.specs.append(Root(self)) @overload def first(self, spectype: Type["Well"]) -> Optional["Well"]: From 9ad2e5a45a0a7315dac03300ed265d067194ae41 Mon Sep 17 00:00:00 2001 From: jmoore Date: Mon, 18 Apr 2022 17:14:57 +0200 Subject: [PATCH 07/20] Load all requirements for the zarr-dev build --- .github/workflows/zarr-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zarr-dev.yml b/.github/workflows/zarr-dev.yml index a09f0c53..e5618327 100644 --- a/.github/workflows/zarr-dev.yml +++ b/.github/workflows/zarr-dev.yml @@ -38,7 +38,7 @@ jobs: - name: Install dependencies shell: bash -l {0} run: | - python -m pip install --upgrade pip wheel pytest tox + python -m pip install -r requirements/requirements-dev.txt python -m pip install \ git+https://github.com/zarr-developers/zarr-python.git@master From 42065e57c249f6047d59163d90d4fc03d1618b4c Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 15 Sep 2022 12:01:29 +0200 Subject: [PATCH 08/20] Migrate bioformats2raw spec Initially prepared as a plugin in https://github.com/ome/ome-zarr-metadata this is being moved to the main repository since it is a part of the main spec. --- .isort.cfg | 2 +- ome_zarr/bioformats2raw.py | 99 ++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 ome_zarr/bioformats2raw.py diff --git a/.isort.cfg b/.isort.cfg index cab00eab..d2de9058 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -known_third_party = dask,entrypoints,numcodecs,numpy,pytest,scipy,setuptools,skimage,zarr +known_third_party = dask,entrypoints,numcodecs,numpy,ome_types,ome_zarr_metadata,pytest,scipy,setuptools,skimage,zarr multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 diff --git a/ome_zarr/bioformats2raw.py b/ome_zarr/bioformats2raw.py new file mode 100644 index 00000000..07a75449 --- /dev/null +++ b/ome_zarr/bioformats2raw.py @@ -0,0 +1,99 @@ +""" +Spec definitions +""" + +import logging +import os +import re +import tempfile +from xml.etree import ElementTree as ET + +import ome_types +from ome_zarr_metadata import __version__ # noqa + +from ome_zarr.io import ZarrLocation +from ome_zarr.reader import Node +from ome_zarr.reader import Spec as Base + +__author__ = "Open Microscopy Environment (OME)" +__copyright__ = "Open Microscopy Environment (OME)" +__license__ = "BSD-2-Clause" + +_logger = logging.getLogger(__name__) + + +class bioformats2raw(Base): + @staticmethod + def matches(zarr: ZarrLocation) -> bool: + layout = zarr.root_attrs.get("bioformats2raw.layout", None) + return layout == 3 + + def __init__(self, node: Node) -> None: + super().__init__(node) + try: + data = self.handle(node) + if data.plates: + _logger.info("Plates detected. Skipping implicit loading") + else: + for idx, image in enumerate(data.images): + series = node.zarr.create(str(idx)) + assert series.exists(), f"{series} is missing" + _logger.info("found %s", series) + subnode = node.add(series) + if subnode: + subnode.metadata["ome-xml:index"] = idx + subnode.metadata["ome-xml:image"] = image + node.metadata["ome-xml"] = data + except Exception: + _logger.exception("failed to parse metadata") + + def fix_xml(self, ns: str, elem: ET.Element) -> None: + """ + Note: elem.insert() was not updating the object correctly. + """ + + if elem.tag == f"{ns}Pixels": + + must_have = {f"{ns}BinData", f"{ns}TiffData", f"{ns}MetadataOnly"} + children = {x.tag for x in elem} + + if not any(x in children for x in must_have): + # Needs fixing + metadata_only = ET.Element(f"{ns}MetadataOnly") + + last_channel = -1 + for idx, child in enumerate(elem): + if child.tag == f"{ns}Channel": + last_channel = idx + elem.insert(last_channel + 1, metadata_only) + + elif elem.tag == f"{ns}Plane": + remove = None + for idx, child in enumerate(elem): + if child.tag == f"{ns}HashSHA1": + remove = child + if remove: + elem.remove(remove) + + def parse_xml(self, filename: str) -> ome_types.model.OME: + # Parse the file and find the current schema + root = ET.parse(filename) + m = re.match(r"\{.*\}", root.getroot().tag) + ns = m.group(0) if m else "" + + # Update the XML to include MetadataOnly + for child in list(root.iter()): + self.fix_xml(ns, child) + fixed = ET.tostring(root.getroot()).decode() + + # Write file out for ome_types + with tempfile.NamedTemporaryFile() as t: + t.write(fixed.encode()) + t.flush() + return ome_types.from_xml(t.name) + + def handle(self, node: Node) -> ome_types.model.OME: + metadata = node.zarr.subpath("OME/METADATA.ome.xml") + _logger.info("Looking for metadata in %s", metadata) + if os.path.exists(metadata): + return self.parse_xml(metadata) diff --git a/setup.py b/setup.py index d2691bff..a3288221 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ def read(fname): ], entry_points={ "console_scripts": ["ome_zarr = ome_zarr.cli:main"], + "ome_zarr.spec": ["bioformats2raw = ome_zarr.bioformats2raw:bioformats2raw"], }, tests_require=["pytest"], ) From ed464ac93c4a0fe37899e9e03f131a5b49e2384c Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 15 Sep 2022 14:40:12 +0200 Subject: [PATCH 09/20] Add ome-types dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a3288221..20806ff7 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ def read(fname): install_requires += (["scikit-image"],) install_requires += (["toolz"],) install_requires += (["entrypoints"],) +install_requires += (["ome-types"],) setup( From 234475136d512aa74e20daea7a8851952561c117 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 15 Sep 2022 14:52:37 +0200 Subject: [PATCH 10/20] Remove traces of ome_zarr_metadata --- .isort.cfg | 2 +- ome_zarr/bioformats2raw.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index d2de9058..d1afe16f 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -known_third_party = dask,entrypoints,numcodecs,numpy,ome_types,ome_zarr_metadata,pytest,scipy,setuptools,skimage,zarr +known_third_party = dask,entrypoints,numcodecs,numpy,ome_types,pytest,scipy,setuptools,skimage,zarr multi_line_output = 3 include_trailing_comma = True force_grid_wrap = 0 diff --git a/ome_zarr/bioformats2raw.py b/ome_zarr/bioformats2raw.py index 07a75449..35c550ac 100644 --- a/ome_zarr/bioformats2raw.py +++ b/ome_zarr/bioformats2raw.py @@ -9,7 +9,6 @@ from xml.etree import ElementTree as ET import ome_types -from ome_zarr_metadata import __version__ # noqa from ome_zarr.io import ZarrLocation from ome_zarr.reader import Node From 873530733917b26a69612ebf96affe5a4238de95 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 15 Sep 2022 15:34:33 +0200 Subject: [PATCH 11/20] Add ome-types for pre.yml --- requirements/requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 5489aeb1..0b8be9c9 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -2,6 +2,7 @@ black cython >= 0.29.16 numpy >= 1.16.0 entrypoints +ome-types pre-commit tox wheel From d42b530fb872d4224d9df203fa78d233b591f941 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 22 Sep 2022 08:42:30 +0200 Subject: [PATCH 12/20] Store series information --- ome_zarr/bioformats2raw.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ome_zarr/bioformats2raw.py b/ome_zarr/bioformats2raw.py index 35c550ac..35a49648 100644 --- a/ome_zarr/bioformats2raw.py +++ b/ome_zarr/bioformats2raw.py @@ -34,6 +34,11 @@ def __init__(self, node: Node) -> None: if data.plates: _logger.info("Plates detected. Skipping implicit loading") else: + + series = node.zarr.root_attrs.get("series", None) + if series is not None: + node.metadata["series"] = series + for idx, image in enumerate(data.images): series = node.zarr.create(str(idx)) assert series.exists(), f"{series} is missing" From 4a9866943191831ac9b50263d8d33cafba3e2f28 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 22 Sep 2022 08:42:08 +0200 Subject: [PATCH 13/20] Add tests --- tests/data/bf2raw/fake-series-2.zarr/.zattrs | 3 ++ tests/data/bf2raw/fake-series-2.zarr/.zgroup | 3 ++ .../data/bf2raw/fake-series-2.zarr/0/.zattrs | 38 +++++++++++++++ .../data/bf2raw/fake-series-2.zarr/0/.zgroup | 3 ++ .../bf2raw/fake-series-2.zarr/0/0/.zarray | 17 +++++++ .../bf2raw/fake-series-2.zarr/0/0/0/0/0/0/0 | Bin 0 -> 1604 bytes .../bf2raw/fake-series-2.zarr/0/1/.zarray | 17 +++++++ .../bf2raw/fake-series-2.zarr/0/1/0/0/0/0/0 | Bin 0 -> 685 bytes .../data/bf2raw/fake-series-2.zarr/1/.zattrs | 38 +++++++++++++++ .../data/bf2raw/fake-series-2.zarr/1/.zgroup | 3 ++ .../bf2raw/fake-series-2.zarr/1/0/.zarray | 17 +++++++ .../bf2raw/fake-series-2.zarr/1/0/0/0/0/0/0 | Bin 0 -> 1616 bytes .../bf2raw/fake-series-2.zarr/1/1/.zarray | 17 +++++++ .../bf2raw/fake-series-2.zarr/1/1/0/0/0/0/0 | Bin 0 -> 562 bytes .../bf2raw/fake-series-2.zarr/OME/.zattrs | 3 ++ .../bf2raw/fake-series-2.zarr/OME/.zgroup | 3 ++ .../fake-series-2.zarr/OME/METADATA.ome.xml | 1 + tests/data/bf2raw/plate-rows-2.zarr/.zattrs | 28 +++++++++++ tests/data/bf2raw/plate-rows-2.zarr/.zgroup | 3 ++ tests/data/bf2raw/plate-rows-2.zarr/A/.zgroup | 3 ++ .../data/bf2raw/plate-rows-2.zarr/A/1/.zattrs | 8 ++++ .../data/bf2raw/plate-rows-2.zarr/A/1/.zgroup | 3 ++ .../bf2raw/plate-rows-2.zarr/A/1/0/.zattrs | 41 ++++++++++++++++ .../bf2raw/plate-rows-2.zarr/A/1/0/.zgroup | 3 ++ .../bf2raw/plate-rows-2.zarr/A/1/0/0/.zarray | 17 +++++++ .../plate-rows-2.zarr/A/1/0/0/0/0/0/0/0 | Bin 0 -> 1604 bytes .../bf2raw/plate-rows-2.zarr/A/1/0/1/.zarray | 17 +++++++ .../plate-rows-2.zarr/A/1/0/1/0/0/0/0/0 | Bin 0 -> 685 bytes tests/data/bf2raw/plate-rows-2.zarr/B/.zgroup | 3 ++ .../data/bf2raw/plate-rows-2.zarr/B/1/.zattrs | 8 ++++ .../data/bf2raw/plate-rows-2.zarr/B/1/.zgroup | 3 ++ .../bf2raw/plate-rows-2.zarr/B/1/0/.zattrs | 41 ++++++++++++++++ .../bf2raw/plate-rows-2.zarr/B/1/0/.zgroup | 3 ++ .../bf2raw/plate-rows-2.zarr/B/1/0/0/.zarray | 17 +++++++ .../plate-rows-2.zarr/B/1/0/0/0/0/0/0/0 | Bin 0 -> 1616 bytes .../bf2raw/plate-rows-2.zarr/B/1/0/1/.zarray | 17 +++++++ .../plate-rows-2.zarr/B/1/0/1/0/0/0/0/0 | Bin 0 -> 562 bytes .../data/bf2raw/plate-rows-2.zarr/OME/.zattrs | 3 ++ .../data/bf2raw/plate-rows-2.zarr/OME/.zgroup | 3 ++ .../plate-rows-2.zarr/OME/METADATA.ome.xml | 1 + tests/test_bf2raw.py | 44 ++++++++++++++++++ 41 files changed, 429 insertions(+) create mode 100644 tests/data/bf2raw/fake-series-2.zarr/.zattrs create mode 100644 tests/data/bf2raw/fake-series-2.zarr/.zgroup create mode 100644 tests/data/bf2raw/fake-series-2.zarr/0/.zattrs create mode 100644 tests/data/bf2raw/fake-series-2.zarr/0/.zgroup create mode 100644 tests/data/bf2raw/fake-series-2.zarr/0/0/.zarray create mode 100644 tests/data/bf2raw/fake-series-2.zarr/0/0/0/0/0/0/0 create mode 100644 tests/data/bf2raw/fake-series-2.zarr/0/1/.zarray create mode 100644 tests/data/bf2raw/fake-series-2.zarr/0/1/0/0/0/0/0 create mode 100644 tests/data/bf2raw/fake-series-2.zarr/1/.zattrs create mode 100644 tests/data/bf2raw/fake-series-2.zarr/1/.zgroup create mode 100644 tests/data/bf2raw/fake-series-2.zarr/1/0/.zarray create mode 100644 tests/data/bf2raw/fake-series-2.zarr/1/0/0/0/0/0/0 create mode 100644 tests/data/bf2raw/fake-series-2.zarr/1/1/.zarray create mode 100644 tests/data/bf2raw/fake-series-2.zarr/1/1/0/0/0/0/0 create mode 100644 tests/data/bf2raw/fake-series-2.zarr/OME/.zattrs create mode 100644 tests/data/bf2raw/fake-series-2.zarr/OME/.zgroup create mode 100644 tests/data/bf2raw/fake-series-2.zarr/OME/METADATA.ome.xml create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/.zattrs create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/.zattrs create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zattrs create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/.zarray create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/0/0/0/0/0 create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/0/1/.zarray create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/A/1/0/1/0/0/0/0/0 create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/.zattrs create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/0/.zattrs create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/0/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/0/0/.zarray create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/0/0/0/0/0/0/0 create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/.zarray create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/0/0/0/0/0 create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/OME/.zattrs create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/OME/.zgroup create mode 100644 tests/data/bf2raw/plate-rows-2.zarr/OME/METADATA.ome.xml create mode 100644 tests/test_bf2raw.py diff --git a/tests/data/bf2raw/fake-series-2.zarr/.zattrs b/tests/data/bf2raw/fake-series-2.zarr/.zattrs new file mode 100644 index 00000000..c5eadb8a --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/.zattrs @@ -0,0 +1,3 @@ +{ + "bioformats2raw.layout" : 3 +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/.zgroup b/tests/data/bf2raw/fake-series-2.zarr/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/0/.zattrs b/tests/data/bf2raw/fake-series-2.zarr/0/.zattrs new file mode 100644 index 00000000..90cf50e1 --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/0/.zattrs @@ -0,0 +1,38 @@ +{ + "multiscales" : [ { + "metadata" : { + "method" : "loci.common.image.SimpleImageScaler", + "version" : "Bio-Formats 6.9.1" + }, + "axes" : [ { + "name" : "t", + "type" : "time" + }, { + "name" : "c", + "type" : "channel" + }, { + "name" : "z", + "type" : "space" + }, { + "name" : "y", + "type" : "space" + }, { + "name" : "x", + "type" : "space" + } ], + "datasets" : [ { + "path" : "0", + "coordinateTransformations" : [ { + "scale" : [ 1.0, 1.0, 1.0, 1.0, 1.0 ], + "type" : "scale" + } ] + }, { + "path" : "1", + "coordinateTransformations" : [ { + "scale" : [ 1.0, 1.0, 1.0, 2.0, 2.0 ], + "type" : "scale" + } ] + } ], + "version" : "0.4" + } ] +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/0/.zgroup b/tests/data/bf2raw/fake-series-2.zarr/0/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/0/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/0/0/.zarray b/tests/data/bf2raw/fake-series-2.zarr/0/0/.zarray new file mode 100644 index 00000000..58baf32f --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/0/0/.zarray @@ -0,0 +1,17 @@ +{ + "chunks" : [ 1, 1, 1, 512, 512 ], + "compressor" : { + "clevel" : 5, + "blocksize" : 0, + "shuffle" : 1, + "cname" : "lz4", + "id" : "blosc" + }, + "dtype" : "|u1", + "fill_value" : 0, + "filters" : null, + "order" : "C", + "shape" : [ 1, 1, 1, 512, 512 ], + "dimension_separator" : "/", + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/0/0/0/0/0/0/0 b/tests/data/bf2raw/fake-series-2.zarr/0/0/0/0/0/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..6c56e50e3346be64f6510a3964189a2f6c160022 GIT binary patch literal 1604 zcmZQ#RAgjeU}0cjU}A7#V_=W~vP_v77$kvMo`I1;?*B(4V-r&|a|=r=Ya3fTdk04+ zXBSsDcMnf5Zy#Sj|A4@t;E>R;@QBE$=$P2J_=LoyYCcR`i91)=9bpB_Kwc3?w;Ph{s|K&O`bA!+VmMSXU(27ci#L33l}Y3vUJ(< z6)RV*UbA-H`VAX5ZQinV+x8thckSM@ci;X42M--Sa`f2o6DLodK6Ccm`3n~>UA}Vl z+VvYZZ{5Cg_ul;n4<9{#^7PsB7cXDEe)IO-`wt&Keg5+G+xH(ofBpXR_uqd8MkZz! zRyKAHPA+a9UOs*SK_OugQ894|NhxU=Svh$HMI~hwRW)@DO)YI5T|IpRLk7k@{0vP0 y(ZN4{2I05-42%f!GS0%T1E((*t)>;D@@CT12^Hg*n9E^Z!PK7Ii~Az=|wF>wh= zDQOv5Ie7&|C1n*=HFXV5Eo~iLJ$(}R;@QBE$=$P2J_=LoyYCcR`i91)=9bpB_Kwc3?w;Ph{s|K&O`bA!+VmMSXU(27ci#L33l}Y3 zvUJ(<6)RV*UbA-H`VAX5ZQinV+x8thckSM@ci;X42M--Sa`f2o6DLodK6Ccm`3n~> zUA}Vl+VvYZZ{5Cg_ul;n4<9{#^7PsB7cXDEe)IO-`wt&Keg5+G+xH(ofBpXR_uqd8 zMkZz!RyKAHPA+a9UOs*SK_OugQ894|NhxU=Svh$HMI~hwRW)@DO)YI5T|IpRLk7m( ztp6G18UHhi@G~&|M+ay58HC^RGcclykMf6a2wVvOrzLJ+M*07d^b|GRa@6pR=uw{$ K69O+F*#Q6r6%M`t literal 0 HcmV?d00001 diff --git a/tests/data/bf2raw/fake-series-2.zarr/1/1/.zarray b/tests/data/bf2raw/fake-series-2.zarr/1/1/.zarray new file mode 100644 index 00000000..fee11a32 --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/1/1/.zarray @@ -0,0 +1,17 @@ +{ + "chunks" : [ 1, 1, 1, 256, 256 ], + "compressor" : { + "clevel" : 5, + "blocksize" : 0, + "shuffle" : 1, + "cname" : "lz4", + "id" : "blosc" + }, + "dtype" : "|u1", + "fill_value" : 0, + "filters" : null, + "order" : "C", + "shape" : [ 1, 1, 1, 256, 256 ], + "dimension_separator" : "/", + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/1/1/0/0/0/0/0 b/tests/data/bf2raw/fake-series-2.zarr/1/1/0/0/0/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..313b070c9b0a990bc631dab546ce79f29ea25679 GIT binary patch literal 562 zcmZQ#RAgje03skZVq#zr0kWikv;ZR`gFHz5{|zG(GYcylI|nBhHxDl#zkr~Su!yLb zxP+vXw2Z8ryn>>VvWlvjx`w8fwvMizz6q12Oq(%l&b$SSmMmMbYR$S0o3?D*v1`x1 z1BZ?rJ8|mFxeJ%BT)T1W&bdm_kpT2zi@$1h&1|}9Z4lW)(0U;4F2`L#l o1tk?V4J{o#1A5ud`2YX^5C6Gvx`ToB3O@r#(Fg^P0>JSE0Ev}`o&W#< literal 0 HcmV?d00001 diff --git a/tests/data/bf2raw/fake-series-2.zarr/OME/.zattrs b/tests/data/bf2raw/fake-series-2.zarr/OME/.zattrs new file mode 100644 index 00000000..fec53b33 --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/OME/.zattrs @@ -0,0 +1,3 @@ +{ + "series" : [ "0", "1" ] +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/OME/.zgroup b/tests/data/bf2raw/fake-series-2.zarr/OME/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/OME/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/fake-series-2.zarr/OME/METADATA.ome.xml b/tests/data/bf2raw/fake-series-2.zarr/OME/METADATA.ome.xml new file mode 100644 index 00000000..6c869aec --- /dev/null +++ b/tests/data/bf2raw/fake-series-2.zarr/OME/METADATA.ome.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/.zattrs b/tests/data/bf2raw/plate-rows-2.zarr/.zattrs new file mode 100644 index 00000000..9bcb2b28 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/.zattrs @@ -0,0 +1,28 @@ +{ + "bioformats2raw.layout" : 3, + "plate" : { + "columns" : [ { + "name" : "1" + } ], + "name" : "Plate Name 0", + "wells" : [ { + "path" : "A/1", + "rowIndex" : 0, + "columnIndex" : 0 + }, { + "path" : "B/1", + "rowIndex" : 1, + "columnIndex" : 0 + } ], + "field_count" : 1, + "rows" : [ { + "name" : "A" + }, { + "name" : "B" + } ], + "acquisitions" : [ { + "id" : 0 + } ], + "version" : "0.4" + } +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/.zgroup b/tests/data/bf2raw/plate-rows-2.zarr/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/.zgroup b/tests/data/bf2raw/plate-rows-2.zarr/A/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/A/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/1/.zattrs b/tests/data/bf2raw/plate-rows-2.zarr/A/1/.zattrs new file mode 100644 index 00000000..f87f4246 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/A/1/.zattrs @@ -0,0 +1,8 @@ +{ + "well" : { + "images" : [ { + "path" : "0", + "acquisition" : 0 + } ] + } +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/1/.zgroup b/tests/data/bf2raw/plate-rows-2.zarr/A/1/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/A/1/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zattrs b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zattrs new file mode 100644 index 00000000..d1e42664 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zattrs @@ -0,0 +1,41 @@ +{ + "multiscales" : [ { + "metadata" : { + "method" : "loci.common.image.SimpleImageScaler", + "version" : "Bio-Formats 6.9.1" + }, + "axes" : [ { + "name" : "t", + "type" : "time" + }, { + "name" : "c", + "type" : "channel" + }, { + "unit" : "micrometer", + "name" : "z", + "type" : "space" + }, { + "unit" : "micrometer", + "name" : "y", + "type" : "space" + }, { + "unit" : "micrometer", + "name" : "x", + "type" : "space" + } ], + "datasets" : [ { + "path" : "0", + "coordinateTransformations" : [ { + "scale" : [ 1.0, 1.0, 1.0, 1.0, 1.0 ], + "type" : "scale" + } ] + }, { + "path" : "1", + "coordinateTransformations" : [ { + "scale" : [ 1.0, 1.0, 1.0, 2.0, 2.0 ], + "type" : "scale" + } ] + } ], + "version" : "0.4" + } ] +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zgroup b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/.zarray b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/.zarray new file mode 100644 index 00000000..58baf32f --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/.zarray @@ -0,0 +1,17 @@ +{ + "chunks" : [ 1, 1, 1, 512, 512 ], + "compressor" : { + "clevel" : 5, + "blocksize" : 0, + "shuffle" : 1, + "cname" : "lz4", + "id" : "blosc" + }, + "dtype" : "|u1", + "fill_value" : 0, + "filters" : null, + "order" : "C", + "shape" : [ 1, 1, 1, 512, 512 ], + "dimension_separator" : "/", + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/0/0/0/0/0 b/tests/data/bf2raw/plate-rows-2.zarr/A/1/0/0/0/0/0/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..6c56e50e3346be64f6510a3964189a2f6c160022 GIT binary patch literal 1604 zcmZQ#RAgjeU}0cjU}A7#V_=W~vP_v77$kvMo`I1;?*B(4V-r&|a|=r=Ya3fTdk04+ zXBSsDcMnf5Zy#Sj|A4@t;E>R;@QBE$=$P2J_=LoyYCcR`i91)=9bpB_Kwc3?w;Ph{s|K&O`bA!+VmMSXU(27ci#L33l}Y3vUJ(< z6)RV*UbA-H`VAX5ZQinV+x8thckSM@ci;X42M--Sa`f2o6DLodK6Ccm`3n~>UA}Vl z+VvYZZ{5Cg_ul;n4<9{#^7PsB7cXDEe)IO-`wt&Keg5+G+xH(ofBpXR_uqd8MkZz! zRyKAHPA+a9UOs*SK_OugQ894|NhxU=Svh$HMI~hwRW)@DO)YI5T|IpRLk7k@{0vP0 y(ZN4{2I05-42%f!GS0%T1E((*t)>;D@@CT12^Hg*n9E^Z!PK7Ii~Az=|wF>wh= zDQOv5Ie7&|C1n*=HFXV5Eo~iLJ$(}R;@QBE$=$P2J_=LoyYCcR`i91)=9bpB_Kwc3?w;Ph{s|K&O`bA!+VmMSXU(27ci#L33l}Y3 zvUJ(<6)RV*UbA-H`VAX5ZQinV+x8thckSM@ci;X42M--Sa`f2o6DLodK6Ccm`3n~> zUA}Vl+VvYZZ{5Cg_ul;n4<9{#^7PsB7cXDEe)IO-`wt&Keg5+G+xH(ofBpXR_uqd8 zMkZz!RyKAHPA+a9UOs*SK_OugQ894|NhxU=Svh$HMI~hwRW)@DO)YI5T|IpRLk7m( ztp6G18UHhi@G~&|M+ay58HC^RGcclykMf6a2wVvOrzLJ+M*07d^b|GRa@6pR=uw{$ K69O+F*#Q6r6%M`t literal 0 HcmV?d00001 diff --git a/tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/.zarray b/tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/.zarray new file mode 100644 index 00000000..fee11a32 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/.zarray @@ -0,0 +1,17 @@ +{ + "chunks" : [ 1, 1, 1, 256, 256 ], + "compressor" : { + "clevel" : 5, + "blocksize" : 0, + "shuffle" : 1, + "cname" : "lz4", + "id" : "blosc" + }, + "dtype" : "|u1", + "fill_value" : 0, + "filters" : null, + "order" : "C", + "shape" : [ 1, 1, 1, 256, 256 ], + "dimension_separator" : "/", + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/0/0/0/0/0 b/tests/data/bf2raw/plate-rows-2.zarr/B/1/0/1/0/0/0/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..313b070c9b0a990bc631dab546ce79f29ea25679 GIT binary patch literal 562 zcmZQ#RAgje03skZVq#zr0kWikv;ZR`gFHz5{|zG(GYcylI|nBhHxDl#zkr~Su!yLb zxP+vXw2Z8ryn>>VvWlvjx`w8fwvMizz6q12Oq(%l&b$SSmMmMbYR$S0o3?D*v1`x1 z1BZ?rJ8|mFxeJ%BT)T1W&bdm_kpT2zi@$1h&1|}9Z4lW)(0U;4F2`L#l o1tk?V4J{o#1A5ud`2YX^5C6Gvx`ToB3O@r#(Fg^P0>JSE0Ev}`o&W#< literal 0 HcmV?d00001 diff --git a/tests/data/bf2raw/plate-rows-2.zarr/OME/.zattrs b/tests/data/bf2raw/plate-rows-2.zarr/OME/.zattrs new file mode 100644 index 00000000..db84cffa --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/OME/.zattrs @@ -0,0 +1,3 @@ +{ + "series" : [ "A/1/0", "B/1/0" ] +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/OME/.zgroup b/tests/data/bf2raw/plate-rows-2.zarr/OME/.zgroup new file mode 100644 index 00000000..aa4bf031 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/OME/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} \ No newline at end of file diff --git a/tests/data/bf2raw/plate-rows-2.zarr/OME/METADATA.ome.xml b/tests/data/bf2raw/plate-rows-2.zarr/OME/METADATA.ome.xml new file mode 100644 index 00000000..4945aa05 --- /dev/null +++ b/tests/data/bf2raw/plate-rows-2.zarr/OME/METADATA.ome.xml @@ -0,0 +1 @@ +ExperimentPlate 0 of 1PlateAcquisition 0 of 1Image Description 01234567890ABCDEF1234567890ABCDEF12345678Image Description 11234567890ABCDEF1234567890ABCDEF12345678 \ No newline at end of file diff --git a/tests/test_bf2raw.py b/tests/test_bf2raw.py new file mode 100644 index 00000000..fc7049b6 --- /dev/null +++ b/tests/test_bf2raw.py @@ -0,0 +1,44 @@ +import pathlib + +import numpy as np +import pytest +import zarr + +from ome_zarr.io import parse_url +from ome_zarr.bioformats2raw import bioformats2raw +from ome_zarr.reader import Multiscales, Reader +from ome_zarr.writer import write_image + + +class TestBf2raw: + + def assert_data(self, path, shape, plate, mode="r"): + loc = parse_url(path, mode=mode) + assert loc, f"no zarr found at {path}" + reader = Reader(loc) + nodes = list(reader()) + for node in nodes: + if bioformats2raw.matches(node.zarr): + pass + elif Multiscales.matches(node.zarr): + assert node.data[0].shape == shape + assert np.max(node.data[0]) > 0 + elif "OME" in str(node.zarr): + assert "series" in node.metadata + else: + raise Exception(node) + + @pytest.mark.parametrize( + "path,plate", + ( + ("fake-series-2.zarr", False), + ("plate-rows-2.zarr", True), + ) + ) + def test_static_data(self, request, path, plate): + shape = (1, 1, 1, 512, 512) + self.assert_data( + f"{request.fspath.dirname}/data/bf2raw/{path}", + shape, + plate + ) From 5c867628065be0337fc14988224c71764c9225fb Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 22 Sep 2022 08:48:21 +0200 Subject: [PATCH 14/20] Add placeholder for bf2raw reading in docs --- docs/source/api.rst | 1 + docs/source/api/bioformats2raw.rst | 5 +++++ docs/source/python.rst | 10 ++++++++++ 3 files changed, 16 insertions(+) create mode 100644 docs/source/api/bioformats2raw.rst diff --git a/docs/source/api.rst b/docs/source/api.rst index 34b9713a..e7ce32d5 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -5,6 +5,7 @@ API :maxdepth: 3 api/cli + api/bioformats2raw api/csv api/data api/format diff --git a/docs/source/api/bioformats2raw.rst b/docs/source/api/bioformats2raw.rst new file mode 100644 index 00000000..c0fc8065 --- /dev/null +++ b/docs/source/api/bioformats2raw.rst @@ -0,0 +1,5 @@ +Format (``ome_zarr.bioformats2raw``) +============================ + +.. automodule:: ome_zarr.bioformat2raw + :members: diff --git a/docs/source/python.rst b/docs/source/python.rst index f6bbdc26..053adf04 100644 --- a/docs/source/python.rst +++ b/docs/source/python.rst @@ -109,3 +109,13 @@ the data is available as `dask` arrays:: viewer = napari.view_image(dask_data, channel_axis=0) if __name__ == '__main__': napari.run() + +Reading bioformats2raw filesets +------------------------------- + +The output from bioformats2raw encapsulates _multiple_ OME-NGFF images. +This structure has been added to the OME-NGFF specification as transitional +`"bioformats2raw.layout" `_ +metadata. To read such filesets: + +TBD From d954ba2106c1f663add11803231da611c33e020f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Sep 2022 06:50:24 +0000 Subject: [PATCH 15/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_bf2raw.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/test_bf2raw.py b/tests/test_bf2raw.py index fc7049b6..4f12f568 100644 --- a/tests/test_bf2raw.py +++ b/tests/test_bf2raw.py @@ -4,14 +4,13 @@ import pytest import zarr -from ome_zarr.io import parse_url from ome_zarr.bioformats2raw import bioformats2raw +from ome_zarr.io import parse_url from ome_zarr.reader import Multiscales, Reader from ome_zarr.writer import write_image class TestBf2raw: - def assert_data(self, path, shape, plate, mode="r"): loc = parse_url(path, mode=mode) assert loc, f"no zarr found at {path}" @@ -33,12 +32,8 @@ def assert_data(self, path, shape, plate, mode="r"): ( ("fake-series-2.zarr", False), ("plate-rows-2.zarr", True), - ) + ), ) def test_static_data(self, request, path, plate): shape = (1, 1, 1, 512, 512) - self.assert_data( - f"{request.fspath.dirname}/data/bf2raw/{path}", - shape, - plate - ) + self.assert_data(f"{request.fspath.dirname}/data/bf2raw/{path}", shape, plate) From 5105916a60fb152b91c415c9270dd9211020b2a4 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 22 Sep 2022 08:54:03 +0200 Subject: [PATCH 16/20] Fix build issues --- docs/source/api/bioformats2raw.rst | 4 ++-- tests/test_bf2raw.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/source/api/bioformats2raw.rst b/docs/source/api/bioformats2raw.rst index c0fc8065..bf6bc234 100644 --- a/docs/source/api/bioformats2raw.rst +++ b/docs/source/api/bioformats2raw.rst @@ -1,5 +1,5 @@ Format (``ome_zarr.bioformats2raw``) -============================ +==================================== -.. automodule:: ome_zarr.bioformat2raw +.. automodule:: ome_zarr.bioformats2raw :members: diff --git a/tests/test_bf2raw.py b/tests/test_bf2raw.py index 4f12f568..2e02bd34 100644 --- a/tests/test_bf2raw.py +++ b/tests/test_bf2raw.py @@ -1,13 +1,9 @@ -import pathlib - import numpy as np import pytest -import zarr from ome_zarr.bioformats2raw import bioformats2raw from ome_zarr.io import parse_url from ome_zarr.reader import Multiscales, Reader -from ome_zarr.writer import write_image class TestBf2raw: From 28155d58a8c6fc342f726b2abee67af0088ad300 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 22 Sep 2022 09:14:20 +0200 Subject: [PATCH 17/20] Update documentation --- docs/source/api/bioformats2raw.rst | 4 ++-- ome_zarr/bioformats2raw.py | 36 ++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/docs/source/api/bioformats2raw.rst b/docs/source/api/bioformats2raw.rst index bf6bc234..5d6c892c 100644 --- a/docs/source/api/bioformats2raw.rst +++ b/docs/source/api/bioformats2raw.rst @@ -1,5 +1,5 @@ -Format (``ome_zarr.bioformats2raw``) -==================================== +Bioformats2raw(``ome_zarr.bioformats2raw``) +=========================================== .. automodule:: ome_zarr.bioformats2raw :members: diff --git a/ome_zarr/bioformats2raw.py b/ome_zarr/bioformats2raw.py index 35a49648..d5467031 100644 --- a/ome_zarr/bioformats2raw.py +++ b/ome_zarr/bioformats2raw.py @@ -1,5 +1,13 @@ -""" -Spec definitions +"""Spec extension for reading bioformats2raw.layout. + +This specification detects and reads filesets which were created by +bioformats2raw and therefore can have multiple multiscale image groups +present. Each such image will be returned by the [ome_zarr.reader.Reader] +as a separate [ome_zarr.reader.Node], but metadata which has been parsed +from the OME-XML metadata associated with the specific image will be +attached. + +TBD: Example """ import logging @@ -22,6 +30,10 @@ class bioformats2raw(Base): + """A spec-type for reading multi-image filesets OME-XML + metadata. + """ + @staticmethod def matches(zarr: ZarrLocation) -> bool: layout = zarr.root_attrs.get("bioformats2raw.layout", None) @@ -30,7 +42,7 @@ def matches(zarr: ZarrLocation) -> bool: def __init__(self, node: Node) -> None: super().__init__(node) try: - data = self.handle(node) + data = self._handle(node) if data.plates: _logger.info("Plates detected. Skipping implicit loading") else: @@ -51,8 +63,12 @@ def __init__(self, node: Node) -> None: except Exception: _logger.exception("failed to parse metadata") - def fix_xml(self, ns: str, elem: ET.Element) -> None: - """ + def _fix_xml(self, ns: str, elem: ET.Element) -> None: + """Correct invalid OME-XML. + + Some versions of bioformats2raw did not include a MetadataOnly + tag. + Note: elem.insert() was not updating the object correctly. """ @@ -79,7 +95,8 @@ def fix_xml(self, ns: str, elem: ET.Element) -> None: if remove: elem.remove(remove) - def parse_xml(self, filename: str) -> ome_types.model.OME: + def _parse_xml(self, filename: str) -> ome_types.model.OME: + """Generate [ome_types.model.OME] from OME-XML""" # Parse the file and find the current schema root = ET.parse(filename) m = re.match(r"\{.*\}", root.getroot().tag) @@ -87,7 +104,7 @@ def parse_xml(self, filename: str) -> ome_types.model.OME: # Update the XML to include MetadataOnly for child in list(root.iter()): - self.fix_xml(ns, child) + self._fix_xml(ns, child) fixed = ET.tostring(root.getroot()).decode() # Write file out for ome_types @@ -96,8 +113,9 @@ def parse_xml(self, filename: str) -> ome_types.model.OME: t.flush() return ome_types.from_xml(t.name) - def handle(self, node: Node) -> ome_types.model.OME: + def _handle(self, node: Node) -> ome_types.model.OME: + """Main parsing method which looks for OME/METADATA.ome.xml""" metadata = node.zarr.subpath("OME/METADATA.ome.xml") _logger.info("Looking for metadata in %s", metadata) if os.path.exists(metadata): - return self.parse_xml(metadata) + return self._parse_xml(metadata) From eb12b2b281f876b867cd611f430d3e8ccbe3192c Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 22 Sep 2022 09:27:39 +0200 Subject: [PATCH 18/20] Attempt to fix pre.yml --- .github/workflows/pre.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre.yml b/.github/workflows/pre.yml index 41e05888..7ed3d6de 100644 --- a/.github/workflows/pre.yml +++ b/.github/workflows/pre.yml @@ -41,6 +41,7 @@ jobs: run: | python -m pip install -U pip setuptools wheel pytest python -m pip install -r requirements/requirements-dev.txt + python -m pip install . # Without -e for plugins tox -e ${{ matrix.toxenv }} --pre # If something goes wrong, we can open an issue in the repo From 6962d49f8f4e91c3dd67affc773f89a3dc0939b2 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 21 Dec 2022 09:01:26 +0100 Subject: [PATCH 19/20] TMP implicit --- ome_zarr/bioformats2raw.py | 21 ++++++++++++++++++--- tests/test_bf2raw.py | 35 ++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/ome_zarr/bioformats2raw.py b/ome_zarr/bioformats2raw.py index d5467031..f84b31b6 100644 --- a/ome_zarr/bioformats2raw.py +++ b/ome_zarr/bioformats2raw.py @@ -36,21 +36,34 @@ class bioformats2raw(Base): @staticmethod def matches(zarr: ZarrLocation) -> bool: + """Pass if the metadata for the zgroup contains + `{"bioformats2raw.layout": 3}`""" layout = zarr.root_attrs.get("bioformats2raw.layout", None) + _logger.error(f"layout={layout == 3} zarr={zarr}") return layout == 3 def __init__(self, node: Node) -> None: + """Load metadata from the three sources associated with this + specification: the OME zgroup metadata, the OME-XML file, and + the images zgroups themselves. + """ super().__init__(node) try: + + # Load OME/METADATA.ome.xml data = self._handle(node) if data.plates: _logger.info("Plates detected. Skipping implicit loading") else: - series = node.zarr.root_attrs.get("series", None) - if series is not None: - node.metadata["series"] = series + # Load the OME/ zgroup metadata + ome = node.zarr.create("OME") + if ome.exists: + series_metadata = ome.zarr.root_attrs.get("series", None) + if series_metadata is not None: + node.metadata["series"] = series_metadata + # Load each individual image for idx, image in enumerate(data.images): series = node.zarr.create(str(idx)) assert series.exists(), f"{series} is missing" @@ -59,7 +72,9 @@ def __init__(self, node: Node) -> None: if subnode: subnode.metadata["ome-xml:index"] = idx subnode.metadata["ome-xml:image"] = image + node.metadata["ome-xml"] = data + except Exception: _logger.exception("failed to parse metadata") diff --git a/tests/test_bf2raw.py b/tests/test_bf2raw.py index 2e02bd34..66a69e2e 100644 --- a/tests/test_bf2raw.py +++ b/tests/test_bf2raw.py @@ -3,33 +3,46 @@ from ome_zarr.bioformats2raw import bioformats2raw from ome_zarr.io import parse_url -from ome_zarr.reader import Multiscales, Reader +from ome_zarr.reader import Multiscales, Node, Reader +TEST_DATA = ( + ("fake-series-2.zarr", False), + ("plate-rows-2.zarr", True), +) class TestBf2raw: + def assert_data(self, path, shape, plate, mode="r"): loc = parse_url(path, mode=mode) assert loc, f"no zarr found at {path}" reader = Reader(loc) nodes = list(reader()) + + assert any([ + bioformats2raw.matches(node.zarr) for node in nodes + ]), "plugin not detected" + + WHY WITH PLATE + for node in nodes: if bioformats2raw.matches(node.zarr): - pass + assert "series" in node.metadata, node.metadata elif Multiscales.matches(node.zarr): assert node.data[0].shape == shape assert np.max(node.data[0]) > 0 elif "OME" in str(node.zarr): - assert "series" in node.metadata + pass # Doesn't get parsed directly else: raise Exception(node) - @pytest.mark.parametrize( - "path,plate", - ( - ("fake-series-2.zarr", False), - ("plate-rows-2.zarr", True), - ), - ) - def test_static_data(self, request, path, plate): + @pytest.mark.parametrize("path,plate", TEST_DATA) + def test_read_static_data(self, request, path, plate): shape = (1, 1, 1, 512, 512) self.assert_data(f"{request.fspath.dirname}/data/bf2raw/{path}", shape, plate) + + @pytest.mark.parametrize("path,plate", TEST_DATA) + def test_node_static_data(self, request, path, plate): + zarr = parse_url(f"{request.fspath.dirname}/data/bf2raw/{path}") + import pdb; pdb.set_trace() + node = Node(zarr, []) + print(node.specs) From 836dfd2967880c29a6f75872b933d1fe3d49b28b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 May 2023 07:08:54 +0000 Subject: [PATCH 20/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ome_zarr/bioformats2raw.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ome_zarr/bioformats2raw.py b/ome_zarr/bioformats2raw.py index f84b31b6..d875ffd2 100644 --- a/ome_zarr/bioformats2raw.py +++ b/ome_zarr/bioformats2raw.py @@ -49,13 +49,11 @@ def __init__(self, node: Node) -> None: """ super().__init__(node) try: - # Load OME/METADATA.ome.xml data = self._handle(node) if data.plates: _logger.info("Plates detected. Skipping implicit loading") else: - # Load the OME/ zgroup metadata ome = node.zarr.create("OME") if ome.exists: @@ -88,7 +86,6 @@ def _fix_xml(self, ns: str, elem: ET.Element) -> None: """ if elem.tag == f"{ns}Pixels": - must_have = {f"{ns}BinData", f"{ns}TiffData", f"{ns}MetadataOnly"} children = {x.tag for x in elem}