Ome zarr v0.5 writing

.github/workflows/build.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.11', '3.12']
         os: ['windows-latest', 'macos-latest', 'ubuntu-latest']
     steps:
       - uses: actions/checkout@v4

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_third_party = dask,fsspec,numcodecs,numpy,pytest,scipy,setuptools,skimage,zarr
+known_third_party = dask,numcodecs,numpy,pytest,scipy,setuptools,skimage,zarr
 multi_line_output = 3
 include_trailing_comma = True
 force_grid_wrap = 0

.readthedocs.yml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.12"
     # You can also specify other tool versions:
     # nodejs: "16"
     # rust: "1.55"

docs/requirements.txt

@@ -1,7 +1,7 @@
-sphinx==7.1.2
-sphinx-rtd-theme==1.3.0
+sphinx==8.1.3
+sphinx-rtd-theme==3.0.2
 fsspec==2023.6.0
-zarr
+zarr==v3.0.0-beta.3
 dask
 numpy
 scipy

ome_zarr/data.py

@@ -127,9 +127,9 @@ def create_zarr(
     """Generate a synthetic image pyramid with labels."""
     pyramid, labels = method()
 
-    loc = parse_url(zarr_directory, mode="w")
+    loc = parse_url(zarr_directory, mode="w", fmt=fmt)
     assert loc
-    grp = zarr.group(loc.store)
+    grp = zarr.group(loc.store, zarr_format=fmt.zarr_format)
     axes = None
     size_c = 1
     if fmt.version not in ("0.1", "0.2"):
@@ -195,30 +195,46 @@ def create_zarr(
         grp,
         axes=axes,
         storage_options=storage_options,
+        fmt=fmt,
         metadata={"omero": image_data},
     )
 
     if labels:
         labels_grp = grp.create_group("labels")
-        labels_grp.attrs["labels"] = [label_name]
+        if fmt.zarr_format == 2:
+            labels_grp.attrs["labels"] = [label_name]
+        else:
+            labels_grp.attrs["ome"] = {"labels": [label_name]}
 
         label_grp = labels_grp.create_group(label_name)
         if axes is not None:
             # remove channel axis for masks
             axes = axes.replace("c", "")
-        write_multiscale(labels, label_grp, axes=axes)
+        write_multiscale(labels, label_grp, axes=axes, fmt=fmt)
 
         colors = []
         properties = []
         for x in range(1, 9):
             rgba = [randrange(0, 256) for i in range(4)]
             colors.append({"label-value": x, "rgba": rgba})
             properties.append({"label-value": x, "class": f"class {x}"})
-        label_grp.attrs["image-label"] = {
-            "version": fmt.version,
-            "colors": colors,
-            "properties": properties,
-            "source": {"image": "../../"},
-        }
+        if fmt.zarr_format == 2:
+            label_grp.attrs["image-label"] = {
+                "version": fmt.version,
+                "colors": colors,
+                "properties": properties,
+                "source": {"image": "../../"},
+            }
+        else:
+            ome_attrs = label_grp.attrs["ome"]
+            label_grp.attrs["ome"] = {
+                "image-label": {
+                    "version": fmt.version,
+                    "colors": colors,
+                    "properties": properties,
+                    "source": {"image": "../../"},
+                },
+                **ome_attrs,
+            }
 
     return grp

ome_zarr/format.py

@@ -5,7 +5,7 @@
 from collections.abc import Iterator
 from typing import Any, Optional
 
-from zarr.storage import FSStore
+from zarr.storage import LocalStore, RemoteStore
 
 LOGGER = logging.getLogger("ome_zarr.format")
 
@@ -25,6 +25,7 @@ def format_implementations() -> Iterator["Format"]:
     """
     Return an instance of each format implementation, newest to oldest.
     """
+    yield FormatV05()
     yield FormatV04()
     yield FormatV03()
     yield FormatV02()
@@ -55,12 +56,17 @@ class Format(ABC):
     def version(self) -> str:  # pragma: no cover
         raise NotImplementedError()
 
+    @property
+    @abstractmethod
+    def zarr_format(self) -> int:  # pragma: no cover
+        raise NotImplementedError()
+
     @abstractmethod
     def matches(self, metadata: dict) -> bool:  # pragma: no cover
         raise NotImplementedError()
 
     @abstractmethod
-    def init_store(self, path: str, mode: str = "r") -> FSStore:
+    def init_store(self, path: str, mode: str = "r") -> RemoteStore:
         raise NotImplementedError()
 
     # @abstractmethod
@@ -129,14 +135,31 @@ class FormatV01(Format):
     def version(self) -> str:
         return "0.1"
 
+    @property
+    def zarr_format(self) -> int:
+        return 2
+
     def matches(self, metadata: dict) -> bool:
         version = self._get_metadata_version(metadata)
         LOGGER.debug("%s matches %s?", self.version, version)
         return version == self.version
 
-    def init_store(self, path: str, mode: str = "r") -> FSStore:
-        store = FSStore(path, mode=mode, dimension_separator=".")
-        LOGGER.debug("Created legacy flat FSStore(%s, %s)", path, mode)
+    def init_store(self, path: str, mode: str = "r") -> RemoteStore | LocalStore:
+        """
+        Not ideal. Stores should remain hidden
+        "dimension_separator" is specified at array creation time
+        """
+
+        if path.startswith(("http", "s3")):
+            store = RemoteStore.from_url(
+                path,
+                storage_options=None,
+                read_only=(mode in ("r", "r+", "a")),
+            )
+        else:
+            # No other kwargs supported
+            store = LocalStore(path, read_only=(mode in ("r", "r+", "a")))
+        LOGGER.debug("Created nested RemoteStore(%s, %s)", path, mode)
         return store
 
     def generate_well_dict(
@@ -180,32 +203,6 @@ class FormatV02(FormatV01):
     def version(self) -> str:
         return "0.2"
 
-    def init_store(self, path: str, mode: str = "r") -> FSStore:
-        """
-        Not ideal. Stores should remain hidden
-        TODO: could also check dimension_separator
-        """
-
-        kwargs = {
-            "dimension_separator": "/",
-            "normalize_keys": False,
-        }
-
-        mkdir = True
-        if "r" in mode or path.startswith(("http", "s3")):
-            # Could be simplified on the fsspec side
-            mkdir = False
-        if mkdir:
-            kwargs["auto_mkdir"] = True
-
-        store = FSStore(
-            path,
-            mode=mode,
-            **kwargs,
-        )  # TODO: open issue for using Path
-        LOGGER.debug("Created nested FSStore(%s, %s, %s)", path, mode, kwargs)
-        return store
-
 
 class FormatV03(FormatV02):  # inherits from V02 to avoid code duplication
     """
@@ -343,4 +340,18 @@ def validate_coordinate_transformations(
                         )
 
 
-CurrentFormat = FormatV04
+class FormatV05(FormatV04):
+    """
+    Changelog: added FormatV05 (December 2024)
+    """
+
+    @property
+    def version(self) -> str:
+        return "0.5"
+
+    @property
+    def zarr_format(self) -> int:
+        return 3
+
+
+CurrentFormat = FormatV05

ome_zarr/io.py

@@ -3,14 +3,14 @@
 Primary entry point is the :func:`~ome_zarr.io.parse_url` method.
 """
 
-import json
 import logging
 from pathlib import Path
 from typing import Optional, Union
 from urllib.parse import urljoin
 
 import dask.array as da
-from zarr.storage import FSStore
+import zarr
+from zarr.storage import LocalStore, RemoteStore, StoreLike
 
 from .format import CurrentFormat, Format, detect_format
 from .types import JSONDict
@@ -20,7 +20,7 @@
 
 class ZarrLocation:
     """
-    IO primitive for reading and writing Zarr data. Uses FSStore for all
+    IO primitive for reading and writing Zarr data. Uses a store for all
     data access.
 
     No assumptions about the existence of the given path string are made.
@@ -29,7 +29,7 @@ class ZarrLocation:
 
     def __init__(
         self,
-        path: Union[Path, str, FSStore],
+        path: StoreLike,
         mode: str = "r",
         fmt: Format = CurrentFormat(),
     ) -> None:
@@ -40,18 +40,21 @@ def __init__(
             self.__path = str(path.resolve())
         elif isinstance(path, str):
             self.__path = path
-        elif isinstance(path, FSStore):
+        elif isinstance(path, RemoteStore):
             self.__path = path.path
+        elif isinstance(path, LocalStore):
+            self.__path = str(path.root)
         else:
             raise TypeError(f"not expecting: {type(path)}")
 
         loader = fmt
         if loader is None:
             loader = CurrentFormat()
-        self.__store: FSStore = (
-            path if isinstance(path, FSStore) else loader.init_store(self.__path, mode)
+        self.__store: RemoteStore = (
+            path
+            if isinstance(path, RemoteStore)
+            else loader.init_store(self.__path, mode)
         )
-
         self.__init_metadata()
         detected = detect_format(self.__metadata, loader)
         LOGGER.debug("ZarrLocation.__init__ %s detected: %s", path, detected)
@@ -67,16 +70,44 @@ def __init_metadata(self) -> None:
         """
         Load the Zarr metadata files for the given location.
         """
-        self.zarray: JSONDict = self.get_json(".zarray")
-        self.zgroup: JSONDict = self.get_json(".zgroup")
+        self.zgroup: JSONDict = {}
+        self.zarray: JSONDict = {}
         self.__metadata: JSONDict = {}
         self.__exists: bool = True
-        if self.zgroup:
-            self.__metadata = self.get_json(".zattrs")
-        elif self.zarray:
-            self.__metadata = self.get_json(".zattrs")
-        else:
-            self.__exists = False
+        # If we want to *create* a new zarr v2 group, we need to specify
+        # zarr_format. This is not needed for reading.
+        zarr_format = None
+        if self.__mode == "w":
+            # For now, let's support writing of zarr v2
+            # TODO: handle writing of zarr v2 OR zarr v3
+            if self.__fmt.version in ("0.1", "0.2", "0.3", "0.4"):
+                zarr_format = 2
+            else:
+                zarr_format = 3
+        try:
+            group = zarr.open_group(
+                store=self.__store, path="/", mode=self.__mode, zarr_format=zarr_format
+            )
+            self.zgroup = group.attrs.asdict()
+            # For zarr v3, everything is under the "ome" namespace
+            if "ome" in self.zgroup:
+                self.zgroup = self.zgroup["ome"]
+            self.__metadata = self.zgroup
+        except (ValueError, FileNotFoundError):
+            try:
+                array = zarr.open_array(
+                    store=self.__store,
+                    path="/",
+                    mode=self.__mode,
+                    zarr_format=zarr_format,
+                )
+                self.zarray = array.attrs.asdict()
+                self.__metadata = self.zarray
+            except (ValueError, FileNotFoundError):
+                # We actually get a ValueError when the file is not found
+                # /zarr-python/src/zarr/abc/store.py", line 189, in _check_writable
+                #   raise ValueError("store mode does not support writing")
+                self.__exists = False
 
     def __repr__(self) -> str:
         """Print the path as well as whether this is a group or an array."""
@@ -104,7 +135,7 @@ def path(self) -> str:
         return self.__path
 
     @property
-    def store(self) -> FSStore:
+    def store(self) -> RemoteStore:
         """Return the initialized store for this location"""
         assert self.__store is not None
         return self.__store
@@ -154,11 +185,9 @@ def get_json(self, subpath: str) -> JSONDict:
         All other exceptions log at the ERROR level.
         """
         try:
-            data = self.__store.get(subpath)
-            if not data:
-                return {}
-            return json.loads(data)
-        except KeyError:
+            array_or_group = zarr.open_group(store=self.__store, path="/")
+            return array_or_group.attrs.asdict()
+        except (KeyError, FileNotFoundError):
             LOGGER.debug("JSON not found: %s", subpath)
             return {}
         except Exception:
@@ -193,10 +222,11 @@ def _isfile(self) -> bool:
         Return whether the current underlying implementation
         points to a local file or not.
         """
-        return self.__store.fs.protocol == "file" or self.__store.fs.protocol == (
-            "file",
-            "local",
-        )
+        # return self.__store.fs.protocol == "file" or self.__store.fs.protocol == (
+        #     "file",
+        #     "local",
+        # )
+        return isinstance(self.__store, LocalStore)
 
     def _ishttp(self) -> bool:
         """

ome_zarr/scale.py

@@ -123,7 +123,7 @@ def __assert_values(self, pyramid: list[np.ndarray]) -> None:
 
     def __create_group(
         self, store: MutableMapping, base: np.ndarray, pyramid: list[np.ndarray]
-    ) -> zarr.hierarchy.Group:
+    ) -> zarr.Group:
         """Create group and datasets."""
         grp = zarr.group(store)
         grp.create_dataset("base", data=base)