Fix s3 writing and test with moto

pyproject.toml

@@ -47,7 +47,7 @@ sphinx-autodoc-typehints = { version = "*", optional = true }
 ipython = "^8.22.2"
 pytest = "^7.4.3"
 pytest-mock = "^3.14.0"
-
+moto = {extras = ["s3", "proxy"], version = "^5.0.13"}
 
 [tool.poetry.extras]
 test = ["pytest", "pytest-cov"]

src/ome2024_ngff_challenge/utils.py

@@ -309,10 +309,20 @@ def __repr__(self):
         )
 
     def check_or_delete_path(self):
-        # If this is local, then delete.
+        # Check remote bucket
         if self.bucket:
-            raise Exception(f"bucket set ({self.bucket}). Refusing to delete.")
+            check = f"s3://{self.bucket}/{self.path / 'zarr.json'}"
+            exists = self.zr_store._fs.exists(check)
+            # https://github.com/getmoto/moto/issues/2964
+            # try:
+            #     exists = bool(self.zr_store._fs.read_text(check))
+            # except FileNotFoundError:
+            #     exists = False
+
+            if exists and not self.overwrite:
+                raise Exception(f"{check} exists. Use --output-overwrite to overwrite")
 
+        # If this is local, then delete.
         if self.path.exists():
             # TODO: This should really be an option on zarr-python
             # as with tensorstore.

tests/test_resave.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
+import boto3
 import pytest
+from moto import mock_aws
 
 from ome2024_ngff_challenge import dispatch
 
+from .test_utils import mock_aio_aws
+
 #
 # Helpers
 #
@@ -120,7 +124,7 @@ def test_rocrate_full_example(tmp_path):
 
 
 #
-# Remote testing
+# Remote read testing
 #
 
 IDR_BUCKET = (
@@ -166,6 +170,83 @@ def test_remote_simple_with_download(tmp_path):
     )
 
 
+#
+# Remote write testing
+#
+
+# Use dummy AWS credentials
+AWS_REGION = "us-west-2"
+AWS_ACCESS_KEY_ID = "dummy_AWS_ACCESS_KEY_ID"
+AWS_SECRET_ACCESS_KEY = "dummy_AWS_SECRET_ACCESS_KEY"
+
+
+@pytest.fixture()
+def aws_credentials(monkeypatch):
+    monkeypatch.setenv("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID)
+    monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY)
+    monkeypatch.setenv("AWS_SECURITY_TOKEN", "testing")
+    monkeypatch.setenv("AWS_SESSION_TOKEN", "testing")
+
+
+@pytest.fixture(scope="session")
+def aws_region():
+    return AWS_REGION
+
+
+AWS_BUCKET = (
+    "--output-bucket=test-bucket",
+    "--output-region=us-west-2",
+)
+
+
+OVERWRITE = ["--output-overwrite"]
+
+
+@pytest.mark.parametrize(
+    ("first_flags", "second_flags", "expected_pass"),
+    [
+        pytest.param([], [], False, id="simple-twice-fail"),
+        pytest.param([], OVERWRITE, True, id="proper-usage-pass"),
+        pytest.param(OVERWRITE, OVERWRITE, True, id="double-safe-pass"),
+    ],
+)
+@mock_aws
+@pytest.mark.filterwarnings("ignore:datetime.datetime.utcnow")
+def test_remote_two_writes(first_flags, second_flags, expected_pass, monkeypatch):
+    with mock_aio_aws(monkeypatch):
+        s3 = boto3.client("s3", region_name="us-east-1")
+        s3.create_bucket(Bucket="test-bucket")
+
+        # First
+        converted = dispatch(
+            [
+                "resave",
+                "--cc-by",
+                "--output-script",  # By pass tensorstore
+                "data/2d.zarr",
+                "path/in/bucket/out.zarr",
+                *AWS_BUCKET,
+                *first_flags,
+            ]
+        )
+        assert converted
+
+        # Second
+        cmd = [
+            "resave",
+            "--cc-by",
+            "data/2d.zarr",
+            "path/in/bucket/out.zarr",
+            *AWS_BUCKET,
+            *second_flags,
+        ]
+        if expected_pass:
+            assert dispatch(cmd)
+        else:
+            with pytest.raises(Exception) as e_info:
+                dispatch(cmd)
+
+
 #
 # Local testing using files under data/
 #