[python] Carrara/tiledbv3 URI support

apis/python/pyproject.toml

@@ -131,4 +131,5 @@ markers = [
     "slow: mark test as slow",
     "spatialdata: test of SpatialData integration",
     "medium_runner: tests that are too large for the MacOS GitHub runners",
+    "carrara: carrara-related tests",
 ]

apis/python/remote_tests/carrara/README.md

@@ -0,0 +1,32 @@
+# Carrara Tests
+
+TileDB v3 aka Carrara introduces a new URL schema which supports relative paths and group membership lifecycle changes.
+
+This directory contains unit tests for Carrara-specific behavior.
+
+NB: this is EXPERIMENTAL and subject to change.
+
+## Running the tests
+
+Prerequisites:
+
+- You have a user account in a Carrara deployment
+- You have set up your tiledb profile so that you can log into the account with a profile name
+- You have created a teamspace for testing use
+
+With that, set the environment variables and install package dependencies:
+
+```bash
+pip install tiledb_client
+export CARRARA_TEST_PROFILE="..."           # Profile name
+export CARRARA_TEST_WORKSPACE="..."         # Workspace name (must match workspace for the profile)
+export CARRARA_TEST_TEAMSPACE="..."         # Teamspace to use for tests
+```
+
+And _manually_ run the tests from the top-level repo directory:
+
+```bash
+pytest apis/python/remote_tests/carrara/ --carrara
+```
+
+Tests will be run within the specified workspace/teamspace, i.e., objects will be created at `tiledb://workspace/teamspace/...`

apis/python/remote_tests/carrara/_util.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import pathlib
+import urllib.parse
+from contextlib import contextmanager
+from typing import Any
+
+
+def parse_tiledb_uri(uri: str) -> tuple[str, str, str | None]:
+    """Given a tiledb URI, return (workspace, teamspace, path)."""
+
+    p_uri = urllib.parse.urlparse(uri)
+    workspace = p_uri.netloc
+
+    p_path = pathlib.Path(p_uri.path)
+    if len(p_path.parts) < 2:
+        raise ValueError(f"Not a tiledb URI - missing teamspace: {uri}")
+    teamspace = p_path.parts[1]
+
+    path = None if len(p_path.parts) == 2 else str(pathlib.PurePath(*p_path.parts[2:]))
+
+    return workspace, teamspace, path
+
+
+def join_tiledb_uri(workspace: str, teamspace: str, path: tuple[str] | str) -> str:
+    path = (path,) if isinstance(path, str) else path
+    return f"tiledb://{workspace}/{teamspace}/{'/'.join(path)}"
+
+
+def get_asset_info(uri: str) -> dict[str, Any]:
+    import tiledb.client
+
+    _, teamspace, path = parse_tiledb_uri(uri)
+    return tiledb.client.assets.get_asset(path, teamspace=teamspace).to_dict()
+
+
+@contextmanager
+def carrara_cleanup_asset(url: str) -> str:
+    import tiledb.client
+
+    _, teamspace, path = parse_tiledb_uri(url)
+    try:
+        yield url
+    finally:
+        tiledb.client.assets.delete_asset(path, teamspace=teamspace, delete_storage=True)
+
+
+def s3_from_tiledb_uri(tiledb_uri: str) -> str:
+    """Given a tiledb URI, return the object's S3 URL.
+
+    This relies on the teamspace bucket being owned by the test user (see notes in conftest.py)"""
+
+    import tiledb.client
+
+    assert tiledb_uri.startswith("tiledb://")
+
+    _, teamspace, path = parse_tiledb_uri(tiledb_uri)
+    asset_info = tiledb.client.assets.get_asset(path, teamspace=teamspace)
+    s3_url = asset_info.uri
+    assert s3_url.startswith("s3://")
+
+    return s3_url

apis/python/remote_tests/carrara/conftest.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Generator
+from uuid import uuid4
+
+import anndata as ad
+import pytest
+import scanpy as sc
+import scipy.sparse as sp
+
+import tiledbsoma as soma
+import tiledb
+
+# Base Carrara URI used for all tests. NB: the teamspace must be owned by the
+# test user, allowing access to the S3 path (using tiledb.client API).
+#
+PROFILE_NAME = os.getenv("CARRARA_TEST_PROFILE") or "qa"
+WORKSPACE_NAME = os.getenv("CARRARA_TEST_WORKSPACE") or "TileDB-Inc."
+TEAMSPACE_NAME = os.getenv("CARRARA_TEST_TEAMSPACE") or "bruce-uat"
+TEST_FOLDER = os.getenv("CARRARA_TEST_FOLDER") or "remote_test"
+BASE_URI = f"tiledb://{WORKSPACE_NAME}/{TEAMSPACE_NAME}/{TEST_FOLDER}"
+
+
+def pytest_addoption(parser):
+    parser.addoption("--carrara", action="store_true", default=False, help="run Carrara tests")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--carrara"):
+        # --carrara given in cli: do not skip slow tests
+        return
+    skip_carrara = pytest.mark.skip(reason="need --carrara option to run")
+    for item in items:
+        if "carrara" in item.keywords:
+            item.add_marker(skip_carrara)
+
+
+@pytest.fixture(scope="session")
+def carrara_context() -> soma.SOMATileDBContext:
+    import tiledb.client
+
+    tiledb.client.login(profile_name=PROFILE_NAME)
+    assert tiledb.client.workspaces.get_workspace(tiledb.client.client.get_workspace_id()).name == WORKSPACE_NAME
+    return soma.SOMATileDBContext(tiledb_ctx=tiledb.Ctx())
+
+
+@pytest.fixture
+def carrara_array_path() -> Generator[str, None, None]:
+    """Fixture returns an Array path that will be recursively deleted after test finishes."""
+    path = f"{BASE_URI}/{uuid4()}"
+    yield path
+
+    tiledb.Array.delete_array(path)
+
+
+@pytest.fixture
+def carrara_group_path() -> Generator[str, None, None]:
+    """Fixture returns a Group path that will be recursively deleted after test finishes."""
+    path = f"{BASE_URI}/{uuid4()}"
+    yield path
+
+    try:
+        with tiledb.Group(path, mode="m") as G:
+            G.delete(recursive=True)
+    except tiledb.TileDBError:
+        pass
+
+
+@pytest.fixture
+def small_pbmc() -> ad.AnnData:
+    adata = sc.datasets.pbmc3k_processed()
+
+    # trim it down to something smallish
+
+    adata = adata[0 : adata.n_obs // 2, 0 : adata.n_vars // 2].copy()
+    del adata.obsm["X_draw_graph_fr"]
+    del adata.obsm["X_pca"]
+
+    del adata.uns["neighbors"]
+    del adata.uns["pca"]
+    del adata.uns["rank_genes_groups"]
+
+    del adata.obsp["distances"]
+
+    adata.X = sp.csr_matrix(adata.X)
+
+    return adata