🐛 AnnDataAccessor.to_memory() broke?

[falexwolf]

From here: https://github.com/laminlabs/lamindb/actions/runs/12605574163/job/35134434226?pr=2319

adata_subset.to_memory()
------------------


---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/numpy/_core/fromnumeric.py:3557, in ndim(a)
   3556 try:
-> 3557     return a.ndim
   3558 except AttributeError:

AttributeError: 'tuple' object has no attribute 'ndim'

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
Cell In[13], line 1
----> 1 adata_subset.to_memory()

File ~/work/lamindb/lamindb/lamindb/core/storage/_anndata_accessor.py:584, in _AnnDataAttrsMixin.to_memory(self)
    583 def to_memory(self):
--> 584     adata = AnnData(**self.to_dict())
    585     return adata

File ~/work/lamindb/lamindb/lamindb/core/storage/_anndata_accessor.py:576, in _AnnDataAttrsMixin.to_dict(self)
    574         get_attr = getattr(self, attr)
    575         for key in self._attrs_keys[attr]:
--> 576             prepare_adata[attr][key] = _to_memory(get_attr[key])
    578 if "raw" in self._attrs_keys:
    579     prepare_adata["raw"] = self.raw.to_dict()

File ~/work/lamindb/lamindb/lamindb/core/storage/_anndata_accessor.py:450, in _MapAccessor.__getitem__(self, key)
    448     return _try_backed_full(self.elem[key])
    449 else:
--> 450     return registry.safer_read_partial(self.elem[key],indices=self.indices)

File ~/work/lamindb/lamindb/lamindb/core/storage/_anndata_accessor.py:144, in AccessRegistry.__getattr__.<locals>.wrapper(*args, **kwargs)
    142     arg_module = get_module_name(arg)
    143     if arg_module in func_registry:
--> 144         return func_registry[arg_module](*args,**kwargs)
    145 raise ValueError(f"{func_name} is not registered for this module.")

File ~/work/lamindb/lamindb/lamindb/core/storage/_anndata_accessor.py:236, in safer_read_partial(elem, indices)
    231         raise ValueError(
    232             "Can not get a subset of the element of type"
    233             f" {type(elem).__name__} with an empty spec."
    234         )
    235 else:
--> 236     result = read_elem_partial(elem,indices=indices)
    237 if indices_inverse is None:
    238     return result

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/anndata/_io/specs/registry.py:503, in read_elem_partial(elem, items, indices, modifiers)
    499 """Read part of an element from an on disk store."""
    500 read_partial = _REGISTRY.get_partial_read(
    501     type(elem), get_spec(elem), frozenset(modifiers)
    502 )
--> 503 return read_partial(elem,items=items,indices=indices)

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/anndata/_io/specs/methods.py:769, in read_sparse_partial(elem, items, indices)
    764 @_REGISTRY.register_read_partial(H5Group, IOSpec("csc_matrix", "0.1.0"))
    765 @_REGISTRY.register_read_partial(H5Group, IOSpec("csr_matrix", "0.1.0"))
    766 @_REGISTRY.register_read_partial(ZarrGroup, IOSpec("csc_matrix", "0.1.0"))
    767 @_REGISTRY.register_read_partial(ZarrGroup, IOSpec("csr_matrix", "0.1.0"))
    768 def read_sparse_partial(elem, *, items=None, indices=(slice(None), slice(None))):
--> 769     return sparse_dataset(elem)[indices]

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/anndata/_core/sparse_dataset.py:431, in BaseCompressedSparseDataset.__getitem__(self, index)
    427 # read into memory data if we do not override access methods
    428 elif not is_sparse_indexing_overridden(
    429     self.format, row_sp_matrix_validated, col_sp_matrix_validated
    430 ):
--> 431     sub = self.to_memory()[row_sp_matrix_validated,col_sp_matrix_validated]
    432 else:
    433     sub = mtx[row, col]

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/scipy/sparse/_index.py:30, in IndexMixin.__getitem__(self, key)
     29 def __getitem__(self, key):
---> 30     index, new_shape = self._validate_indices(key)
     32     # 1D array
     33     if len(index) == 1:

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/scipy/sparse/_index.py:218, in IndexMixin._validate_indices(self, key)
    216 elif idx is None:
    217     index_1st.append(idx)
--> 218 elif isinstance(idx, slice) or isintlike(idx):
    219     index_1st.append(idx)
    220     prelim_ndim += 1

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/scipy/sparse/_sputils.py:356, in isintlike(x)
    [351](https://github.com/laminlabs/lamindb/actions/runs/12605574163/job/35134434226?pr=2319#step:15:352) """Is x appropriate as an index into a sparse matrix? Returns True
    [352](https://github.com/laminlabs/lamindb/actions/runs/12605574163/job/35134434226?pr=2319#step:15:353) if it can be cast safely to a machine int.
    [353](https://github.com/laminlabs/lamindb/actions/runs/12605574163/job/35134434226?pr=2319#step:15:354) """
    354 # Fast-path check to eliminate non-scalar values. operator.index would
    355 # catch this case too, but the exception catching is slow.
--> 356 if np.ndim(x) != 0:
    357     return False
    358 try:

File /opt/hostedtoolcache/Python/3.12.8/x64/lib/python3.12/site-packages/numpy/_core/fromnumeric.py:[355](https://github.com/laminlabs/lamindb/actions/runs/12605574163/job/35134434226?pr=2319#step:15:356)9, in ndim(a)
   3557     return a.ndim
   3558 except AttributeError:
-> 3559     return asarray(a).ndim

ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (2,) + inhomogeneous part.

[falexwolf]

To not block everything else, I commented the call out here:

• 🚸 Only auto-search ._name_field on sub-classes of CanCurate #2319

It's strange that this seems only to be called in this notebook though.

[Koncopd]

I think it is a new scipy version, released yesterday.
Probably fixed here scverse/anndata#1806
I will try bounding scipy for now.

[Koncopd]

#2321
Lets keep this issue open to track if it is solved in anndata.

[Zethson]

This seems to be fixed in anndata and is in 0.11.2

https://anndata.readthedocs.io/en/latest/release-notes/index.html#v0-11-2

[Koncopd]

It is, yes, removing the pin makes all previous versions of anndata problematic.