Implement scverse datastucture

.conda/meta.yaml

@@ -16,30 +16,33 @@ build:
 
 requirements:
   host:
-    - python >=3.7
+    - python >=3.8
     - pip!=22.1 # https://github.com/pypa/pip/issues/11110
     - flit
     - setuptools_scm
     - pytoml
     - importlib_metadata
 
   run:
-    - python >=3.7
-    - anndata >=0.7.6
-    - scanpy >=1.6.0
+    - python >=3.8
+    - anndata >=0.9rc1
+    - awkward >=2.1.0
+    - mudata >=0.2.2
+    - scanpy >=1.9.3
     - pandas >=1.5,<2
     - numpy >=1.17.0
     - scipy
     - parasail-python
     - scikit-learn
     - python-levenshtein
     - python-igraph !=0.10.0,!=0.10.1
-    - adjusttext >=0.7
     - networkx >=2.5
     - squarify
-    - tqdm >=4.44.1
     - airr >=1.2
+    - tqdm >=4.63
+    - adjusttext >=0.7
     - numba >=0.41.0
+    - pooch >=1.7.0
 
 test:
   source_files:

.github/workflows/conda.yml

@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [master]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   tests:
     if: "!contains(github.event.head_commit.message, 'skip ci')"

.github/workflows/docs.yml

@@ -8,6 +8,10 @@ on:
   release:
     types: [created]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   docs:
     if: "!contains(github.event.head_commit.message, 'skip ci')"
@@ -17,9 +21,10 @@ jobs:
       matrix:
         python-version: [3.9]
         os:
-         - ubuntu-latest
-         # - macos-latest
-         - windows-latest
+          - ubuntu-latest
+          # - macos-latest
+          - windows-latest
+
     steps:
       - uses: actions/checkout@v2
         with:
@@ -67,9 +72,7 @@ jobs:
           pip install .[doc,test,rpack,dandelion]
       - name: run sphinx
         run: |
-          # cd docs && make html SPHINXOPTS="-W --keep-going"
-          # TODO do not ignore sphinx warnings
-          cd docs && make html
+          cd docs && make html SPHINXOPTS="-W --keep-going"
 
       - name: Get target folder for page deploy from github ref
         if: ( matrix.os == 'ubuntu-latest' ) &&  ( matrix.python-version == '3.8' )

.github/workflows/test.yml

@@ -8,6 +8,10 @@ on:
   schedule:
     - cron: "0 5 * * 0"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     if: "!contains(github.event.head_commit.message, 'skip ci')"

.pre-commit-config.yaml

@@ -4,3 +4,20 @@ repos:
     hooks:
       - id: black
         language_version: python3.10
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+  - repo: https://github.com/myint/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place
+          - --remove-all-unused-imports
+          - --remove-unused-variable
+          - --ignore-init-module-imports
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict

README.rst

@@ -50,7 +50,7 @@ The case study from our paper is available `here <https://icbi-lab.github.io/sci
 
 Installation
 ^^^^^^^^^^^^
-You need to have Python 3.7 or newer installed on your system. If you don't have
+You need to have Python 3.8 or newer installed on your system. If you don't have
 Python installed, we recommend installing `Miniconda <https://docs.conda.io/en/latest/miniconda.html>`_.
 
 There are several alternative options to install scirpy:

docs/api.rst

@@ -1,3 +1,5 @@
+.. _api:
+
 API
 ===
 
@@ -20,10 +22,17 @@ Input/Output: `io`
 .. currentmodule:: scirpy
 
 .. note::
-   In scirpy v0.7.0 the way VDJ data is stored in `adata.obs` has changed to
-   be fully compliant with the `AIRR Rearrangement <https://docs.airr-community.org/en/latest/datarep/rearrangements.html#productive>`__
-   schema. Please use :func:`~scirpy.io.upgrade_schema` to make `AnnData` objects
-   from previous scirpy versions compatible with the most recent scirpy workflow.
+    **scirpy's data structure has been updated in v0.13.0.**
+
+    Previously, receptor data was expanded into columns of `adata.obs`, now they are stored as an :term:`awkward array` in `adata.obsm["airr"]`. 
+    Moreover, we now use :class:`~mudata.MuData` to handle paired transcriptomics and :term:`AIRR` data. 
+
+    :class:`~anndata.AnnData` objects created with older versions of scirpy can be upgraded with :func:`scirpy.io.upgrade_schema` to be compatible with the latest version of scirpy.
+
+    Please check out
+
+     * the `release notes <https://github.com/scverse/scirpy/releases/tag/v0.13.0>`_ for details about the changes and
+     * the documentation about :ref:`Scirpy's data structure <data-structure>`
 
    .. autosummary::
       :toctree: ./generated
@@ -37,6 +46,7 @@ formats.
 .. autosummary::
    :toctree: ./generated
 
+   io.read_h5mu
    io.read_h5ad
    io.read_10x_vdj
    io.read_tracer
@@ -75,10 +85,25 @@ Preprocessing: `pp`
 .. autosummary::
    :toctree: ./generated
 
-   pp.merge_with_ir
-   pp.merge_airr_chains
+   pp.index_chains
+   pp.merge_airr
    pp.ir_dist
 
+Get: `get`
+----------
+
+The `get` module allows retrieving :term:`AIRR` data stored in `adata.obsm["airr"]` as a per-cell :class:`~pandas.DataFrame`
+or :class:`~pandas.Series`. 
+
+.. module:: scirpy.get
+.. currentmodule:: scirpy
+
+.. autosummary::
+   :toctree: ./generated
+
+   get.airr
+   get.obs_context
+   get.airr_context
 
 Tools: `tl`
 -----------
@@ -211,6 +236,9 @@ Datasets: `datasets`
 .. module:: scirpy.datasets
 .. currentmodule:: scirpy
 
+Example datasets
+^^^^^^^^^^^^^^^^
+
 .. autosummary::
    :toctree: ./generated
 
@@ -241,6 +269,7 @@ Utility functions: `util`
 .. autosummary::
    :toctree: ./generated
 
+   util.DataHandler
    util.graph.layout_components
    util.graph.layout_fr_size_aware
    util.graph.igraph_from_sparse_matrix

docs/conf.py

@@ -76,6 +76,10 @@
     sklearn=("https://scikit-learn.org/stable/", None),
     networkx=("https://networkx.org/documentation/networkx-1.10/", None),
     dandelion=("https://sc-dandelion.readthedocs.io/en/latest/", None),
+    muon=("https://muon.readthedocs.io/en/latest", None),
+    mudata=("https://mudata.readthedocs.io/en/latest/", None),
+    awkward=("https://awkward-array.org/doc/main/", None),
+    pooch=("https://www.fatiando.org/pooch/latest/", None),
 )
 
 
@@ -130,7 +134,8 @@ def setup(app):
     ("py:class", "D.get(k,d), also set D[k]=d if k not in D"),
     ("py:class", "None.  Update D from mapping/iterable E and F."),
     ("py:class", "an object providing a view on D's values"),
-    # Will work once scipy 1.8 is released
-    ("py:class", "scipy.sparse.base.spmatrix"),
-    ("py:class", "scipy.sparse.csr.csr_matrix"),
+    # don't know why these are not working
+    ("py:class", "seaborn.matrix.ClusterGrid"),
+    ("py:meth", "mudata.MuData.update"),
+    ("py:class", "awkward.highlevel.Array"),
 ]