Fix adata copy (coarse transition matrix) (#1223)

* chore: bump ruff pre-commit hook version

* Fix copy

* Fix some warnings

* Fix reading from adata

* Test on 3.12

* Update tox

* Use `macos-14`

* Drop 3.8 support

* Update pre-commits

* Update tests (pre-commit)

* Run more pre-commits

* Fix more pre-commits

* Switch mostly to `np.random.default_rng`

* Fix rest of the pre-commits

* Update ruff version

* Use 3.9 type-hints

* Update black-docs

* Fix workflow Python version

* Try setting up conda on MacOS

* Fix if statement

* Use `observed=False`

* Update test figure

* Fix test on 3.12

* Unpin pytest version

* Remove `worker_id`

* Regenerate GT figures

* Regenerate rest of the figures

* Fix seeds in log_odds tests

* Update `cluster_lineage` figures

* Fix GAMR in failed model

* Reduce CI verbosity

* Run `format-references`

* Polish `cite.rst`

* Fix extra `clustering_kwargs`

---------

Co-authored-by: Heinz-Alexander Fuetterer <[email protected]>

Author: michalk8

.github/workflows/lint.yml

@@ -31,8 +31,8 @@ jobs:
         python-version: '3.10'
 
     - name: Cache pre-commit
-      uses: actions/cache@v4
       if: ${{ matrix.lint-kind == 'code' }}
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pre-commit
         key: pre-commit-${{ env.pythonLocation }}-${{ hashFiles('**/.pre-commit-config.yaml') }}

.github/workflows/test.yml

@@ -19,11 +19,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python: ['3.8', '3.10', '3.11']
+        python: ['3.9', '3.10', '3.11', '3.12']
         slepc: [noslepc]
         include:
-        - os: macos-latest
-          python: '3.9'
+        - os: macos-14
+          python: '3.10'
           slepc: noslepc
         - os: ubuntu-latest
           python: '3.10'
@@ -41,11 +41,17 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install tox
 
+    - name: Setup Conda
+      if: runner.os == 'macOS'
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        auto-update-conda: true
+
     - name: Test
       run: |
-        tox -e py${{ matrix.python }}-${{ matrix.slepc }} -vv
+        tox -e py${{ matrix.python }}-${{ matrix.slepc }} -v
       env:
-        PYTEST_ADDOPTS: -vv
+        PYTEST_ADDOPTS: -v
 
     - name: Upload coverage
       uses: codecov/codecov-action@v4

.gitignore

@@ -168,3 +168,6 @@ docs/api/_autosummary
 
 # tests
 tests/figures
+
+# Ruff
+.ruff_cache

.pre-commit-config.yaml

@@ -7,22 +7,22 @@ default_stages:
 minimum_pre_commit_version: 3.0.0
 repos:
 - repo: https://github.com/psf/black
-  rev: 23.3.0
+  rev: 24.8.0
   hooks:
   - id: black
     additional_dependencies: [toml]
 - repo: https://github.com/timothycrosley/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
     additional_dependencies: [toml]
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.9.0
+  rev: v2.14.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, '2']
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.6.0
   hooks:
   - id: check-merge-conflict
   - id: check-ast
@@ -37,27 +37,27 @@ repos:
   - id: check-yaml
   - id: check-toml
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.4.0
+  rev: v3.17.0
   hooks:
   - id: pyupgrade
     args: [--py3-plus, --py38-plus, --keep-runtime-typing]
 - repo: https://github.com/asottile/blacken-docs
-  rev: 1.13.0
+  rev: 1.18.0
   hooks:
   - id: blacken-docs
-    additional_dependencies: [black==23.1.0]
+    additional_dependencies: [black==24.8.0]
 - repo: https://github.com/rstcheck/rstcheck
-  rev: v6.1.2
+  rev: v6.2.4
   hooks:
   - id: rstcheck
     additional_dependencies: [tomli]
     args: [--config=pyproject.toml]
 - repo: https://github.com/PyCQA/doc8
-  rev: v1.1.1
+  rev: v1.1.2
   hooks:
   - id: doc8
-- repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: v0.0.270
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.5
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix]

docs/_ext/typed_returns.py

@@ -1,5 +1,5 @@
 import re
-from typing import Iterable, Iterator, List
+from collections.abc import Iterable, Iterator
 
 from sphinx.application import Sphinx
 from sphinx.ext.napoleon import NumpyDocstring
@@ -18,9 +18,9 @@ def process_return(lines: Iterable[str]) -> Iterator[str]:
             yield line
 
 
-def _parse_returns_section(self: NumpyDocstring, section: str) -> List[str]:
+def _parse_returns_section(self: NumpyDocstring, section: str) -> list[str]:
     lines_raw = list(process_return(self._dedent(self._consume_to_next_section())))
-    lines: List[str] = self._format_block(":returns: ", lines_raw)
+    lines: list[str] = self._format_block(":returns: ", lines_raw)
     if lines and lines[-1]:
         lines.append("")
     return lines

docs/about/cite.rst

@@ -6,15 +6,12 @@ CellRank's :class:`~cellrank.kernels.VelocityKernel` with classical RNA velocity
 .. code-block:: bibtex
 
     @article{lange:22,
-        title = {CellRank for directed single-cell fate mapping},
-        author = {Lange, Marius and Bergen, Volker and Klein, Michal and Setty, Manu and
-                  Reuter, Bernhard and Bakhti, Mostafa and Lickert, Heiko and
-                  Ansari, Meshal and Schniering, Janine and Schiller, Herbert B. and
-                  Pe'er, Dana and Theis, Fabian J.},
-        journal = {Nat. Methods},
-        year = {2022},
-        doi = {10.1038/s41592-021-01346-6},
-        publisher = {Nature Publishing Group}
+        author    = {Lange, Marius and Bergen, Volker and Klein, Michal and Setty, Manu and Reuter, Bernhard and Bakhti, Mostafa and Lickert, Heiko and Ansari, Meshal and Schniering, Janine and Schiller, Herbert B. and Pe'er, Dana and Theis, Fabian J.},
+        publisher = {Nature Publishing Group},
+        doi       = {10.1038/s41592-021-01346-6},
+        journal   = {Nat. Methods},
+        title     = {CellRank for directed single-cell fate mapping},
+        year      = {2022},
     }
 
 If you are using the :class:`~cellrank.kernels.PseudotimeKernel`, :class:`~cellrank.kernels.CytoTRACEKernel`, :class:`~cellrank.kernels.RealTimeKernel`, or the :class:`~cellrank.kernels.VelocityKernel` with velocities inferred
@@ -23,15 +20,18 @@ from metabolic labeling data using the CellRank 2 approach, cite :cite:`weiler:2
 .. code-block:: bibtex
 
     @article{weiler:24,
-        title = {CellRank 2: unified fate mapping in multiview single-cell data},
-        author = {Weiler, Philipp and Lange, Marius and Klein, Michal and Pe{\textquotesingle}er, Dana and Theis, Fabian},
-        doi = {10.1038/s41592-024-02303-9},
-        url = {https://doi.org/10.1038/s41592-024-02303-9},
-        year = {2024},
-        journal = {Nature Methods},
-        volume = {21},
-        number = {7},
-        pages = {1196--1205},
+        author    = {Weiler, Philipp and Lange, Marius and Klein, Michal and Pe'er, Dana and Theis, Fabian},
+        publisher = {Springer Science and Business Media LLC},
+        url       = {https://doi.org/10.1038/s41592-024-02303-9},
+        doi       = {10.1038/s41592-024-02303-9},
+        issn      = {1548-7105},
+        journal   = {Nature Methods},
+        month     = jun,
+        number    = {7},
+        pages     = {1196--1205},
+        title     = {CellRank 2: unified fate mapping in multiview single-cell data},
+        volume    = {21},
+        year      = {2024},
     }
 
 In addition, if you use the :class:`~cellrank.estimators.GPCCA` estimator to compute initial, terminal or intermediate
@@ -42,12 +42,12 @@ please cite GPCCA :cite:`reuter:19` as:
 .. code-block:: bibtex
 
     @article{reuter:19,
-        author = {Reuter,Bernhard  and Fackeldey,Konstantin  and Weber,Marcus },
-        title = {Generalized Markov modeling of nonreversible molecular kinetics},
-        journal = {The Journal of Chemical Physics},
-        volume = {150},
-        number = {17},
-        pages = {174103},
-        year = {2019},
-        doi = {10.1063/1.5064530},
+      author  = {Reuter, Bernhard and Fackeldey, Konstantin and Weber, Marcus},
+      doi     = {10.1063/1.5064530},
+      journal = {The Journal of Chemical Physics},
+      number  = {17},
+      pages   = {174103},
+      title   = {Generalized Markov modeling of nonreversible molecular kinetics},
+      volume  = {150},
+      year    = {2019},
     }

docs/notebooks

@@ -804,16 +804,16 @@ @article{bastidas:17
 }
 
 @article{weiler:24,
-    title = {CellRank 2: unified fate mapping in multiview single-cell data},
-    volume = {21},
-    ISSN = {1548-7105},
-    url = {http://dx.doi.org/10.1038/s41592-024-02303-9},
-    DOI = {10.1038/s41592-024-02303-9},
-    number = {7},
-    journal = {Nature Methods},
-    publisher = {Springer Science and Business Media LLC},
-    author = {Weiler,  Philipp and Lange,  Marius and Klein,  Michal and Pe'er,  Dana and Theis,  Fabian},
-    year = {2024},
-    month = jun,
-    pages = {1196--1205}
+  author    = {Weiler, Philipp and Lange, Marius and Klein, Michal and Pe'er, Dana and Theis, Fabian},
+  publisher = {Springer Science and Business Media LLC},
+  url       = {http://dx.doi.org/10.1038/s41592-024-02303-9},
+  doi       = {10.1038/s41592-024-02303-9},
+  issn      = {1548-7105},
+  journal   = {Nature Methods},
+  month     = jun,
+  number    = {7},
+  pages     = {1196--1205},
+  title     = {CellRank 2: unified fate mapping in multiview single-cell data},
+  volume    = {21},
+  year      = {2024},
 }

docs/references.bib

@@ -7,7 +7,7 @@ name = "cellrank"
 dynamic = ["version"]
 description = "CellRank: dynamics from multi-view single-cell data"
 readme = "README.rst"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = {file = "LICENSE"}
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -19,10 +19,10 @@ classifiers = [
     "Operating System :: Microsoft :: Windows",
     "Typing :: Typed",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering :: Bio-Informatics",
     "Topic :: Scientific/Engineering :: Mathematics",
     "Topic :: Scientific/Engineering :: Visualization",
@@ -69,8 +69,7 @@ dev = [
     "tox>=4",
 ]
 test = [
-    "pytest>=7,<8.1.0",
-    "pytest-xdist>=3",
+    "pytest>=8",
     "pytest-mock>=3.5.0",
     "pytest-cov>=4",
     "coverage[toml]>=7",
@@ -106,7 +105,10 @@ include-package-data = true
 [tool.setuptools_scm]
 
 [tool.ruff]
-target-version = "py38"
+target-version = "py39"
+line-length = 120
+
+[tool.ruff.lint]
 exclude = [
     ".eggs",
     ".git",
@@ -136,7 +138,6 @@ ignore = [
     # Missing docstring in magic method
     "D105",
 ]
-line-length = 120
 select = [
     "D", # flake8-docstrings
     "E", # pycodestyle
@@ -154,21 +155,21 @@ select = [
     "T20",  # flake8-print
     "RET", # flake8-raise
 ]
-unfixable = ["B", "UP", "C4", "BLE", "T20", "RET"]
-[tool.ruff.per-file-ignores]
+unfixable = ["B", "C4", "BLE", "T20", "RET"]
+[tool.ruff.lint.per-file-ignores]
 "tests/*" = ["D"]
 "*/__init__.py" = ["F401"]
 "docs/*" = ["D"]
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "numpy"
-[tool.ruff.flake8-tidy-imports]
+[tool.ruff.lint.flake8-tidy-imports]
 ban-relative-imports = "all"
-[tool.ruff.flake8-quotes]
+[tool.ruff.lint.flake8-quotes]
 inline-quotes = "double"
 
 [tool.black]
 line-length = 120
-target-version = ['py38']
+target-version = ['py39']
 include = '\.pyi?$'
 
 [tool.isort]
@@ -237,7 +238,7 @@ legacy_tox_ini = """
 # TODO(michalk8): upgrade to `tox>=4.0` once `tox-conda` supports it
 requires = tox-conda
 isolated_build = true
-envlist = lint-code,py{3.8,3.9,3.10,3.11}-{slepc,noslepc}
+envlist = lint-code,py{3.9,3.10,3.11,3.12}-{slepc,noslepc}
 skip_missing_interpreters = true
 
 [testenv]

pyproject.toml

@@ -1,4 +1,5 @@
-from typing import Any, List, Optional, Sequence, Tuple, Union
+from collections.abc import Sequence
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
@@ -11,14 +12,14 @@
 
 
 def _create_colors(
-    base_color: Union[str, Tuple[float, float, float]],
+    base_color: Union[str, tuple[float, float, float]],
     n: int,
-    hue_range: Optional[Tuple[float, float]] = (-0.1, 0.1),
-    saturation_range: Optional[Tuple[float, float]] = (-0.3, 0.3),
-    value_range: Optional[Tuple[float, float]] = (-0.3, 0.3),
+    hue_range: Optional[tuple[float, float]] = (-0.1, 0.1),
+    saturation_range: Optional[tuple[float, float]] = (-0.3, 0.3),
+    value_range: Optional[tuple[float, float]] = (-0.3, 0.3),
     convert_to_rgb: bool = True,
     as_hex: bool = True,
-) -> List[Any]:
+) -> list[Any]:
     """Create variations of colors from base color.
 
     Parameters
@@ -74,7 +75,7 @@ def _create_colors(
     return res[::2]  # we've created twice as many colors, select every other
 
 
-def _convert_to_hex_colors(cols: Sequence[Any]) -> List[str]:
+def _convert_to_hex_colors(cols: Sequence[Any]) -> list[str]:
     if not all(colors.is_color_like(c) for c in cols):
         raise ValueError("Not all values are color-like.")
 
@@ -108,7 +109,7 @@ def _create_categorical_colors(n_categories: Optional[int] = None):
     raise RuntimeError(f"Unable to create `{n_categories}` colors.")
 
 
-def _insert_categorical_colors(seen_colors: Union[np.ndarray, List], n_categories: int):
+def _insert_categorical_colors(seen_colors: Union[np.ndarray, list], n_categories: int):
     seen_colors = set(_convert_to_hex_colors(seen_colors))
     candidates = list(filter(lambda c: c not in seen_colors, _create_categorical_colors()))[:n_categories]
 
@@ -135,7 +136,7 @@ def _get_black_or_white(value: float, cmap) -> str:
     return _contrasting_color(r, g, b)
 
 
-def _get_bg_fg_colors(color, sat_scale: Optional[float] = None) -> Tuple[str, str]:
+def _get_bg_fg_colors(color, sat_scale: Optional[float] = None) -> tuple[str, str]:
     if not colors.is_color_like(color):
         raise ValueError(f"Value `{color}` is not color-like.")
 
@@ -155,7 +156,7 @@ def _map_names_and_colors(
     series_query: pd.Series,
     colors_reference: Optional[np.array] = None,
     en_cutoff: Optional[float] = None,
-) -> Union[pd.Series, Tuple[pd.Series, List[Any]]]:
+) -> Union[pd.Series, tuple[pd.Series, list[Any]]]:
     """Map annotations and colors from one series to another.
 
     Parameters
@@ -270,7 +271,7 @@ def _map_names_and_colors(
     return (names_query_new, list(_convert_to_hex_colors(colors_query_new))) if process_colors else names_query_new
 
 
-def _compute_mean_color(cols: List[str]) -> str:
+def _compute_mean_color(cols: list[str]) -> str:
     """Compute mean color."""
     if not all(colors.is_color_like(c) for c in cols):
         raise ValueError(f"Not all values are valid colors `{cols}`.")