diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 4182b74797c..a827a6aa71d 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -219,7 +219,7 @@ jobs: echo "Processing $fn example..." python $fn done - workingDirectory: "examples/" + workingDirectory: "_samples/" # skip for PR if there is nothing to test, note that outside PR there is default 'unittests' condition: and(succeeded(), ne(variables['TEST_DIRS'], '')) displayName: "Examples" diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index c92c921ee01..8c409df6f88 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -65,6 +65,9 @@ jobs: - name: Full build for deployment if: github.event_name != 'pull_request' run: echo "SPHINX_FETCH_ASSETS=1" >> $GITHUB_ENV + - name: Disable Gallery build + if: matrix.target != 'html' + run: echo "SPHINX_ENABLE_GALLERY=0" >> $GITHUB_ENV - name: make ${{ matrix.target }} working-directory: ./docs run: make ${{ matrix.target }} --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going" diff --git a/.gitignore b/.gitignore index cbe31a9b316..d29d7fc696b 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,7 @@ coverage.xml docs/_build/ docs/source/api/ docs/source/generated/ +docs/source/gallery/ docs/source/*.md docs/source/_static/fetched-s3-assets diff --git a/README.md b/README.md index 82e370c91f5..e614376b23d 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ fig3, ax3 = acc.plot(values)

-For examples of plotting different metrics try running [this example file](examples/plotting.py). +For examples of plotting different metrics try running [this example file](_samples/plotting.py). ## Contribute! diff --git a/examples/bert_score-own_model.py b/_samples/bert_score-own_model.py similarity index 100% rename from examples/bert_score-own_model.py rename to _samples/bert_score-own_model.py diff --git a/examples/detection_map.py b/_samples/detection_map.py similarity index 100% rename from examples/detection_map.py rename to _samples/detection_map.py diff --git a/examples/plotting.py b/_samples/plotting.py similarity index 100% rename from examples/plotting.py rename to _samples/plotting.py diff --git a/examples/rouge_score-own_normalizer_and_tokenizer.py b/_samples/rouge_score-own_normalizer_and_tokenizer.py similarity index 100% rename from examples/rouge_score-own_normalizer_and_tokenizer.py rename to _samples/rouge_score-own_normalizer_and_tokenizer.py diff --git a/docs/source/conf.py b/docs/source/conf.py index 239490b59d2..5c2698306f0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,6 +30,7 @@ SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) SPHINX_FETCH_ASSETS = int(os.environ.get("SPHINX_FETCH_ASSETS", False)) SPHINX_PIN_RELEASE_VERSIONS = int(os.getenv("SPHINX_PIN_RELEASE_VERSIONS", False)) +SPHINX_ENABLE_GALLERY = int(os.getenv("SPHINX_ENABLE_GALLERY", True)) html_favicon = "_static/images/icon.svg" @@ -128,15 +129,22 @@ def _set_root_image_path(page_path: str) -> None: "sphinx.ext.autosummary", "sphinx.ext.napoleon", "sphinx.ext.mathjax", - "myst_parser", "sphinx.ext.autosectionlabel", - "nbsphinx", + "sphinx.ext.githubpages", "sphinx_autodoc_typehints", "sphinx_paramlinks", - "sphinx.ext.githubpages", - "lai_sphinx_theme.extensions.lightning", + "myst_parser", "matplotlib.sphinxext.plot_directive", + "lai_sphinx_theme.extensions.lightning", ] +if SPHINX_ENABLE_GALLERY: + extensions.append("sphinx_gallery.gen_gallery") +else: + # write a dummy file as placeholder + path_gallery = os.path.join(_PATH_HERE, "gallery") + os.makedirs(path_gallery, exist_ok=True) + with open(os.path.join(path_gallery, "index.rst"), "w") as fopen: + fopen.write("Gallery is disabled\n===================") # Set that source code from plotting is always included plot_include_source = True @@ -146,13 +154,19 @@ def _set_root_image_path(page_path: str) -> None: # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] -# https://berkeley-stat159-f17.github.io/stat159-f17/lectures/14-sphinx..html#conf.py-(cont.) -# https://stackoverflow.com/questions/38526888/embed-ipython-notebook-in-sphinx-document -# I execute the notebooks manually in advance. If notebooks test the code, -# they should be run at build time. -nbsphinx_execute = "never" -nbsphinx_allow_errors = True -nbsphinx_requirejs_path = "" +sphinx_gallery_conf = { + "examples_dirs": os.path.join(_PATH_ROOT, "examples"), # path to your example scripts + "gallery_dirs": "gallery", # path to where to save gallery generated output + "filename_pattern": ".", + "ignore_pattern": r"__init__\.py", + "example_extensions": {".py"}, + "line_numbers": True, + "promote_jupyter_magic": True, + "matplotlib_animations": True, + "abort_on_example_error": True, + # 'only_warn_on_example_error': True + "thumbnail_size": (400, 280), +} myst_update_mathjax = False @@ -162,7 +176,7 @@ def _set_root_image_path(page_path: str) -> None: ".rst": "restructuredtext", ".txt": "markdown", ".md": "markdown", - ".ipynb": "nbsphinx", + # ".ipynb": "nbsphinx", } # The master toctree document. diff --git a/docs/source/index.rst b/docs/source/index.rst index 880a6a2657e..58dac2c6fb1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -129,6 +129,7 @@ Or directly from conda pages/quickstart all-metrics + gallery/index pages/overview pages/plotting pages/implement diff --git a/examples/README.rst b/examples/README.rst new file mode 100644 index 00000000000..8df126e4d02 --- /dev/null +++ b/examples/README.rst @@ -0,0 +1,6 @@ +TorchMetrics' gallery +===================== + +Welcome to a comprehensive guide on leveraging TorchMetrics, that facilitates the precise and consistent evaluation of machine learning models. As an integral tool for developers and researchers, TorchMetrics offers an array of metrics critical for assessing model performance across a variety of applications. Whether you are fine-tuning a neural network, comparing model iterations, or tracking performance improvements, this page provides a gallery of real-world applications where Torch Metrics can be effectively implemented. + +By touring through this application gallery, users can gain insights into how TorchMetrics is utilized across different sectors and scale settings, empowering them with the knowledge to implement metrics effectively in their own projects. Whether your interest lies in academic research or commercial product development, the examples provided here will help demonstrate the versatility and utility of Torch Metrics in enhancing machine learning model assessment. diff --git a/examples/image/README.rst b/examples/image/README.rst new file mode 100644 index 00000000000..2dd804e9691 --- /dev/null +++ b/examples/image/README.rst @@ -0,0 +1,4 @@ +Image domain +============ + +Image-domain metrics are pivotal for gauging the performance of models in tasks like object detection, and segmentation. TorchMetrics provides a suite of specialized metrics designed for these purposes. Using these image-specific metrics from Torch Metrics helps in developing more precise and robust image-based models, ensuring that performance evaluations are both meaningful and directly applicable to practical vision tasks. diff --git a/examples/image/spatial_correlation_coef.py b/examples/image/spatial_correlation_coef.py new file mode 100644 index 00000000000..d5a5296aa82 --- /dev/null +++ b/examples/image/spatial_correlation_coef.py @@ -0,0 +1,57 @@ +""" +Spatial Correlation Coefficient +=============================== + +The Spatial Correlation Coefficient can be applied to compare the spatial structure of two images, which can be valuable in various domains such as medical imaging, remote sensing, and quality assessment in manufacturing or design processes. + +Let's consider a use case in medical imaging where Spatial Correlation Coefficient is used to compare the spatial correlation between a reference image and a reconstructed medical scan. +This can be particularly relevant in evaluating the accuracy of image reconstruction techniques or assessing the quality of medical imaging data. +""" + +# %% +# Here's a hypothetical Python example demonstrating the usage of the Spatial Correlation Coefficient to compare two medical images: + +import matplotlib.pyplot as plt +import numpy as np +import torch +from skimage.data import shepp_logan_phantom +from skimage.transform import iradon, radon, rescale +from torchmetrics.image import SpatialCorrelationCoefficient + +# %% +# Create a Shepp-Logan phantom image +phantom = shepp_logan_phantom() +phantom = rescale(phantom, scale=512 / 400) # Rescaling to 512x512 + +# %% +# Simulate projection data (sinogram) using Radon transform +theta = np.linspace(0.0, 180.0, max(phantom.shape), endpoint=False) +sinogram = radon(phantom, theta=theta) + +# %% +# Perform reconstruction using the inverse Radon transform +reconstruction = iradon(sinogram, theta=theta, circle=True) + +# %% +# Display the results +fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(10, 4)) +ax1.set_title("Original") +ax1.imshow(phantom, cmap=plt.cm.Greys_r) +ax2.set_title("Radon transform (Sinogram)") +ax2.imshow(sinogram, cmap=plt.cm.Greys_r, extent=(0, 180, 0, sinogram.shape[0]), aspect="equal") +ax3.set_title("Reconstruction from sinogram") +ax3.imshow(reconstruction, cmap=plt.cm.Greys_r) +fig.tight_layout() + +# %% +# Convert the images to PyTorch tensors +phantom_tensor = torch.from_numpy(phantom).float().unsqueeze(0).unsqueeze(0) +reconstructed_tensor = torch.from_numpy(reconstruction).float().unsqueeze(0).unsqueeze(0) + +# %% +# Calculating the Spatial Correlation Coefficient +scc = SpatialCorrelationCoefficient() +score = scc(preds=reconstructed_tensor, target=phantom_tensor) + +print(f"Spatial Correlation Coefficient between the images: {score}") +fig.suptitle(f"Spatial Correlation Coefficient: {score:.5}", y=-0.01) diff --git a/pyproject.toml b/pyproject.toml index efc34e27a8f..a53281d17ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,18 +55,24 @@ unfixable = ["F401"] [tool.ruff.lint.per-file-ignores] "setup.py" = ["ANN202", "ANN401"] "docs/source/conf.py" = ["A001", "D103"] +"examples/*" = [ + "E501", # Line too long (XXX > 120) + "D205", # 1 blank line required between summary line and description + "D212", # [*] Multi-line docstring summary should start at the first line + "D415", # First line should end with a period, question mark, or exclamation point +] "src/**" = [ "ANN401", - "S310", # todo: Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected. # todo - ] + "S310", # todo: Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected. +] "tests/**" = [ "ANN001", "ANN201", "ANN202", "ANN401", "S101", - "S301", # todo: `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue # todo - ] + "S301", # todo: `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue +] [tool.ruff.lint.pydocstyle] # Use Google-style docstrings. diff --git a/requirements/_docs.txt b/requirements/_docs.txt index ea8bbcd7978..2cab3cff11c 100644 --- a/requirements/_docs.txt +++ b/requirements/_docs.txt @@ -1,6 +1,5 @@ sphinx ==5.3.0 myst-parser ==1.0.0 -nbsphinx ==0.9.3 pandoc ==2.3 docutils ==0.19 sphinxcontrib-fulltoc >=1.0 @@ -10,6 +9,7 @@ sphinx-autodoc-typehints ==1.23.0 sphinx-paramlinks ==0.6.0 sphinx-togglebutton ==0.3.2 sphinx-copybutton ==0.5.2 +sphinx-gallery ==0.15.0 lightning >=1.8.0, <2.3.0 lightning-utilities ==0.11.2 @@ -23,4 +23,9 @@ pydantic > 1.0.0, < 3.0.0 -r image.txt -r multimodal.txt -r text.txt --r text_test.txt + +# Gallery extra requirements +# -------------------------- +# todo: until this has resolution - https://github.com/sphinx-gallery/sphinx-gallery/issues/1290 +# Image +scikit-image ~=0.21