Merge pull request #118 from sentinel-hub/develop

Release version 1.3.2

.flake8

@@ -1,8 +1,9 @@
 [flake8]
-ignore = E203, W503
+ignore = E203, W503, C408, SIM117
 exclude = .git, __pycache__
 max-line-length= 120
 max-complexity = 10
+min_python_version = 3.8
 per-file-ignores =
     # imported but unused
     __init__.py: F401

.github/workflows/ci_action.yml

@@ -1,45 +1,39 @@
 name: build
 
 on:
+  pull_request:
   push:
+    branches:
+      - "main"
+      - "develop"
   schedule:
-    - cron: '0 0 * * *'
+    - cron: "0 0 * * *"
 
 env:
   # The only way to simulate if-else statement
-  CHECKOUT_BRANCH: ${{ github.event_name == 'push' && github.ref || 'develop' }}
+  CHECKOUT_BRANCH: ${{ github.event_name == 'schedule' && 'develop' || github.ref }}
 
 jobs:
-
-  check-code-black-isort-flake8:
+  check-pre-commit-hooks:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
         uses: actions/checkout@v2
         with:
           ref: ${{ env.CHECKOUT_BRANCH }}
-    
+
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
           architecture: x64
 
-      - name: Prepare linters
-        run: pip install black[jupyter] isort flake8 nbqa
-
-      - name: Check code compliance with black
-        run: black . --check --diff
-
-      - name: Check code compliance with isort
+      - name: Prepare pre-commit validators
         run: |
-          isort . --check --diff
-          nbqa isort . --nbqa-diff
+          pip install pre-commit
 
-      - name: Check code compliance with flake8
-        run: |
-          flake8 .
-          nbqa flake8 .
+      - name: Check code compliance with pre-commit validators
+        run: pre-commit run --all-files
 
   check-code-mypy:
     runs-on: ubuntu-latest
@@ -54,11 +48,11 @@ jobs:
         with:
           python-version: "3.8"
           architecture: x64
-      
+
       - name: Install packages
         run: |
           pip install -e .[DEV]
-      
+
       - name: Run mypy
         run: |
           mypy eogrow
@@ -68,9 +62,13 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - '3.8'
-          - '3.9'
-          - '3.10'
+          - "3.8"
+          - "3.10"
+        include:
+          # A flag marks whether full or partial tests should be run
+          # We don't run integration tests on pull requests from outside repos, because they don't have secrets
+          - python-version: "3.9"
+            full_test_suite: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
     steps:
       - name: Checkout branch
         uses: actions/checkout@v2
@@ -87,19 +85,19 @@ jobs:
         run: |
           sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y
           sudo apt-get update
-          sudo apt-get install -y build-essential libgdal-dev graphviz proj-bin gcc libproj-dev libspatialindex-dev
+          sudo apt-get install -y build-essential gdal-bin libgdal-dev graphviz proj-bin gcc libproj-dev libspatialindex-dev
           export CPLUS_INCLUDE_PATH=/usr/include/gdal
           export C_INCLUDE_PATH=/usr/include/gdal
           pip install -e .[DEV]
           pip install gdal==$(gdal-config --version | awk -F'[.]' '{print $1"."$2}')
 
       - name: Run fast tests
-        if: matrix.python-version != '3.9'
+        if: ${{ !matrix.full_test_suite }}
         run: |
           pytest -m "fast"
 
       - name: Run fast and chain integration tests
-        if: matrix.python-version == '3.9'
+        if: ${{ matrix.full_test_suite }}
         run: |
           sentinelhub.config \
             --sh_client_id "${{ secrets.SH_CLIENT_ID }}" \

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+      - id: debug-statements
+      # - id: check-json  # does not agree with comments in JSONs :(
+      - id: check-toml
+      - id: check-yaml
+      - id: check-merge-conflict
+      - id: debug-statements
+
+  - repo: https://github.com/psf/black
+    rev: 22.8.0
+    hooks:
+      - id: black
+        language_version: python3
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort (python)
+
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v1.5.3
+    hooks:
+      - id: autoflake
+        args:
+          [
+            --remove-all-unused-imports,
+            --in-place,
+            --ignore-init-module-imports,
+          ]
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-simplify
+          - flake8-typing-imports
+
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.4.0
+    hooks:
+      - id: nbqa-black
+      - id: nbqa-isort
+      - id: nbqa-flake8

docs/source/_templates/module.rst_t

@@ -6,4 +6,3 @@
 {%- for option in automodule_options %}
    :{{ option }}:
 {%- endfor %}
-

docs/source/conf.py

@@ -15,6 +15,7 @@
 import os
 import shutil
 import sys
+from typing import Any, Dict, Optional
 
 # -- Project information -----------------------------------------------------
 
@@ -29,10 +30,11 @@
 # built documents.
 #
 # The release is read from __init__ file and version is shortened release string.
-for line in open(os.path.join(os.path.dirname(__file__), "../../eogrow/__init__.py")):
-    if line.find("__version__") >= 0:
-        release = line.split("=")[1].strip()
-        release = release.strip('"').strip("'")
+with open(os.path.join(os.path.dirname(__file__), "../../eogrow/__init__.py")) as init_file:
+    for line in init_file:
+        if line.find("__version__") >= 0:
+            release = line.split("=")[1].strip()
+            release = release.strip('"').strip("'")
 version = release.rsplit(".", 1)[0]
 
 # -- General configuration ------------------------------------------------
@@ -52,10 +54,11 @@
     "sphinx.ext.viewcode",
     "sphinx.ext.todo",
     "sphinx.ext.coverage",
-    "nbsphinx",
     "sphinx.ext.mathjax",
     "sphinx.ext.viewcode",
     "sphinx.ext.githubpages",
+    "nbsphinx",
+    "sphinx_rtd_theme",
     "m2r2",
     "sphinxcontrib.autodoc_pydantic",
 ]
@@ -70,7 +73,7 @@
 autodoc_pydantic_settings_show_config_summary = False
 autodoc_pydantic_settings_show_validator_summary = False
 
-# Incude typehints in descriptions
+# Include typehints in descriptions
 autodoc_typehints = "description"
 
 # Both the class’ and the __init__ method’s docstring are concatenated and inserted.
@@ -266,6 +269,7 @@ def process_readme():
 
 # Auto-generate documentation pages
 current_dir = os.path.abspath(os.path.dirname(__file__))
+repository_dir = os.path.join(current_dir, "..", "..")
 reference_dir = os.path.join(current_dir, "reference")
 module = os.path.join(current_dir, "..", "..", "eogrow")
 
@@ -283,5 +287,71 @@ def run_apidoc(_):
     main(["-e", "-o", reference_dir, module, *APIDOC_EXCLUDE, *APIDOC_OPTIONS])
 
 
+def configure_github_link(_app: Any, pagename: str, _templatename: Any, context: Dict[str, Any], _doctree: Any) -> None:
+    """Because some pages are auto-generated and some are copied from their original location the link "Edit on GitHub"
+    of a page is wrong. This function computes a custom link for such pages and saves it to a custom meta parameter
+    `github_url` which is then picked up by `sphinx_rtd_theme`.
+
+    Resources to understand the implementation:
+    - https://www.sphinx-doc.org/en/master/extdev/appapi.html#event-html-page-context
+    - https://dev.readthedocs.io/en/latest/design/theme-context.html
+    - https://sphinx-rtd-theme.readthedocs.io/en/latest/configuring.html?highlight=github_url#file-wide-metadata
+    - https://github.com/readthedocs/sphinx_rtd_theme/blob/1.0.0/sphinx_rtd_theme/breadcrumbs.html#L35
+    """
+    # ReadTheDocs automatically sets the following parameters but for local testing we set them manually:
+    show_link = context.get("display_github")
+    context["display_github"] = True if show_link is None else show_link
+    context["github_user"] = context.get("github_user") or "sentinel-hub"
+    context["github_repo"] = context.get("github_repo") or "eo-grow"
+    context["github_version"] = context.get("github_version") or "develop"
+    context["conf_py_path"] = context.get("conf_py_path") or "/docs/source/"
+
+    if not pagename.startswith("reference/"):
+        return
+
+    filename = pagename.split("/", 1)[1]
+    filename = filename.replace(".", "/")
+    full_path = os.path.join(repository_dir, f"{filename}.py")
+    is_module = os.path.exists(full_path)
+
+    github_url = create_github_url(
+        context,
+        theme_vcs_pageview_mode="blob" if is_module else "tree",
+        conf_py_path="/",
+        pagename=filename,
+        page_source_suffix=".py" if is_module else "",
+    )
+
+    context["meta"] = context.get("meta") or {}
+    context["meta"]["github_url"] = github_url
+
+
+def create_github_url(
+    context: Dict[str, Any],
+    theme_vcs_pageview_mode: Optional[str] = None,
+    conf_py_path: Optional[str] = None,
+    pagename: Optional[str] = None,
+    page_source_suffix: Optional[str] = None,
+) -> str:
+    """Creates a GitHub URL from context in exactly the same way as in
+    https://github.com/readthedocs/sphinx_rtd_theme/blob/1.0.0/sphinx_rtd_theme/breadcrumbs.html#L39
+
+    The function allows URL customization by overwriting certain parameters.
+    """
+    github_host = context.get("github_host") or "github.com"
+    github_user = context.get("github_user", "")
+    github_repo = context.get("github_repo", "")
+    theme_vcs_pageview_mode = theme_vcs_pageview_mode or context.get("theme_vcs_pageview_mode") or "blob"
+    github_version = context.get("github_version", "")
+    conf_py_path = conf_py_path or context.get("conf_py_path", "")
+    pagename = pagename or context.get("pagename", "")
+    page_source_suffix = context.get("page_source_suffix", "") if page_source_suffix is None else page_source_suffix
+    return (
+        f"https://{github_host}/{github_user}/{github_repo}/{theme_vcs_pageview_mode}/"
+        f"{github_version}{conf_py_path}{pagename}{page_source_suffix}"
+    )
+
+
 def setup(app):
     app.connect("builder-inited", run_apidoc)
+    app.connect("html-page-context", configure_github_link)

eogrow/__init__.py

@@ -2,4 +2,4 @@
 The main init module
 """
 
-__version__ = "1.3.1"
+__version__ = "1.3.2"

eogrow/core/area/base.py

@@ -301,4 +301,4 @@ def _add_bbox_column(grid: List[gpd.GeoDataFrame]) -> None:
         """Adds a column with bounding boxes to all dataframes in a grid"""
         for bbox_df in grid:
             crs = CRS(bbox_df.crs)
-            bbox_df["BBOX"] = bbox_df.geometry.apply(lambda geo: BBox(geo.bounds, crs))
+            bbox_df["BBOX"] = bbox_df.geometry.apply(lambda geo: BBox(geo.bounds, crs))  # noqa B023

eogrow/core/area/batch.py

@@ -1,6 +1,7 @@
 """
 Area managers for Sentinel Hub batch grids
 """
+import contextlib
 import warnings
 from typing import Any, List, Tuple, cast
 
@@ -182,10 +183,8 @@ def _get_target_grid(
         """First, it tries to obtain a target grid from target area manager. That would work if the grid is already
         cached or if target area manager is able to generate it. If not, then it creates the target grid from the
         given source grid."""
-        try:
+        with contextlib.suppress(MissingBatchIdError):
             return target_area_manager.get_grid(add_bbox_column=True)
-        except MissingBatchIdError:
-            pass
 
         basic_grid = self._join_batch_grid(source_grid, self.subsplit, self.absolute_buffer)
         target_grid = self._split_batch_grid(

eogrow/core/logging.py

@@ -1,6 +1,7 @@
 """
 Module with utilities for logging
 """
+import contextlib
 import json
 import logging
 import sys
@@ -263,10 +264,8 @@ def __init__(self, path: str, filesystem: Union[FS, bytes], encoding: Optional[s
     def close(self) -> None:
         """Closes logging and closes the local file"""
         super().close()
-        try:
+        with contextlib.suppress(FilesystemClosed):
             self.local_file.close()
-        except FilesystemClosed:
-            pass
 
 
 class RegularBackupHandler(FilesystemHandler):
@@ -389,6 +388,8 @@ class EOExecutionFilter(Filter):
         "rasterio",
         "numba",
         "fiona.ogrext",
+        "fiona.env",
+        "fiona._env",
     )
 
     def __init__(self, ignore_packages: Optional[Sequence[str]] = None, *args: Any, **kwargs: Any):

eogrow/core/storage.py

@@ -12,7 +12,6 @@
 from sentinelhub import SHConfig
 
 from ..utils.types import AwsAclType
-from ..utils.validators import dict_factory
 from .base import EOGrowObject
 from .schemas import ManagerSchema
 
@@ -41,7 +40,7 @@ class Schema(ManagerSchema, BaseSettings):
             )
         )
         structure: Dict[str, str] = Field(
-            default_factory=dict_factory,
+            default_factory=dict,
             description="A flat key: value store mapping each key to a path in the project.",
         )