Merge pull request #1684 from opendatacube/cherrypicks-and-new-rc-rel…

…ease

Cherrypicks and new rc release

.github/workflows/main.yml

@@ -175,7 +175,7 @@ jobs:
         TWINE_PASSWORD: ${{ secrets.PyPiToken }}
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         file: ./coverage.xml
         fail_ci_if_error: false

datacube/api/core.py

@@ -683,7 +683,7 @@ def mk_group(group: Iterable[Dataset]) -> tuple[Any, Iterable[Dataset]]:
         coords = numpy.asarray([coord for coord, _ in groups])
         data = numpy.empty(len(coords), dtype=object)
         for i, (_, dss) in enumerate(groups):
-            data[i] = dss
+            data[i] = dss  # type: ignore[assignment, call-overload]
 
         sources = xarray.DataArray(data,
                                    dims=[group_by.dimension],

datacube/model/schema/dataset-type-schema.yaml

@@ -99,7 +99,7 @@ definitions:
             nodata:
               oneOf:
                 - type: number
-                - enum: [NaN, Inf, -Inf]
+                - enum: [nan, NAN, NaN, Inf, -Inf]
             scale_factor:
               description: 'Defines mapping to some "real" space like so: real_value = pixel_value*scale_factor + add_offset'
               type: number

datacube/storage/_load.py

@@ -9,6 +9,7 @@
 
 """
 import logging
+import numbers
 from collections import OrderedDict
 import numpy as np
 from xarray.core.dataarray import DataArray as XrDataArray, DataArrayCoordinates
@@ -94,7 +95,12 @@ def copyto_fuser(dest: np.ndarray, src: np.ndarray) -> None:
 
                 if not roi_is_empty(roi):
                     fuse_func(destination[roi], buffer_[roi])
-                    buffer_[roi] = dst_nodata  # clean up for next read
+                    if dst_nodata is not None:
+                        buffer_[roi] = dst_nodata  # clean up for next read
+                    elif issubclass(destination.dtype.type, numbers.Real):
+                        buffer_[roi] = float("nan")
+                    else:
+                        buffer_[roi] = 0
 
             if progress_cbk:
                 progress_cbk(n_so_far, len(datasources))

datacube/utils/geometry/tools.py

@@ -60,8 +60,8 @@ def gbox_boundary(gbox, pts_per_side=16):
 
     """
     H, W = gbox.shape[:2]
-    xx = np.linspace(0, W, pts_per_side, dtype='float32')
-    yy = np.linspace(0, H, pts_per_side, dtype='float32')
+    xx = np.linspace(0, W, pts_per_side, dtype='float64')
+    yy = np.linspace(0, H, pts_per_side, dtype='float64')
 
     return polygon_path(xx, yy).T[:-1]
 
@@ -72,11 +72,11 @@ def roi_boundary(roi, pts_per_side=2):
 
     roi needs to be in the normalised form, i.e. no open-ended start/stop, see roi_normalise
 
-    :returns: Nx2 float32 array of X,Y points on the perimeter of the envelope defined by `roi`
+    :returns: Nx2 float64 array of X,Y points on the perimeter of the envelope defined by `roi`
     """
     yy, xx = roi
-    xx = np.linspace(xx.start, xx.stop, pts_per_side, dtype='float32')
-    yy = np.linspace(yy.start, yy.stop, pts_per_side, dtype='float32')
+    xx = np.linspace(xx.start, xx.stop, pts_per_side, dtype='float64')
+    yy = np.linspace(yy.start, yy.stop, pts_per_side, dtype='float64')
 
     return polygon_path(xx, yy).T[:-1]
 

docker/Dockerfile

@@ -5,8 +5,8 @@
 ## SPDX-License-Identifier: Apache-2.0
 ##
 # gdal:ubuntu-small no longer comes with netcdf support compiled into gdal
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.3 AS builder
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.9.3
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.10.0 AS builder
+FROM ghcr.io/osgeo/gdal:ubuntu-full-3.10.0
 ARG V_PG=16
 ARG V_PGIS=16-postgis-3
 

docker/constraints.in

@@ -48,8 +48,8 @@ pyproj>=3.5
 python-dateutil
 # For Python 3.12 support
 pyyaml>=6.0.1
-# For Python 3.12 support
-rasterio>=1.3.11
+# For latest gdal support
+rasterio>=1.4.0
 # For Python 3.12 support
 typing-extensions>=4.7
 recommonmark

docker/constraints.txt

@@ -234,7 +234,6 @@ numpy==2.1.3
     #   pandas
     #   rasterio
     #   shapely
-    #   snuggs
     #   xarray
 odc-geo==0.4.8
     # via -r constraints.in
@@ -288,7 +287,7 @@ pylint==3.2.2
 pyparsing==3.0.9
     # via
     #   matplotlib
-    #   snuggs
+    #   rasterio
 pyproj==3.6.1
     # via
     #   -r constraints.in
@@ -326,7 +325,7 @@ pyyaml==6.0.1
     #   dask
     #   distributed
     #   owslib
-rasterio==1.3.11
+rasterio==1.4.3
     # via -r constraints.in
 readme-renderer==37.3
     # via twine
@@ -383,8 +382,6 @@ six==1.16.0
     #   python-dateutil
 snowballstemmer==2.2.0
     # via sphinx
-snuggs==1.4.7
-    # via rasterio
 sortedcontainers==2.4.0
     # via
     #   distributed

docs/README.rst

@@ -50,6 +50,12 @@ Requires a Unix like system that includes ``make``.
 
    pip install -r requirements.txt
 
+#. Install sphinx-autobuild with pip
+
+.. code-block:: bash
+
+   pip install sphinx-autobuild
+
 #. Run the autobuild.
 
 .. code-block:: bash

docs/about/whats_new.rst

@@ -8,22 +8,26 @@ What's New
 v1.9.next
 =========
 
-- Remove multiple location support from postgis driver. (:pull:`1658`)
-- Remove unnecessary logic copied from eodatasets3 (:pull:`1658`)
-* Documentation fixes (:pull:`1659`)
-* Don't use importlib_metadata (:pull:`1657`)
-* Pin upstream libraries to get CI tests running with numpy2 (:pull:`1661`)
-* Calculate spatial extent in epsg:4326 and reproject when no dedicated spatial extent for
+v1.9.0-rc12 (9th December 2024)
+===============================
+
+- Remove multiple location support from postgis driver and remove unnecessary logic copied from eodatasets3
+  (:pull:`1658`)
+- Documentation fixes (:pull:`1659`)
+- Don't use importlib_metadata (:pull:`1657`)
+- Pin upstream libraries to get CI tests running with numpy2 (:pull:`1661`)
+- Calculate spatial extent in epsg:4326 and reproject when no dedicated spatial extent for
   the requested CRS is available (:pull:`1662`)
-* Fix broken alembic migration (:pull:`1667`)
-* Fix broken documentation build (:pull:`1668`)
-* Rename `DatasetType` to `Product` in all the tests (:pull:`1671`)
-* Documentation updates for 1.9 release (:pull:`1664`, :pull:`1699`)
-* Fix metadata issues with new Lineage API. (:pull:`1677`)
-* Fix metadata issues with new Lineage API. (:pull:`1679`)
-* Suppress annoying config warning when configuring by environment variable (:pull:`1680`)
-* Suppress internal deprecation warnings when running tests against deprecated code (:pull:`1681`)
-* Split index.abstract module into a package (:pull:`1682`)
+- Fix broken alembic migration (:pull:`1667`)
+- Fix broken documentation build (:pull:`1668`)
+- Rename `DatasetType` to `Product` in all the tests (:pull:`1671`)
+- Fix a bunch of typos and spelling mistakes (:pull:`1672`)
+- Documentation updates for 1.9 release (:pull:`1664`, :pull:`1669`)
+- Fix metadata issues with new Lineage API. (:pull:`1679`)
+- Suppress annoying config warning when configuring by environment variable (:pull:`1680`)
+- Suppress internal deprecation warnings when running tests against deprecated code (:pull:`1681`)
+- Split index.abstract module into a package (:pull:`1682`)
+- Cherry picks from 1.8 and prepare for 1.9.0-rc12 release (:pull:`1684`)
 
 v1.9.0-rc11 (28th October 2024)
 ===============================
@@ -165,12 +169,22 @@ v1.9.0-rc1 (27th March 2024)
 
 v1.8.next
 =========
+
+v1.8.20 (9th December 2024)
+===========================
+- Update docker image and CI (:pull:`1624`, :pull:`1625`, :pull:`1626`, :pull:`1639`, :pull:`1641`, :pull:`1642`)
+- Update Slack links to Discord (:pull:`1620`)
+- Automatic updates of upstream library/image versions (:pull:`1629`, :pull:`1644`, :pull:`1653`, :pull:`1660`)
 - Don't error when adding a dataset whose product doesn't have an id value (:pull:`1630`)
+- Update docs (:pull:`1631`,:pull:`1651`)
+- Pin out bad versions of dask (:pull:`1663`)
+- Ensure correct boundary scalar precision with numpy 2 (:pull:`1673`)
+- Add more string representations of nan (:pull:`1678`)
 
 v1.8.19 (2nd July 2024)
 =======================
 
-- Update whats_new for 1.8.19 release (:pull:`1612`)
+- Update whats_new.rst for 1.8.19 release (:pull:`1612`)
 - Always write floating point bands to cogs with nodata=nan for ESRI and GDAL compatibility (:pull:`1602`)
 - Add deprecation warning for config environment names that will not be supported in 1.9 (:pull:`1592`)
 - Update docker image to GDAL 3.9/Python 3.12/Ubuntu 24.04 (:pull:`1587`)
@@ -195,7 +209,6 @@ v1.8.18 (27th March 2024)
 - Add generic NOT operator and for ODC queries and ``Not`` type wrapper (:pull:`1563`)
 - Update whats_new.rst for release (:pull:`1568`)
 
-
 v1.8.17 (8th November 2023)
 ===========================
 - Fix schema creation with postgres driver when initialising system with ``--no-init-users`` (:pull:`1504`)