Fix the issue of input data nodata changing by the odc loading (#163)

* fix the nodata issue when default to nan with float dtype * flexible to the case that numpy change convention * very explicit with mangroves dtype * simplify l3 cultivated mask --------- Co-authored-by: Emma Ai <[email protected]>
opendatacube · Oct 28, 2024 · d799a99 · d799a99
1 parent 980ded2
commit d799a99
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 11 deletions.
diff --git a/odc/stats/plugins/_base.py b/odc/stats/plugins/_base.py
@@ -2,6 +2,7 @@
 from typing import Mapping, Optional, Sequence, Tuple
 
 import xarray as xr
+import numpy as np
 from datacube.model import Dataset
 from datacube.utils.geometry import GeoBox
 from odc.algo import to_rgba
@@ -47,6 +48,12 @@ def measurements(self) -> Tuple[str, ...]:
         pass
 
     def native_transform(self, xx: xr.Dataset) -> xr.Dataset:
+        for var in xx.data_vars:
+            if (
+                xx[var].attrs.get("nodata") is None
+                and np.dtype(xx[var].dtype).kind == "f"
+            ):
+                xx[var].attrs["nodata"] = xx[var].dtype.type(None)
         return xx
 
     def fuser(self, xx: xr.Dataset) -> xr.Dataset:

diff --git a/odc/stats/plugins/lc_level3.py b/odc/stats/plugins/lc_level3.py
@@ -21,16 +21,22 @@ def measurements(self) -> Tuple[str, ...]:
         _measurements = ["level3_class"]
         return _measurements
 
+    def fuser(self, xx: xr.Dataset) -> xr.Dataset:
+        return xx
+
     def reduce(self, xx: xr.Dataset) -> xr.Dataset:
 
         # Cultivated pipeline applies a mask which feeds only terrestrial veg (110) to the model
-        # Just exclude no data (255) and apply the cultivated results
+        # Just exclude no data (255 or nan) and apply the cultivated results
+        # 255: load with product definition; nan: load without
+        # hence accormmodate both
+
         res = expr_eval(
-            "where(a<nodata, a, b)",
+            "where((a!=a)|(a>=nodata), b, a)",
             {"a": xx.cultivated_class.data, "b": xx.classes_l3_l4.data},
             name="mask_cultivated",
             dtype="float32",
-            **{"nodata": NODATA},
+            **{"nodata": xx.cultivated_class.attrs.get("nodata")},
         )
 
         # Mask urban results with bare sfc (210)
@@ -42,10 +48,22 @@ def reduce(self, xx: xr.Dataset) -> xr.Dataset:
                 "b": xx.urban_classes.data,
             },
             name="mark_urban",
-            dtype="uint8",
+            dtype="float32",
             **{"_u": 210},
         )
 
+        # Mark nodata to 255 in case any nan
+
+        res = expr_eval(
+            "where(a==a, a, nodata)",
+            {
+                "a": res,
+            },
+            name="mark_nodata",
+            dtype="uint8",
+            **{"nodata": NODATA},
+        )
+
         attrs = xx.attrs.copy()
         attrs["nodata"] = NODATA
         dims = xx.classes_l3_l4.dims[1:]

diff --git a/odc/stats/plugins/lc_veg_class_a1.py b/odc/stats/plugins/lc_veg_class_a1.py
@@ -94,9 +94,6 @@ def measurements(self) -> Tuple[str, ...]:
         _measurements = ["classes_l3_l4", "water_seasonality"]
         return _measurements
 
-    def native_transform(self, xx):
-        return xx
-
     def fuser(self, xx):
         return xx
 
@@ -175,12 +172,16 @@ def l3_class(self, xx: xr.Dataset):
                     )
                 elif b == "canopy_cover_class":
                     # aquatic_veg: (mangroves > 0) & (mangroves != nodata)
+                    # mangroves.nodata = 255 or nan
                     l3_mask = expr_eval(
-                        "where((a>0)&(a<nodata), m, b)",
+                        "where((a>0)&((a<nodata)|(nodata!=nodata)), m, b)",
                         {"a": xx[b].data, "b": l3_mask},
                         name="mark_mangroves",
                         dtype="uint8",
-                        **{"nodata": NODATA, "m": self.output_classes["aquatic_veg"]},
+                        **{
+                            "nodata": xx[b].attrs["nodata"],
+                            "m": self.output_classes["aquatic_veg"],
+                        },
                     )
 
         # all unmarked values (0) is terretrial veg
@@ -199,10 +200,9 @@ def l3_class(self, xx: xr.Dataset):
 
         # Mask nans with NODATA
         l3_mask = expr_eval(
-            "where((a!=a)|(b>=nodata), nodata, e)",
+            "where((a!=a), nodata, e)",
             {
                 "a": si5,
-                "b": xx.veg_frequency.data,
                 "e": l3_mask,
             },
             name="mark_nodata",