From b8f77e578d6fed2b48ab259fd9d652f167a70dbf Mon Sep 17 00:00:00 2001 From: Toktam Ebadi Date: Fri, 22 Nov 2024 04:00:05 +0000 Subject: [PATCH] Cleaned the code and removed redundant checks at level4 --- .../plugins/l34_utils/l4_bare_gradation.py | 18 +- odc/stats/plugins/l34_utils/l4_cultivated.py | 81 +++++---- odc/stats/plugins/l34_utils/l4_natural_veg.py | 98 ++++++----- odc/stats/plugins/l34_utils/l4_surface.py | 19 ++- odc/stats/plugins/l34_utils/l4_veg_cover.py | 11 -- odc/stats/plugins/l34_utils/l4_water.py | 11 +- .../plugins/l34_utils/lc_intertidal_mask.py | 18 -- odc/stats/plugins/l34_utils/lc_level3.py | 8 + odc/stats/plugins/l34_utils/lc_lifeform.py | 33 ---- odc/stats/plugins/lc_level34.py | 13 +- tests/test_lc_l4_ctv.py | 19 +-- tests/test_lc_l4_natural_surface.py | 6 +- tests/test_lc_l4_nav.py | 16 +- tests/test_lc_l4_ntv.py | 16 +- tests/test_lc_l4_water.py | 7 +- tests/test_lc_water_seasonality.py | 159 ++++++++++++++++++ 16 files changed, 320 insertions(+), 213 deletions(-) delete mode 100644 odc/stats/plugins/l34_utils/lc_intertidal_mask.py delete mode 100644 odc/stats/plugins/l34_utils/lc_lifeform.py create mode 100644 tests/test_lc_water_seasonality.py diff --git a/odc/stats/plugins/l34_utils/l4_bare_gradation.py b/odc/stats/plugins/l34_utils/l4_bare_gradation.py index da675d46..45e08804 100644 --- a/odc/stats/plugins/l34_utils/l4_bare_gradation.py +++ b/odc/stats/plugins/l34_utils/l4_bare_gradation.py @@ -7,28 +7,20 @@ def bare_gradation(xx: xr.Dataset, bare_threshold, veg_cover): + # Address nodata bs_pc_50 = expr_eval( "where((a!=a), nodata, a)", {"a": xx.bs_pc_50.data}, - name="mark_veg", + name="mark_bare_gradation_nodata", dtype="float32", **{"nodata": NODATA}, ) - # Map any data > 100 ---> 100 - bs_pc_50 = expr_eval( - "where((a>100)&(a!=nodata), 100, a)", - {"a": bs_pc_50}, - name="mark_veg", - dtype="uint8", - **{"nodata": NODATA}, - ) - # 60% <= data --> 15 bs_mask = expr_eval( "where((a>=m)&(a!=nodata), 15, a)", {"a": bs_pc_50}, - name="mark_veg", + name="mark_bare", dtype="uint8", **{"m": bare_threshold[1], "nodata": NODATA}, ) @@ -37,7 +29,7 @@ def bare_gradation(xx: xr.Dataset, bare_threshold, veg_cover): bs_mask = expr_eval( "where((a>=m)&(a 100 ---> 100 - pv_pc_50 = expr_eval( - "where((a>100) & (a!=nodata), 100, a)", - { - "a": pv_pc_50, - }, - name="mark_veg", - dtype="uint8", - **{"nodata": NODATA}, - ) - # data < 1 ---> 0 veg_mask = expr_eval( "where(a 1 woody - # 114 ----> 2 herbaceous - - lifeform_mask = expr_eval( - "where((a!=a)|(a>=nodata), nodata, a)", - {"a": xx.woody.data}, - name="mark_lifeform", - dtype="float32", - **{"nodata": NODATA}, - ) - - lifeform_mask = expr_eval( - "where(a==113, 1, a)", - {"a": lifeform_mask}, - name="mark_lifeform", - dtype="uint8", - ) - lifeform_mask = expr_eval( - "where(a==114, 2, a)", - {"a": lifeform_mask}, - name="mark_lifeform", - dtype="uint8", - ) - - return lifeform_mask diff --git a/odc/stats/plugins/lc_level34.py b/odc/stats/plugins/lc_level34.py index 69a51b5d..b6b30ea4 100644 --- a/odc/stats/plugins/lc_level34.py +++ b/odc/stats/plugins/lc_level34.py @@ -19,8 +19,6 @@ l4_surface, l4_bare_gradation, l4_water, - lc_lifeform, - lc_intertidal_mask, ) @@ -60,10 +58,8 @@ def reduce(self, xx: xr.Dataset) -> xr.Dataset: xx, self.watper_threshold ) - intertidal_mask = lc_intertidal_mask.intertidal_mask(xx) - # #TODO WATER (99-104) - l4 = l4_water.water_classification(xx, intertidal_mask, water_persistence) + l4 = l4_water.water_classification(xx, water_persistence) # Generate Level3 classes level3 = lc_level3.lc_level3(xx) @@ -71,14 +67,11 @@ def reduce(self, xx: xr.Dataset) -> xr.Dataset: # Vegetation cover veg_cover = l4_veg_cover.canopyco_veg_con(xx, self.veg_threshold) - # Define life form - lifeform = lc_lifeform.lifeform(xx) - # Apply cultivated Level-4 classes (1-18) - l4 = l4_cultivated.lc_l4_cultivated(l4, level3, lifeform, veg_cover) + l4 = l4_cultivated.lc_l4_cultivated(l4, level3, xx.woody, veg_cover) # Apply terrestrial vegetation classes [19-36] - l4 = l4_natural_veg.lc_l4_natural_veg(l4, level3, lifeform, veg_cover) + l4 = l4_natural_veg.lc_l4_natural_veg(l4, level3, xx.woody, veg_cover) # Bare gradation bare_gradation = l4_bare_gradation.bare_gradation( diff --git a/tests/test_lc_l4_ctv.py b/tests/test_lc_l4_ctv.py index 8592c9dc..6f0035ab 100644 --- a/tests/test_lc_l4_ctv.py +++ b/tests/test_lc_l4_ctv.py @@ -7,7 +7,6 @@ l4_cultivated, lc_level3, l4_veg_cover, - lc_lifeform, ) import pandas as pd @@ -131,10 +130,8 @@ def test_ctv_classes_woody(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ctv.compute() == expected_cultivated_classes).all() @@ -211,10 +208,9 @@ def test_ctv_classes_herbaceous(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ctv.compute() == expected_cultivated_classes).all() @@ -290,10 +286,9 @@ def test_ctv_classes_woody_herbaceous(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) - veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) + veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ctv.compute() == expected_cultivated_classes).all() @@ -370,8 +365,8 @@ def test_ctv_classes_no_vegcover(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) + veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) + assert (l4_ctv.compute() == expected_cultivated_classes).all() diff --git a/tests/test_lc_l4_natural_surface.py b/tests/test_lc_l4_natural_surface.py index 81306a18..819155cf 100644 --- a/tests/test_lc_l4_natural_surface.py +++ b/tests/test_lc_l4_natural_surface.py @@ -15,7 +15,6 @@ l4_natural_aquatic, l4_surface, l4_bare_gradation, - lc_lifeform, ) import pandas as pd @@ -191,14 +190,13 @@ def test_ns(): ) level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_threshold = [1, 4, 15, 40, 65, 100] veg_cover = l4_veg_cover.canopyco_veg_con(xx, veg_threshold) # Apply cultivated to match the code in Level4 processing - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) - l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) + l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, xx.woody, veg_cover) l4_ctv_ntv_nav = l4_natural_aquatic.natural_auquatic_veg( l4_ctv_ntv, veg_cover, xx.water_season diff --git a/tests/test_lc_l4_nav.py b/tests/test_lc_l4_nav.py index 87022208..f91ac6fb 100644 --- a/tests/test_lc_l4_nav.py +++ b/tests/test_lc_l4_nav.py @@ -13,7 +13,6 @@ l4_veg_cover, l4_natural_veg, l4_natural_aquatic, - lc_lifeform, ) import pandas as pd @@ -169,12 +168,11 @@ def test_ntv_classes_woody_herbaceous(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) # Apply cultivated to match the code in Level4 processing - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) - l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) + l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, xx.woody, veg_cover) l4_ctv_ntv_nav = l4_natural_aquatic.natural_auquatic_veg( l4_ctv_ntv, veg_cover, xx.water_season @@ -280,12 +278,11 @@ def test_ntv_herbaceous_seasonal_water_veg_cover(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) # Apply cultivated to match the code in Level4 processing - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) - l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) + l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, xx.woody, veg_cover) l4_ctv_ntv_nav = l4_natural_aquatic.natural_auquatic_veg( l4_ctv_ntv, veg_cover, xx.water_season @@ -391,12 +388,11 @@ def test_ntv_woody_seasonal_water_veg_cover(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) # Apply cultivated to match the code in Level4 processing - l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, lifeform, veg_cover) - l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, lifeform, veg_cover) + l4_ctv = l4_cultivated.lc_l4_cultivated(xx.level_3_4, level3, xx.woody, veg_cover) + l4_ctv_ntv = l4_natural_veg.lc_l4_natural_veg(l4_ctv, level3, xx.woody, veg_cover) l4_ctv_ntv_nav = l4_natural_aquatic.natural_auquatic_veg( l4_ctv_ntv, veg_cover, xx.water_season diff --git a/tests/test_lc_l4_ntv.py b/tests/test_lc_l4_ntv.py index 9cea5d55..0d89005e 100644 --- a/tests/test_lc_l4_ntv.py +++ b/tests/test_lc_l4_ntv.py @@ -11,7 +11,6 @@ lc_level3, l4_veg_cover, l4_natural_veg, - lc_lifeform, ) import pandas as pd @@ -134,9 +133,9 @@ def test_ntv_classes_herbaceous(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) + veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, lifeform, veg_cover) + l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ntv.compute() == expected_natural_terrestrial_veg_classes).all() @@ -212,10 +211,9 @@ def test_ntv_classes_woody(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, lifeform, veg_cover) + l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ntv.compute() == expected_natural_terrestrial_veg_classes).all() @@ -291,9 +289,9 @@ def test_ntv_classes_no_veg(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) + veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, lifeform, veg_cover) + l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ntv.compute() == expected_natural_terrestrial_veg_classes).all() @@ -369,7 +367,7 @@ def test_ntv_classes_no_lifeform(): stats_l4 = StatsLccsLevel4() level3 = lc_level3.lc_level3(xx) - lifeform = lc_lifeform.lifeform(xx) + veg_cover = l4_veg_cover.canopyco_veg_con(xx, stats_l4.veg_threshold) - l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, lifeform, veg_cover) + l4_ntv = l4_natural_veg.lc_l4_natural_veg(xx.level_3_4, level3, xx.woody, veg_cover) assert (l4_ntv.compute() == expected_natural_terrestrial_veg_classes).all() diff --git a/tests/test_lc_l4_water.py b/tests/test_lc_l4_water.py index 3a70db88..b612ee50 100644 --- a/tests/test_lc_l4_water.py +++ b/tests/test_lc_l4_water.py @@ -10,7 +10,6 @@ from odc.stats.plugins.l34_utils import ( l4_water_persistence, l4_water, - lc_intertidal_mask, ) import pandas as pd @@ -163,7 +162,6 @@ def test_water_classes(): ) stats_l4 = StatsLccsLevel4() - intertidal_mask = lc_intertidal_mask.intertidal_mask(xx) # Water persistence water_persistence = l4_water_persistence.water_persistence( @@ -171,7 +169,7 @@ def test_water_classes(): ) l4_water_classes = l4_water.water_classification( - xx, intertidal_mask, water_persistence + xx, water_persistence ) assert (l4_water_classes.compute() == expected_water_classes).all() @@ -272,7 +270,6 @@ def test_water_intertidal(): ) stats_l4 = StatsLccsLevel4() - intertidal_mask = lc_intertidal_mask.intertidal_mask(xx) # Water persistence water_persistence = l4_water_persistence.water_persistence( @@ -280,7 +277,7 @@ def test_water_intertidal(): ) l4_water_classes = l4_water.water_classification( - xx, intertidal_mask, water_persistence + xx, water_persistence ) assert (l4_water_classes.compute() == expected_water_classes).all() diff --git a/tests/test_lc_water_seasonality.py b/tests/test_lc_water_seasonality.py new file mode 100644 index 00000000..397fe761 --- /dev/null +++ b/tests/test_lc_water_seasonality.py @@ -0,0 +1,159 @@ +from odc.stats.plugins.lc_level34 import StatsLccsLevel4 +import numpy as np +import pandas as pd +import xarray as xr +import dask.array as da + +import pytest + + +NODATA = 255 + + +@pytest.fixture(scope="module") +def image_groups(): + l34 = np.array( + [ + [ + [210, 210, 210], + [210, 210, 210], + [210, 210, 210], + [210, 210, 210], + ] + ], + dtype="uint8", + ) + + urban = np.array( + [ + [ + [216, 216, 215], + [216, 216, 216], + [215, 215, 215], + [215, 215, 215], + ] + ], + dtype="uint8", + ) + + woody = np.array( + [ + [ + [113, 113, 113], + [113, 113, 255], + [114, 114, 114], + [114, 114, 255], + ] + ], + dtype="uint8", + ) + + pv_pc_50 = np.array( + [ + [ + [1, 64, 65], + [66, 40, 41], + [3, 61, 78], + [4, 23, 42], + ] + ], + dtype="uint8", + ) + + bs_pc_50 = np.array( + [ + [ + [1, 64, NODATA], + [66, 40, 41], + [1, 40, 66], + [NODATA, 1, 42], + ] + ], + dtype="uint8", + ) + + cultivated = np.array( + [ + [ + [255, 255, 255], + [255, 255, 255], + [255, 255, 255], + [255, 255, 255], + ] + ], + dtype="uint8", + ) + + water_frequency = np.array( + [ + [ + [1, 3, 2], + [4, 5, 6], + [9, 2, 11], + [10, 11, 12], + ] + ], + dtype="uint8", + ) + + tuples = [ + (np.datetime64("2000-01-01T00"), np.datetime64("2000-01-01")), + ] + index = pd.MultiIndex.from_tuples(tuples, names=["time", "solar_day"]) + coords = { + "x": np.linspace(10, 20, l34.shape[2]), + "y": np.linspace(0, 5, l34.shape[1]), + "spec": index, + } + + data_vars = { + "classes_l3_l4": xr.DataArray( + da.from_array(l34, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + "urban_classes": xr.DataArray( + da.from_array(urban, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + "cultivated_class": xr.DataArray( + da.from_array(cultivated, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + "woody_cover": xr.DataArray( + da.from_array(woody, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + "pv_pc_50": xr.DataArray( + da.from_array(pv_pc_50, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + "bs_pc_50": xr.DataArray( + da.from_array(bs_pc_50, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + "water_frequency": xr.DataArray( + da.from_array(water_frequency, chunks=(1, -1, -1)), + dims=("spec", "y", "x"), + attrs={"nodata": 255}, + ), + } + + xx = xr.Dataset(data_vars=data_vars, coords=coords) + return xx + + +def test_l4_classes(image_groups): + expected_l3 = [[216, 216, 215], [216, 216, 216], [215, 215, 215], [215, 215, 215]] + + expected_l4 = [[95, 97, 93], [97, 96, 96], [93, 93, 93], [93, 93, 93]] + stats_l4 = StatsLccsLevel4() + ds = stats_l4.reduce(image_groups) + + assert (ds.level3.compute() == expected_l3).all() + assert (ds.level4.compute() == expected_l4).all()