Merge pull request #116 from opendatacube/burn-area-provisional-produ…

…ct-updates

Burn area provisional product updates

Makefile

@@ -95,6 +95,20 @@ index-s2be:
 			s3-to-dc --no-sign-request 's3://dea-public-data-dev/s2be/*odc-metadata.yaml' s2_barest_earth\
 		"
 
+# Add s2 c3 datasets
+product-s2-c3:
+	docker-compose exec alchemist \
+		datacube product add \
+		https://explorer.dev.dea.ga.gov.au/products/ga_s2am_ard_provisional_3.odc-product.yaml
+	docker-compose exec alchemist \
+		datacube product add \
+		https://explorer.dev.dea.ga.gov.au/products/ga_s2bm_ard_provisional_3.odc-product.yaml
+
+index-s2-c3:
+	docker-compose exec alchemist \
+		datacube dataset add --ignore-lineage --confirm-ignore-lineage \
+			s3://dea-public-data/baseline/ga_s2am_ard_provisional_3/51/KWV/2021/08/18_nrt/20210818T033715/ga_s2am_ard_provisional_3-2-1_51KWV_2021-08-18_nrt.odc-metadata.yaml \
+
 # Specific BE dataset for local testing
 index-one-s2be:
 	docker-compose exec alchemist \
@@ -125,10 +139,20 @@ index-s2-nrt:
 			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-05-18/S2A_OPER_MSI_ARD_TL_VGS4_20210518T025201_A030830_T53KLV_N03.00/ARD-METADATA.yaml \
 			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-05-16/S2A_OPER_MSI_ARD_TL_VGS1_20210516T054329_A030802_T50JMS_N03.00/ARD-METADATA.yaml \
 			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-06-18/S2A_OPER_MSI_ARD_TL_VGS4_20210618T022813_A031273_T54LXJ_N03.00/ARD-METADATA.yaml \
-			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-06-20/S2B_OPER_MSI_ARD_TL_VGS4_20210620T015752_A022393_T55LBC_N03.00/ARD-METADATA.yaml
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-06-20/S2B_OPER_MSI_ARD_TL_VGS4_20210620T015752_A022393_T55LBC_N03.00/ARD-METADATA.yaml \
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-07-02/S2B_OPER_MSI_ARD_TL_VGS1_20210702T024204_A022565_T53KNT_N03.01/ARD-METADATA.yaml \
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-07-07/S2B_OPER_MSI_ARD_TL_VGS1_20210707T014852_A022636_T54HWC_N03.01/ARD-METADATA.yaml \
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-07-08/S2A_OPER_MSI_ARD_TL_VGS4_20210708T022548_A031559_T54HTJ_N03.01/ARD-METADATA.yaml \
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-07-03/S2A_OPER_MSI_ARD_TL_VGS4_20210703T013119_A031487_T55HFB_N03.01/ARD-METADATA.yaml \
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-07-31/S2B_OPER_MSI_ARD_TL_VGS4_20210731T011212_A022979_T56JKT_N03.01/ARD-METADATA.yaml \
+			s3://dea-public-data/L2/sentinel-2-nrt/S2MSIARD/2021-08-05/S2A_OPER_MSI_ARD_TL_VGS4_20210805T013351_A031959_T56JKT_N03.01/ARD-METADATA.yaml
 
 quickstart: initdb metadata product index index-geomedian metadata-s2-nrt product-s2-nrt metadata-eo_plus index-s2-nrt product-s2a index-s2a product-s2be index-s2be
 
+index-ba-bm-s2:
+    docker-compose exec alchemist \
+	    datacube dataset add --ignore-lineage --confirm-ignore-lineage \
+		    s3://dea-public-data/derivative/ga_s2_ba_bm_3/1-6-0/54/HVE/2021/07/22/20210722T015557/ga_s2_ba_bm_3_54HVE_2021-07-22_interim.odc-metadata.yaml
 
 # Landsat 8, 7 and 5 respectively
 THREE_SCENES=600645a5-5256-4632-a13d-fa13d1c11a8f 8b215983-ae1b-45bd-ad63-7245248bd41b 3fda2741-e810-4d3e-a54a-279fc3cd795f
@@ -151,7 +175,12 @@ fc-one:
 dnbr-one:
 	docker-compose exec alchemist \
 		datacube-alchemist run-one --config-file ./examples/c3_config_dnbr_3band_s2be.yaml \
-		--uuid f9a66dde-d423-47b5-8421-a71cfb1d8883
+		--uuid 5c70a4a2-cf36-4779-92a8-b35b8039cb0a
+
+ba-bm-one:
+	docker-compose exec alchemist \
+		datacube-alchemist run-one --config-file ./examples/c3_config_bm_provisional.yaml \
+		--uuid d7a7f9d1-8d89-409c-b478-5d02da6cb90a
 
 bai-one:
 	docker-compose exec alchemist \

datacube_alchemist/transforms.py

@@ -211,22 +211,21 @@ def compute(self, data) -> xr.Dataset:
             .astype(numpy.single)
         )
 
-        # Burn Scar Index (BSI) = ((B11 + B04) - (B08 - B02)) / ((B11 + B04) + (B08 - B02))
+        # Bare Soil Index (Rikimaru, Miyatake 2002)
+        # (BSI) = ((Swir2 + red) - (nir + Blue)) / ((Swir2 + red) + (nir + Blue))
         logger.info("Starting BSI calculation")
         pre_bsi = (
-            (gm_data.nbart_swir_2 / 10000 + gm_data.nbart_red / 10000)
-            - (gm_data.nbart_nir_1 / 10000 - gm_data.nbart_blue / 10000)
+            (gm_data.nbart_swir_2 + gm_data.nbart_red)
+            - (gm_data.nbart_nir_1 + gm_data.nbart_blue)
         ) / (
-            (gm_data.nbart_swir_2 / 10000 + gm_data.nbart_red / 10000)
-            + (gm_data.nbart_nir_1 / 10000 - gm_data.nbart_blue / 10000)
+            (gm_data.nbart_swir_2 + gm_data.nbart_red)
+            + (gm_data.nbart_nir_1 + gm_data.nbart_blue)
         )
         data = xr.merge([data, {"pre_bsi": pre_bsi}])
         data["post_bsi"] = (
-            (data.nbart_swir_2 / 10000 + data.nbart_red / 10000)
-            - (data.nbart_nir_1 / 10000 - data.nbart_blue / 10000)
+            (data.nbart_swir_2 + data.nbart_red) - (data.nbart_nir_1 + data.nbart_blue)
         ) / (
-            (data.nbart_swir_2 / 10000 + data.nbart_red / 10000)
-            + (data.nbart_nir_1 / 10000 - data.nbart_blue / 10000)
+            (data.nbart_swir_2 + data.nbart_red) + (data.nbart_nir_1 + data.nbart_blue)
         )
 
         # TODO - Review NaN handling on BSI data here
@@ -236,6 +235,9 @@ def compute(self, data) -> xr.Dataset:
             .where(data.nbart_nir1 != numpy.NaN)
             .astype(numpy.single)
         )
+        data["delta_bsi"] = (
+            data.delta_bsi * -1
+        )  # multiply by -1 to scale the same as other models
 
         # Normalized Difference Vegetation Index (NDVI) = (B08 - B04)/(B08 + B04)
         logger.info("Starting NDVI calculation")
@@ -248,7 +250,7 @@ def compute(self, data) -> xr.Dataset:
         )
 
         # TODO - Review NaN handling on NDVI data here
-        data["delta_ndvi"] = data.post_ndvi - data.pre_ndvi
+        data["delta_ndvi"] = data.pre_ndvi - data.post_ndvi
         data["delta_ndvi"] = (
             data.delta_ndvi.where(data.nbart_nir_1 != -999)
             .where(data.nbart_nir1 != numpy.NaN)
@@ -324,7 +326,7 @@ def compute(self, data) -> xr.Dataset:
         # TODO - remove this section, for debugging only. Find the S2 data for the geomedian
         dc = Datacube()
         gm_query = dc.find_datasets(
-            product=["s2_barest_earth"],
+            product="s2_barest_earth",
             time=gm_base_year,
             like=data.geobox,
         )
@@ -392,7 +394,7 @@ def compute(self, data) -> xr.Dataset:
 
         logger.debug("starting dnbr calculations\n")
 
-        # Delta Normalised Burn Ratio (dNBR) = (B08 - B11)/(B08 + B11)
+        # Normalised Burn Ratio (NBR) = (B08 - B11)/(B08 + B11)
         pre_nbr = (gm_data.s2be_nir_1 - gm_data.s2be_swir_2) / (
             gm_data.s2be_nir_1 + gm_data.s2be_swir_2
         )
@@ -402,7 +404,7 @@ def compute(self, data) -> xr.Dataset:
         data["post_nbr"] = (data.nbart_nir_1 - data.nbart_swir_2) / (
             data.nbart_nir_1 + data.nbart_swir_2
         )
-
+        # Delta NBR preNBR - postNBR
         data["delta_nbr"] = data.pre_nbr - data.post_nbr
         # Filter output NBR data based on:
         # 1. contiguity layer instead of nir_1
@@ -418,24 +420,23 @@ def compute(self, data) -> xr.Dataset:
             numpy.single
         )
 
-        # Burn Scar Index (BSI) = ((B11 + B04) - (B08 - B02)) / ((B11 + B04) + (B08 - B02))
+        # Bare Soil Index (Rikimaru, Miyatake 2002)
+        # (BSI) = ((Swir2 + red) - (nir + Blue)) / ((Swir2 + red) + (nir + Blue))
         logger.info("Starting BSI calculation")
         pre_bsi = (
-            (gm_data.s2be_swir_2 / 10000 + gm_data.s2be_red / 10000)
-            - (gm_data.s2be_nir_1 / 10000 - gm_data.s2be_blue / 10000)
+            (gm_data.s2be_swir_2 + gm_data.s2be_red)
+            - (gm_data.s2be_nir_1 + gm_data.s2be_blue)
         ) / (
-            (gm_data.s2be_swir_2 / 10000 + gm_data.s2be_red / 10000)
-            + (gm_data.s2be_nir_1 / 10000 - gm_data.s2be_blue / 10000)
+            (gm_data.s2be_swir_2 + gm_data.s2be_red)
+            + (gm_data.s2be_nir_1 + gm_data.s2be_blue)
         )
         data = xr.merge([data, {"pre_bsi": pre_bsi}])
         data["post_bsi"] = (
-            (data.nbart_swir_2 / 10000 + data.nbart_red / 10000)
-            - (data.nbart_nir_1 / 10000 - data.nbart_blue / 10000)
+            (data.nbart_swir_2 + data.nbart_red) - (data.nbart_nir_1 + data.nbart_blue)
         ) / (
-            (data.nbart_swir_2 / 10000 + data.nbart_red / 10000)
-            + (data.nbart_nir_1 / 10000 - data.nbart_blue / 10000)
+            (data.nbart_swir_2 + data.nbart_red) + (data.nbart_nir_1 + data.nbart_blue)
         )
-
+        # delta BSI preBSI - post BSI
         data["delta_bsi"] = data.pre_bsi - data.post_bsi
         # Filter output BSI data based on:
         # 1. contiguity layer instead of nir_1
@@ -450,6 +451,9 @@ def compute(self, data) -> xr.Dataset:
         data["delta_bsi"] = data.delta_bsi.where(data.nbart_nir_1 != numpy.NaN).astype(
             numpy.single
         )
+        data["delta_bsi"] = (
+            data.delta_bsi * -1
+        )  # multiply by -1 to scale the same as other models
 
         # Normalized Difference Vegetation Index (NDVI) = (B08 - B04)/(B08 + B04)
         logger.info("Starting NDVI calculation")
@@ -465,7 +469,7 @@ def compute(self, data) -> xr.Dataset:
         # 1. contiguity layer instead of nir_1
         # 2. barest earth nodata value
         # 3. only keep finite values on output band.
-        data["delta_ndvi"] = data.post_ndvi - data.pre_ndvi
+        data["delta_ndvi"] = data.pre_ndvi - data.post_ndvi
         data["delta_ndvi"] = data.delta_ndvi.where(data.nbart_nir_1 != -999).astype(
             numpy.single
         )

examples/c3_config_bm_provisional.yaml

@@ -0,0 +1,42 @@
+specification:
+  products:
+    - ga_s2am_ard_provisional_3
+    - ga_s2bm_ard_provisional_3
+  measurements: ['nbart_nir_1', 'nbart_red_edge_1', 'nbart_blue', 'nbart_red', 'fmask']
+  measurement_renames:
+    nbart_red_edge_1: nbart_swir_2
+
+  aws_unsigned: True
+  transform: datacube_alchemist.transforms.DeltaNBR_3band_s2be
+
+  override_product_family: ard
+  basis: nbart_nir_1
+
+output:
+  location: s3://dea-public-data-dev/derivative/
+  preview_image:
+    red: delta_nbr
+    green: delta_ndvi
+    blue: delta_bsi
+    static_stretch: [-1, 1]
+  explorer_url: https://explorer.dev.dea.ga.gov.au
+  write_data_settings:
+    overview_resampling: average
+  reference_source_dataset: True
+  write_stac: True
+  inherit_geometry: True
+
+  metadata:
+    product_family: ba
+    producer: ga.gov.au
+    dataset_version: 1.6.0
+    collection_number: 3
+    naming_conventions: dea_c3
+  properties:
+    dea:dataset_maturity: interim
+    dea:product_maturity: provisional
+
+processing:
+  dask_chunks:
+    x: -1
+    y: 4096

examples/c3_config_dnbr_3band_s2be.yaml

@@ -27,13 +27,14 @@ output:
   inherit_geometry: True
 
   metadata:
-    product_family: ba_bm
+    product_family: ba
     producer: ga.gov.au
     dataset_version: 1.6.0
     collection_number: 3
     naming_conventions: dea_c3
   properties:
     dea:dataset_maturity: interim
+    dea:product_maturity: provisional
 
 processing:
   dask_chunks:

tests/c3_config_dnbr_3band_s2be.yaml

@@ -27,13 +27,14 @@ output:
   inherit_geometry: True
 
   metadata:
-    product_family: ba_bm
+    product_family: ba
     producer: ga.gov.au
     dataset_version: 1.6.0
     collection_number: 3
     naming_conventions: dea_c3
   properties:
     dea:dataset_maturity: interim
+    dea:product_maturity: provisional
 
 processing:
   dask_chunks: