Skip to content

Commit

Permalink
remove redundant bands from SH results file when transforming to pars…
Browse files Browse the repository at this point in the history
…ed files
  • Loading branch information
zcernigoj committed Nov 17, 2023
1 parent dc8df47 commit c3202e7
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 8 deletions.
7 changes: 6 additions & 1 deletion rest/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,12 @@ def add_job_to_queue(job_id):
# START OF POST_PROCESSING
# post-process gtiffs to appropriate formats

parse_sh_gtiff_to_format(job, bucket)
try:
parse_sh_gtiff_to_format(job, bucket)
except Exception as e:
print("parsing didn't succeed")
print(e)


# END OF POST_PROCESSING

Expand Down
55 changes: 48 additions & 7 deletions rest/post_processing/gtiff_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@
from processing.const import CustomMimeType
from openeoerrors import Internal

import json

# assume it's only 1 time and 1 bands dimension
def check_dimensions(time_dimensions, bands_dimensions):
if len(time_dimensions) == 0:
raise Internal("No time dimensions exist. Only 1 time dimension is supported.")
print("No time dimensions exist. Only 1 time dimension is supported.")
# raise Internal("No time dimensions exist. Only 1 time dimension is supported.")

if len(time_dimensions) > 1:
raise Internal("More than 1 time dimension exist. Only 1 time dimension is supported.")
Expand All @@ -29,18 +32,52 @@ def get_timestamps_arrays(datacube_time_as_bands, time_dimensions, bands_dimensi
num_of_img_bands = len(datacube_time_as_bands["band"])
num_of_bands_dimension = len(bands_dimensions[0]["labels"])

num_time_labels = 0
for time_dim in time_dimensions:
num_time_labels += len(time_dim["labels"])

num_band_labels = 0
for band_dim in bands_dimensions:
num_band_labels += len(band_dim["labels"])

num_actual_img_bands = (num_time_labels or 1) * (num_band_labels or 1)

list_of_timestamps = []
list_of_timestamp_arrays = []

for i in range(0, num_of_img_bands, num_of_bands_dimension):
date = time_dimensions[0]["labels"][int(i / num_of_bands_dimension)]
printdata = {
"data": datacube_time_as_bands,
"data_length": len(datacube_time_as_bands),
"data_len/num_of_bands_dimension": len(datacube_time_as_bands) / num_of_bands_dimension,
"bands_dimensions": bands_dimensions,
"time_dimensions": time_dimensions,
"num_img_bands": num_of_img_bands,
"num_bands_dim": num_of_bands_dimension,
"range": range(0, num_actual_img_bands, num_of_bands_dimension),
"num_actual_img_bands": num_actual_img_bands,
"num_band_labels": num_band_labels,
"num_time_labels": num_time_labels,
}

print("get_timestamps_arrays")
print(json.dumps(printdata, sort_keys=True, indent=4, default=str))

for i in range(0, num_actual_img_bands, num_of_bands_dimension):

print("for loop", {
"time_dims": time_dimensions,
"time_dim_index": int(i / num_of_bands_dimension)
})


date = time_dimensions[0]["labels"][int(i / num_of_bands_dimension)] if num_time_labels > 0 else ""
timestamp_array = datacube_time_as_bands[i : i + num_of_bands_dimension]

if output_format in [CustomMimeType.NETCDF, CustomMimeType.ZARR]:
pandas_time = pd.to_datetime(parser.parse(date))
timestamp_array = timestamp_array.assign_coords(band=bands_dimensions[0]["labels"])
timestamp_array = timestamp_array.assign_coords(t=pandas_time)
timestamp_array = timestamp_array.expand_dims(dim="t")
if num_time_labels > 0:
timestamp_array = timestamp_array.assign_coords(t=pd.to_datetime(parser.parse(date)))
timestamp_array = timestamp_array.expand_dims(dim="t")

list_of_timestamps.append(date)
list_of_timestamp_arrays.append(timestamp_array)
Expand All @@ -51,7 +88,8 @@ def get_timestamps_arrays(datacube_time_as_bands, time_dimensions, bands_dimensi
def save_as_gtiff(list_of_timestamps, list_of_timestamp_arrays, output_dir, output_name):
output_file_paths = []
for array, date in zip(list_of_timestamp_arrays, list_of_timestamps):
file_name = f"{output_name['name']}_{date}{output_name['ext']}"
date_string = "" if date == "" else f"_{date}"
file_name = f"{output_name['name']}{date_string}{output_name['ext']}"
file_path = os.path.join(output_dir, file_name)
output_file_paths.append(file_path)

Expand Down Expand Up @@ -92,6 +130,9 @@ def parse_multitemporal_gtiff_to_format(input_tiff, input_metadata, output_dir,
if len(bands_dimensions) == 0:
bands_dimensions = [{"name": "bands", "type": "bands", "labels": ["results"]}]

# if len(time_dimensions) == 0:
# time_dimensions = [{"name": "t", "type": "temporal", "labels": [""]}]

check_dimensions(time_dimensions, bands_dimensions)

list_of_timestamps, list_of_timestamp_arrays = get_timestamps_arrays(
Expand Down

0 comments on commit c3202e7

Please sign in to comment.