From 4db115ae294bd3030aec102ce6fba6a669472e17 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Thu, 21 Nov 2024 14:52:02 +0100 Subject: [PATCH] Log files to check what we are dealing with. Probably too many files get uploaded too often. https://github.com/Open-EO/openeo-geopyspark-driver/issues/940 --- openeogeotrellis/deploy/batch_job.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openeogeotrellis/deploy/batch_job.py b/openeogeotrellis/deploy/batch_job.py index 15bf99b64..1736d5b24 100644 --- a/openeogeotrellis/deploy/batch_job.py +++ b/openeogeotrellis/deploy/batch_job.py @@ -508,8 +508,9 @@ def write_metadata(metadata, metadata_file, job_dir: Path): bucket = os.environ.get('SWIFT_BUCKET') s3_instance = s3_client() - logger.info("Writing results to object storage") - for file_path in filter(lambda x: x.is_file(), job_dir.rglob("*")): + paths = list(filter(lambda x: x.is_file(), job_dir.rglob("*"))) + logger.info(f"Writing results to object storage. paths={paths}") + for file_path in paths: # TODO: Get list of files to upload from metadata file. # AFAIK, nothing else is created locally and needs to be uploaded to s3 afterwards. if UDF_PYTHON_DEPENDENCIES_FOLDER_NAME in str(file_path):