From 875a1cde904fd1a9234973690cce0ba98b6c249f Mon Sep 17 00:00:00 2001 From: LanderOtto Date: Fri, 29 Nov 2024 18:37:23 +0100 Subject: [PATCH 1/2] This commit fixes a DAX translation error. A step can have multiple outputs and these can be the workflow output. Before this commit, if a step had multiple outputs that were workflow outputs, an exception was raised at translation time. --- swirlc/translator/dax_translator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swirlc/translator/dax_translator.py b/swirlc/translator/dax_translator.py index 43106bc..499f3fe 100644 --- a/swirlc/translator/dax_translator.py +++ b/swirlc/translator/dax_translator.py @@ -112,8 +112,8 @@ def _translate(self) -> Workflow: swirl_data_name ) if data["stageOut"]: - # Generate a unique id for the collector step based on the DAX id of the previous step - collect_id = "COLLECT-" + replica["id"] + # Generate a unique id for the collector step + collect_id = f"COLLECT-{replica['id']}-{swirl_data_name}" dax_step_name_id.setdefault( f"{replica['name']}-{swirl_data_name}-collector", [] ).append(collect_id) From d7d234e1576dd74a670b6c635d4161fc29fa2117 Mon Sep 17 00:00:00 2001 From: LanderOtto Date: Fri, 29 Nov 2024 18:41:57 +0100 Subject: [PATCH 2/2] This commit fixes a DAX translator error. Before this commit, it was assumed that each input data and command defined in `replica` and `transformations` files were used by the steps defined in the `workflow` file. Now, it is checked if they are used, otherwise a warning log is shown and they are skipped. --- swirlc/translator/dax_translator.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/swirlc/translator/dax_translator.py b/swirlc/translator/dax_translator.py index 499f3fe..c2bf361 100644 --- a/swirlc/translator/dax_translator.py +++ b/swirlc/translator/dax_translator.py @@ -18,6 +18,7 @@ Data, ) from swirlc.core.translator import AbstractTranslator +from swirlc.log_handler import logger MANDATORY_FILES = ("replicas", "sites", "transformations", "workflow") @@ -238,6 +239,11 @@ def _translate(self) -> Workflow: (physical_path["site"], physical_path["pfn"]) ) location_name = location_binding_dax_swirl[physical_path["site"]] + if replica["lfn"] not in data_binding_dax_swirl.keys(): + logger.warning( + f"The input data `{replica['lfn']}` is not used by any step" + ) + continue data_name = data_binding_dax_swirl[replica["lfn"]] if location_name in workflow.locations.keys(): workflow.locations[location_name].data[data_name] = Data( @@ -249,6 +255,11 @@ def _translate(self) -> Workflow: for transformation in transformations_config["transformations"]: for binding in transformation["sites"]: location_name = location_binding_dax_swirl[binding["name"]] + if transformation["name"] not in dax_step_name_id.keys(): + logger.warning( + f"The command `{transformation['name']}` is not used by any step" + ) + continue for dax_step_id in dax_step_name_id[transformation["name"]]: step_name = step_binding_dax_swirl[dax_step_id] workflow.steps[step_name].command = binding["pfn"]