Skip to content

Added warning unused data (DAX translator) #46

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Added warning unused data (DAX translator) #46

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
875a1cd
This commit fixes a DAX translation error.
LanderOtto Nov 29, 2024
d7d234e
This commit fixes a DAX translator error.
LanderOtto Nov 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions swirlc/translator/dax_translator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
Data,
)
from swirlc.core.translator import AbstractTranslator
from swirlc.log_handler import logger

MANDATORY_FILES = ("replicas", "sites", "transformations", "workflow")

Expand Down Expand Up @@ -112,8 +113,8 @@ def _translate(self) -> Workflow:
swirl_data_name
)
if data["stageOut"]:
# Generate a unique id for the collector step based on the DAX id of the previous step
collect_id = "COLLECT-" + replica["id"]
# Generate a unique id for the collector step
collect_id = f"COLLECT-{replica['id']}-{swirl_data_name}"
dax_step_name_id.setdefault(
f"{replica['name']}-{swirl_data_name}-collector", []
).append(collect_id)
Expand Down Expand Up @@ -238,6 +239,11 @@ def _translate(self) -> Workflow:
(physical_path["site"], physical_path["pfn"])
)
location_name = location_binding_dax_swirl[physical_path["site"]]
if replica["lfn"] not in data_binding_dax_swirl.keys():
logger.warning(
f"The input data `{replica['lfn']}` is not used by any step"
)
continue
data_name = data_binding_dax_swirl[replica["lfn"]]
if location_name in workflow.locations.keys():
workflow.locations[location_name].data[data_name] = Data(
Expand All @@ -249,6 +255,11 @@ def _translate(self) -> Workflow:
for transformation in transformations_config["transformations"]:
for binding in transformation["sites"]:
location_name = location_binding_dax_swirl[binding["name"]]
if transformation["name"] not in dax_step_name_id.keys():
logger.warning(
f"The command `{transformation['name']}` is not used by any step"
)
continue
for dax_step_id in dax_step_name_id[transformation["name"]]:
step_name = step_binding_dax_swirl[dax_step_id]
workflow.steps[step_name].command = binding["pfn"]
Expand Down