Skip to content

Commit f8de6fa

Browse files
tclosetclose
committed
debugged function and shell command task conversion
1 parent 3e97cc7 commit f8de6fa

11 files changed

+316
-9
lines changed

‎conftest.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,14 @@ def gen_test_conftest():
1717
return PKG_DIR / "scripts" / "pkg_gen" / "resources" / "conftest.py"
1818

1919

20-
@pytest.fixture(params=[str(p.stem) for p in (EXAMPLE_TASKS_DIR).glob("*.yaml")])
20+
@pytest.fixture(
21+
params=[
22+
str(p.relative_to(EXAMPLE_TASKS_DIR)).replace("/", "-")[:-5]
23+
for p in (EXAMPLE_TASKS_DIR).glob("**/*.yaml")
24+
]
25+
)
2126
def task_spec_file(request):
22-
return (EXAMPLE_TASKS_DIR / request.param).with_suffix(".yaml")
27+
return EXAMPLE_TASKS_DIR.joinpath(*request.param.split("-")).with_suffix(".yaml")
2328

2429

2530
@pytest.fixture(params=[str(p.stem) for p in EXAMPLE_WORKFLOWS_DIR.glob("*.yaml")])
@@ -35,7 +40,7 @@ def work_dir():
3540

3641
@pytest.fixture
3742
def outputs_dir():
38-
outputs_dir = PKG_DIR / "outputs" / 'workflows'
43+
outputs_dir = PKG_DIR / "outputs" / "workflows"
3944
outputs_dir.mkdir(parents=True, exist_ok=True)
4045
return outputs_dir
4146

‎example-specs/task/function/mapmri_reconstruction.yaml

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
# This file is used to manually specify the semi-automatic conversion of
2+
# 'qsiprep.interfaces.dipy.MAPMRIReconstruction' from Nipype to Pydra.
3+
#
4+
# Please fill-in/edit the fields below where appropriate
5+
#
6+
# Docs
7+
# ----
8+
#
9+
task_name: MAPMRIReconstruction
10+
nipype_name: MAPMRIReconstruction
11+
nipype_module: qsiprep.interfaces.dipy
12+
inputs:
13+
omit:
14+
# list[str] - fields to omit from the Pydra interface
15+
rename:
16+
# dict[str, str] - fields to rename in the Pydra interface
17+
types:
18+
# dict[str, type] - override inferred types (use "mime-like" string for file-format types,
19+
# e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
20+
# from the nipype interface, but you may want to be more specific, particularly
21+
# for file types, where specifying the format also specifies the file that will be
22+
# passed to the field in the automatically generated unittests.
23+
bval_file: generic/file
24+
# type=file|default=<undefined>:
25+
bvec_file: generic/file
26+
# type=file|default=<undefined>:
27+
dwi_file: generic/file
28+
# type=file|default=<undefined>:
29+
mask_file: generic/file
30+
# type=file|default=<undefined>:
31+
local_bvec_file: generic/file
32+
# type=file|default=<undefined>:
33+
metadata:
34+
# dict[str, dict[str, any]] - additional metadata to set on any of the input fields (e.g. out_file: position: 1)
35+
outputs:
36+
omit:
37+
- ng
38+
- perng
39+
- parng
40+
- extrapolated_dwi
41+
- extrapolated_bvals
42+
- extrapolated_bvecs
43+
- extrapolated_b
44+
# list[str] - fields to omit from the Pydra interface
45+
rename:
46+
# dict[str, str] - fields to rename in the Pydra interface
47+
types:
48+
# dict[str, type] - override inferred types (use "mime-like" string for file-format types,
49+
# e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
50+
# from the nipype interface, but you may want to be more specific, particularly
51+
# for file types, where specifying the format also specifies the file that will be
52+
# passed to the field in the automatically generated unittests.
53+
rtop: generic/file
54+
# type=file:
55+
lapnorm: generic/file
56+
# type=file:
57+
msd: generic/file
58+
# type=file:
59+
qiv: generic/file
60+
# type=file:
61+
rtap: generic/file
62+
# type=file:
63+
rtpp: generic/file
64+
# type=file:
65+
ng: generic/file
66+
# type=file:
67+
perng: generic/file
68+
# type=file:
69+
parng: generic/file
70+
# type=file:
71+
mapmri_coeffs: generic/file
72+
# type=file:
73+
fibgz: generic/file
74+
# type=file:
75+
fod_sh_mif: generic/file
76+
# type=file:
77+
extrapolated_dwi: generic/file
78+
# type=file:
79+
extrapolated_bvals: generic/file
80+
# type=file:
81+
extrapolated_bvecs: generic/file
82+
# type=file:
83+
extrapolated_b: generic/file
84+
# type=file:
85+
odf_amplitudes: generic/file
86+
# type=file:
87+
odf_directions: generic/file
88+
# type=file:
89+
callables:
90+
# dict[str, str] - names of methods/callable classes defined in the adjacent `*_callables.py`
91+
# to set to the `callable` attribute of output fields
92+
templates:
93+
# dict[str, str] - `output_file_template` values to be provided to output fields
94+
requirements:
95+
# dict[str, list[str]] - input fields that are required to be provided for the output field to be present
96+
tests:
97+
- inputs:
98+
# dict[str, str] - values to provide to inputs fields in the task initialisation
99+
# (if not specified, will try to choose a sensible value)
100+
radial_order:
101+
# type=int|default=6:
102+
laplacian_regularization:
103+
# type=bool|default=True:
104+
laplacian_weighting:
105+
# type=traitcompound|default=None:
106+
positivity_constraint:
107+
# type=bool|default=False:
108+
pos_grid:
109+
# type=int|default=15:
110+
pos_radius:
111+
# type=traitcompound|default='adaptive':
112+
anisotropic_scaling:
113+
# type=bool|default=True:
114+
eigenvalue_threshold:
115+
# type=float|default=0.0001:
116+
bval_threshold:
117+
# type=float|default=0.0:
118+
dti_scale_estimation:
119+
# type=bool|default=True:
120+
static_diffusivity:
121+
# type=float|default=0.0007:
122+
cvxpy_solver:
123+
# type=str|default='':
124+
bval_file:
125+
# type=file|default=<undefined>:
126+
bvec_file:
127+
# type=file|default=<undefined>:
128+
dwi_file:
129+
# type=file|default=<undefined>:
130+
mask_file:
131+
# type=file|default=<undefined>:
132+
local_bvec_file:
133+
# type=file|default=<undefined>:
134+
big_delta:
135+
# type=traitcompound|default=None:
136+
little_delta:
137+
# type=traitcompound|default=None:
138+
b0_threshold:
139+
# type=cfloat|default=50:
140+
write_fibgz:
141+
# type=bool|default=True:
142+
write_mif:
143+
# type=bool|default=True:
144+
extrapolate_scheme:
145+
# type=enum|default='HCP'|allowed['ABCD','HCP']:
146+
imports:
147+
# list[nipype2pydra.task.importstatement] - list import statements required by the test, with each list item
148+
# consisting of 'module', 'name', and optionally 'alias' keys
149+
expected_outputs:
150+
# dict[str, str] - expected values for selected outputs, noting that tests will typically
151+
# be terminated before they complete for time-saving reasons, and therefore
152+
# these values will be ignored, when running in CI
153+
timeout: 10
154+
# int - the value to set for the timeout in the generated test,
155+
# after which the test will be considered to have been initialised
156+
# successfully. Set to 0 to disable the timeout (warning, this could
157+
# lead to the unittests taking a very long time to complete)
158+
xfail: true
159+
# bool - whether the unittest is expected to fail or not. Set to false
160+
# when you are satisfied with the edits you have made to this file
161+
doctests: []

‎example-specs/task/function/mapmri_reconstruction_callables.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Module to put any functions that are referred to in MAPMRIReconstruction.yaml"""

‎example-specs/task/function/tensor_reconstruction.yaml

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
# This file is used to manually specify the semi-automatic conversion of
2+
# 'qsiprep.interfaces.dipy.TensorReconstruction' from Nipype to Pydra.
3+
#
4+
# Please fill-in/edit the fields below where appropriate
5+
#
6+
# Docs
7+
# ----
8+
#
9+
task_name: TensorReconstruction
10+
nipype_name: TensorReconstruction
11+
nipype_module: qsiprep.interfaces.dipy
12+
inputs:
13+
omit:
14+
# list[str] - fields to omit from the Pydra interface
15+
rename:
16+
# dict[str, str] - fields to rename in the Pydra interface
17+
types:
18+
# dict[str, type] - override inferred types (use "mime-like" string for file-format types,
19+
# e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
20+
# from the nipype interface, but you may want to be more specific, particularly
21+
# for file types, where specifying the format also specifies the file that will be
22+
# passed to the field in the automatically generated unittests.
23+
bval_file: generic/file
24+
# type=file|default=<undefined>:
25+
bvec_file: generic/file
26+
# type=file|default=<undefined>:
27+
dwi_file: generic/file
28+
# type=file|default=<undefined>:
29+
mask_file: generic/file
30+
# type=file|default=<undefined>:
31+
local_bvec_file: generic/file
32+
# type=file|default=<undefined>:
33+
metadata:
34+
# dict[str, dict[str, any]] - additional metadata to set on any of the input fields (e.g. out_file: position: 1)
35+
outputs:
36+
omit:
37+
# list[str] - fields to omit from the Pydra interface
38+
rename:
39+
# dict[str, str] - fields to rename in the Pydra interface
40+
types:
41+
# dict[str, type] - override inferred types (use "mime-like" string for file-format types,
42+
# e.g. 'medimage/nifti-gz'). For most fields the type will be correctly inferred
43+
# from the nipype interface, but you may want to be more specific, particularly
44+
# for file types, where specifying the format also specifies the file that will be
45+
# passed to the field in the automatically generated unittests.
46+
color_fa_image: generic/file
47+
# type=file:
48+
fa_image: generic/file
49+
# type=file:
50+
md_image: generic/file
51+
# type=file:
52+
rd_image: generic/file
53+
# type=file:
54+
ad_image: generic/file
55+
# type=file:
56+
cnr_image: generic/file
57+
# type=file:
58+
fibgz: generic/file
59+
# type=file:
60+
fod_sh_mif: generic/file
61+
# type=file:
62+
extrapolated_dwi: generic/file
63+
# type=file:
64+
extrapolated_bvals: generic/file
65+
# type=file:
66+
extrapolated_bvecs: generic/file
67+
# type=file:
68+
extrapolated_b: generic/file
69+
# type=file:
70+
odf_amplitudes: generic/file
71+
# type=file:
72+
odf_directions: generic/file
73+
# type=file:
74+
callables:
75+
# dict[str, str] - names of methods/callable classes defined in the adjacent `*_callables.py`
76+
# to set to the `callable` attribute of output fields
77+
templates:
78+
# dict[str, str] - `output_file_template` values to be provided to output fields
79+
requirements:
80+
# dict[str, list[str]] - input fields that are required to be provided for the output field to be present
81+
tests:
82+
- inputs:
83+
# dict[str, str] - values to provide to inputs fields in the task initialisation
84+
# (if not specified, will try to choose a sensible value)
85+
bval_file:
86+
# type=file|default=<undefined>:
87+
bvec_file:
88+
# type=file|default=<undefined>:
89+
dwi_file:
90+
# type=file|default=<undefined>:
91+
mask_file:
92+
# type=file|default=<undefined>:
93+
local_bvec_file:
94+
# type=file|default=<undefined>:
95+
big_delta:
96+
# type=traitcompound|default=None:
97+
little_delta:
98+
# type=traitcompound|default=None:
99+
b0_threshold:
100+
# type=cfloat|default=50:
101+
write_fibgz:
102+
# type=bool|default=True:
103+
write_mif:
104+
# type=bool|default=True:
105+
extrapolate_scheme:
106+
# type=enum|default='HCP'|allowed['ABCD','HCP']:
107+
imports:
108+
# list[nipype2pydra.task.importstatement] - list import statements required by the test, with each list item
109+
# consisting of 'module', 'name', and optionally 'alias' keys
110+
expected_outputs:
111+
# dict[str, str] - expected values for selected outputs, noting that tests will typically
112+
# be terminated before they complete for time-saving reasons, and therefore
113+
# these values will be ignored, when running in CI
114+
timeout: 10
115+
# int - the value to set for the timeout in the generated test,
116+
# after which the test will be considered to have been initialised
117+
# successfully. Set to 0 to disable the timeout (warning, this could
118+
# lead to the unittests taking a very long time to complete)
119+
xfail: true
120+
# bool - whether the unittest is expected to fail or not. Set to false
121+
# when you are satisfied with the edits you have made to this file
122+
doctests: []

‎example-specs/task/function/tensor_reconstruction_callables.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Module to put any functions that are referred to in TensorReconstruction.yaml"""

‎example-specs/task/ants_n4_bias_field_correction.yaml renamed to ‎example-specs/task/shell_command/ants_n4_bias_field_correction.yaml

File renamed without changes.

‎example-specs/task/ants_registration.yaml renamed to ‎example-specs/task/shell_command/ants_registration.yaml

File renamed without changes.

‎example-specs/task/apply_vol_transform.yaml renamed to ‎example-specs/task/shell_command/apply_vol_transform.yaml

File renamed without changes.

‎example-specs/task/extract_roi.yaml renamed to ‎example-specs/task/shell_command/extract_roi.yaml

File renamed without changes.

‎nipype2pydra/task/function.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from .base import BaseTaskConverter
1010

1111

12-
@attrs.define
12+
@attrs.define(slots=False)
1313
class FunctionTaskConverter(BaseTaskConverter):
1414
def generate_task_str(self, filename, input_fields, nonstd_types, output_fields):
1515
"""writing pydra task to the dile based on the input and output spec"""
@@ -342,6 +342,8 @@ def get_imports_and_functions_to_include(
342342
const_name in used_symbols
343343
and (const_name, const_def) not in used_constants
344344
):
345+
if const_name == "LOGGER":
346+
continue
345347
used_constants.add((const_name, const_def))
346348
const_def_symbols = re.findall(r"(\w+)", const_def)
347349
used_symbols.update(const_def_symbols)

‎tests/test_task.py

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
from importlib import import_module
22
import yaml
3-
from conftest import show_cli_trace
43
import pytest
54
import logging
5+
from conftest import show_cli_trace
66
from nipype2pydra.cli import task as task_cli
77
from nipype2pydra.utils import add_to_sys_path
88

@@ -49,17 +49,32 @@ def test_task_conversion(task_spec_file, cli_runner, work_dir, gen_test_conftest
4949
import_module(task_spec["nipype_module"]), task_spec["nipype_name"]
5050
)
5151

52-
nipype_trait_names = nipype_interface.input_spec().all_trait_names()
52+
nipype_input_names = nipype_interface.input_spec().all_trait_names()
53+
inputs_omit = task_spec["inputs"]["omit"] if task_spec["inputs"]["omit"] else []
5354

54-
assert sorted(f[0] for f in pydra_task.input_spec.fields) == sorted(
55+
assert sorted(f[0] for f in pydra_task().input_spec.fields if not f[0].startswith("_")) == sorted(
5556
n
56-
for n in nipype_trait_names
57+
for n in nipype_input_names
5758
if not (
5859
n in INBUILT_NIPYPE_TRAIT_NAMES
59-
or (n.endswith("_items") and n[: -len("_items")] in nipype_trait_names)
60+
or n in inputs_omit
61+
or (n.endswith("_items") and n[: -len("_items")] in nipype_input_names)
6062
)
6163
)
6264

65+
nipype_output_names = nipype_interface.output_spec().all_trait_names()
66+
outputs_omit = task_spec["outputs"]["omit"] if task_spec["outputs"]["omit"] else []
67+
68+
assert sorted(f[0] for f in pydra_task().output_spec.fields if not f[0].startswith("_")) == sorted(
69+
n
70+
for n in nipype_output_names
71+
if not (
72+
n in INBUILT_NIPYPE_TRAIT_NAMES
73+
or n in outputs_omit
74+
or (n.endswith("_items") and n[: -len("_items")] in nipype_output_names)
75+
)
76+
)
77+
6378
tests_fspath = pkg_root.joinpath(*output_module_path.split(".")).parent / "tests"
6479

6580
logging.info("Running generated tests for %s", output_module_path)

0 commit comments

Comments
 (0)