Skip to content

Commit a34aa90

Browse files
mlinmlin
committed
update wdlviz links
1 parent dbfe115 commit a34aa90

File tree

2 files changed

+8
-164
lines changed

2 files changed

+8
-164
lines changed

‎docs/wdlviz.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# wdlviz
22

3-
In this lab, we'll develop a Python program using miniwdl's API to generate a [graphviz](https://www.graphviz.org/) visualization of a WDL workflow's internal dependency structure. We'll keep this example brief and barebones, while a more-elaborate version can be found [in the miniwdl repo](https://github.com/chanzuckerberg/miniwdl/blob/main/examples/wdlviz.py).
3+
In this lab, we'll develop a Python program using miniwdl's API to generate a [graphviz](https://www.graphviz.org/) visualization of a WDL workflow's internal dependency structure. We'll keep this example barebones for educational purposes, while a more-complete version can be found [in its own project](https://github.com/miniwdl-ext/wdlviz).
44

55
Begin by installing (i) graphviz using your OS package manager (e.g. `apt install graphviz`), and (ii) either `pip3 install miniwdl graphviz` or `conda install miniwdl python-graphviz` as you prefer.
66

@@ -260,4 +260,4 @@ which generates this interesting graphic:
260260

261261
![](wdlviz_ex2.png)
262262

263-
A more-elaborate version of this barebones example can be found [in the miniwdl repo](https://github.com/chanzuckerberg/miniwdl/blob/main/examples/wdlviz.py). Pull requests with feature and visual improvements are welcome!
263+
A more-complete version of this barebones example can be found [in its own project](https://github.com/miniwdl-ext/wdlviz). Pull requests with feature and visual improvements are welcome!

‎examples/wdlviz.py

Lines changed: 6 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -1,166 +1,10 @@
11
#!/usr/bin/env python3
22
"""
3-
Visualize a WDL workflow using miniwdl and graphviz
3+
This example has grown into its own project, see:
4+
https://github.com/miniwdl-ext/wdlviz
45
"""
5-
# black -l 100 wdlviz.py && pylint wdlviz.py
6-
import os
7-
import sys
8-
import argparse
9-
import tempfile
10-
import WDL
11-
import graphviz
12-
from urllib import request, parse
6+
import ast
137

14-
15-
def main(args=None):
16-
# read command-line arguments
17-
parser = argparse.ArgumentParser(
18-
description="Visualize a WDL workflow using miniwdl and graphviz"
19-
)
20-
parser.add_argument(
21-
"wdl", metavar="FILE", help="WDL workflow file or URL (- for standard input)"
22-
)
23-
parser.add_argument("--inputs", action="store_true", help="include input declarations")
24-
parser.add_argument("--outputs", action="store_true", help="include output declarations")
25-
parser.add_argument(
26-
"--no-quant-check",
27-
dest="check_quant",
28-
action="store_false",
29-
help="relax static typechecking of optional types, and permit coercion of T to Array[T] (discouraged; for backwards compatibility with older WDL)",
30-
)
31-
parser.add_argument(
32-
"-p",
33-
"--path",
34-
metavar="DIR",
35-
type=str,
36-
action="append",
37-
help="local directory to search for imports",
38-
)
39-
args = parser.parse_args(args if args is not None else sys.argv[1:])
40-
41-
# load WDL document
42-
doc = WDL.load(
43-
args.wdl if args.wdl != "-" else "/dev/stdin",
44-
args.path or [],
45-
check_quant=args.check_quant,
46-
read_source=read_source,
47-
)
48-
assert doc.workflow, "No workflow in WDL document"
49-
50-
# visualize workflow
51-
dot = wdlviz(doc.workflow, args.inputs, args.outputs)
52-
print(dot.source)
53-
dot.render(doc.workflow.name + ".dot", view=True)
54-
55-
56-
def wdlviz(workflow: WDL.Workflow, inputs=False, outputs=False):
57-
"""
58-
Project the workflow's built-in dependency graph onto a graphviz representation
59-
"""
60-
# References:
61-
# 1. WDL object model -- https://miniwdl.readthedocs.io/en/latest/WDL.html#module-WDL.Tree
62-
# 2. graphviz API -- https://graphviz.readthedocs.io/en/stable/manual.html
63-
64-
# initialiaze Digraph
65-
top = graphviz.Digraph(comment=workflow.name)
66-
top.attr(compound="true", rankdir="LR")
67-
fontname = "Roboto"
68-
top.attr("node", fontname=fontname)
69-
top.attr("edge", color="#00000080")
70-
node_ids = set()
71-
72-
# recursively add graphviz nodes for each decl/call/scatter/conditional workflow node.
73-
74-
def add_node(graph: graphviz.Digraph, node: WDL.WorkflowNode):
75-
nonlocal node_ids
76-
if isinstance(node, WDL.WorkflowSection):
77-
# scatter/conditional section: add a cluster subgraph to contain its body
78-
with graph.subgraph(name="cluster-" + node.workflow_node_id) as sg:
79-
label = "scatter" if isinstance(node, WDL.Scatter) else "if"
80-
sg.attr(label=label + f"({str(node.expr)})", fontname=fontname, rank="same")
81-
for child in node.body:
82-
add_node(sg, child)
83-
# Add an invisible node inside the subgraph, which provides a sink for dependencies
84-
# of the scatter/conditional expression itself
85-
sg.node(node.workflow_node_id, "", style="invis", height="0", width="0", margin="0")
86-
node_ids.add(node.workflow_node_id)
87-
node_ids |= set(g.workflow_node_id for g in node.gathers.values())
88-
elif isinstance(node, WDL.Call) or (
89-
isinstance(node, WDL.Decl)
90-
and (inputs or node_ids.intersection(node.workflow_node_dependencies))
91-
):
92-
# node for call or decl
93-
graph.node(
94-
node.workflow_node_id,
95-
node.name,
96-
shape=("cds" if isinstance(node, WDL.Call) else "plaintext"),
97-
)
98-
node_ids.add(node.workflow_node_id)
99-
100-
for node in workflow.body:
101-
add_node(top, node)
102-
103-
# cluster of the input decls
104-
if inputs:
105-
with top.subgraph(name="cluster-inputs") as sg:
106-
for inp in workflow.inputs or []:
107-
assert inp.workflow_node_id.startswith("decl-")
108-
sg.node(inp.workflow_node_id, inp.workflow_node_id[5:], shape="plaintext")
109-
node_ids.add(inp.workflow_node_id)
110-
sg.attr(label="inputs", fontname=fontname)
111-
112-
# cluster of the output decls
113-
if outputs:
114-
with top.subgraph(name="cluster-outputs") as sg:
115-
for outp in workflow.outputs or []:
116-
assert outp.workflow_node_id.startswith("output-")
117-
sg.node(outp.workflow_node_id, outp.workflow_node_id[7:], shape="plaintext")
118-
node_ids.add(outp.workflow_node_id)
119-
sg.attr(label="outputs", fontname=fontname)
120-
121-
# add edge for each dependency between workflow nodes
122-
def add_edges(node):
123-
for dep_id in node.workflow_node_dependencies:
124-
dep = workflow.get_node(dep_id)
125-
# leave Gather nodes invisible by replacing any dependencies on them with their
126-
# final_referee
127-
if isinstance(dep, WDL.Tree.Gather):
128-
dep = dep.final_referee
129-
dep_id = dep.workflow_node_id
130-
if dep_id in node_ids and node.workflow_node_id in node_ids:
131-
lhead = None
132-
if isinstance(node, WDL.WorkflowSection):
133-
lhead = "cluster-" + node.workflow_node_id
134-
top.edge(dep_id, node.workflow_node_id, lhead=lhead)
135-
if isinstance(node, WDL.WorkflowSection):
136-
for child in node.body:
137-
add_edges(child)
138-
139-
for node in (workflow.inputs or []) + workflow.body + (workflow.outputs or []):
140-
add_edges(node)
141-
142-
return top
143-
144-
145-
async def read_source(uri, path, importer):
146-
"""
147-
This function helps miniwdl read the WDL source code directly from http[s] URIs.
148-
"""
149-
if uri.startswith("http:") or uri.startswith("https:"):
150-
fn = os.path.join(
151-
tempfile.mkdtemp(prefix="miniwdl_import_uri_"),
152-
os.path.basename(parse.urlsplit(uri).path),
153-
)
154-
request.urlretrieve(uri, filename=fn)
155-
with open(fn, "r") as infile:
156-
return WDL.ReadSourceResult(infile.read(), uri)
157-
elif importer and (
158-
importer.pos.abspath.startswith("http:") or importer.pos.abspath.startswith("https:")
159-
):
160-
assert not os.path.isabs(uri), "absolute import from downloaded WDL"
161-
return await read_source(parse.urljoin(importer.pos.abspath, uri), [], importer)
162-
return await WDL.read_source_default(uri, path, importer)
163-
164-
165-
if __name__ == "__main__":
166-
main()
8+
with open(__file__, "r") as file:
9+
tree = ast.parse(file.read())
10+
print(ast.get_docstring(tree))

0 commit comments

Comments
 (0)