Coverage for datakiln/show_graph.py: 0%
88 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2Shows the dependency graph of the tables and derive scripts.
3"""
4from html import escape
5from operator import attrgetter
6from pathlib import Path
7from typing import Iterable, List
9import networkx as nx
11from datakiln.arguments import add_directory, add_include
12from datakiln.metadata.database import DatabaseMeta
13from datakiln.metadata.graph import DatabaseGraph
14from datakiln.metadata.table import TableMeta
17TABLE_STYLE = "fill:#e8f1fb,stroke:#4a78b5,color:#1f3b5c"
18SCRIPT_STYLE = "fill:#fdf1dc,stroke:#c98a2b,color:#5c3d0e"
20# Without a layout, Mermaid draws stepped arrows, and arrows that go the same
21# way run along each other. The ELK layout places the nodes so that fewer
22# arrows cross, and runs the arrows between them at right angles.
23HEADER = "---\nconfig:\n layout: elk\n---\n"
25# Mermaid 12.2 draws the ELK layout without a plugin
26MERMAID_JS = (
27 "https://cdn.jsdelivr.net/npm/mermaid@12.2.0/dist/mermaid.min.js")
28PAGE = """<!DOCTYPE html>
29<html lang="en">
30<head>
31<meta charset="utf-8">
32<title>Dependency graph</title>
33<style>body {{ font-family: sans-serif; margin: 24px; }}</style>
34</head>
35<body>
36<pre class="mermaid">
37{graph}</pre>
38<script src="{script}"></script>
39<script>mermaid.initialize({{ startOnLoad: true }});</script>
40</body>
41</html>
42"""
45def is_schema(node) -> bool:
46 """Whether the node is a script that creates a schema.
48 :param node: Table or script.
49 :type node: datakiln.metadata.table.TableMeta |
50 datakiln.metadata.script.ScriptMeta
52 :rtype: bool
53 """
54 return "." not in node.full_name and "(" not in node.full_name
57def label(node) -> List[str]:
58 """Lines to tell a node by: its name, and the file of a script if the
59 name does not tell it.
61 :param node: Table or script.
62 :type node: datakiln.metadata.table.TableMeta |
63 datakiln.metadata.script.ScriptMeta
65 :rtype: list[str]
66 """
67 lines = [node.full_name]
68 if not isinstance(node, TableMeta):
69 if node.path.stem != node.full_name.rpartition(".")[2]:
70 lines.append(str(node.path))
71 return lines
74def edges(graph: DatabaseGraph, nodes: Iterable) -> List[tuple]:
75 """Pairs of two tables or two scripts where the second one needs the
76 first. There are none between a table and a script.
78 An arrow is left out if the nodes are also linked through other nodes. If
79 c needs b and b needs a, then c needs a through b, so the pairs are (a, b)
80 and (b, c) but not (a, c).
82 :param graph: Dependency graph.
83 :type graph: datakiln.metadata.graph.DatabaseGraph
85 :param nodes: Nodes that can be shown.
86 :type nodes: Iterable
88 :rtype: list[tuple]
89 """
90 candidates = set(nodes)
91 shown = nx.DiGraph()
92 for node in candidates:
93 for required in graph.dependencies(node):
94 if required in candidates and (
95 isinstance(required, TableMeta) == isinstance(node, TableMeta)
96 ):
97 shown.add_edge(required, node)
99 # Leave out the arrows already given by a longer path, as (a, c) is by
100 # (a, b) and (b, c). Only the nodes that are shown are in the graph, so
101 # if b is not shown, (a, c) stays
102 pairs = list(nx.transitive_reduction(shown).edges)
103 pairs.sort(key=lambda pair: (pair[1].full_name, pair[0].full_name))
104 return pairs
107def select(graph: DatabaseGraph, include: Iterable[str] = None,
108 schemas: bool = False) -> List:
109 """Nodes to show, ordered by name: the ones that have an arrow, as the
110 others would only be a row of boxes, and the ones asked for.
112 :param graph: Dependency graph.
113 :type graph: datakiln.metadata.graph.DatabaseGraph
115 :param include: Names of the nodes to show, with the nodes they need. All
116 by default.
117 :type include: Iterable[str]
119 :param schemas: Whether to show the scripts that create schemas, which
120 every script of the schema needs.
121 :type schemas: bool
123 :raises ValueError: If any of the names is not a node.
125 :rtype: list
126 """
127 nodes = set(graph.nodes.values())
128 asked = set()
129 if include:
130 asked = graph.find(include)
131 nodes = graph.requirements(asked)
133 candidates = set()
134 for node in nodes:
135 if schemas or not is_schema(node):
136 candidates.add(node)
138 connected = asked & candidates
139 for required, node in edges(graph, candidates):
140 connected.update((required, node))
141 return sorted(connected, key=attrgetter("full_name"))
144def mermaid(graph: DatabaseGraph, nodes: List, direction: str = "LR") -> str:
145 """Graph as a Mermaid flowchart, which GitLab and GitHub draw in a
146 Markdown file.
148 :param graph: Dependency graph.
149 :type graph: datakiln.metadata.graph.DatabaseGraph
151 :param nodes: Nodes shown, in order.
152 :type nodes: list
154 :param direction: Direction of the flow, LR or TB.
155 :type direction: str
157 :rtype: str
158 """
159 ids = {}
160 lines = [HEADER + f"flowchart {direction}"]
161 for node in nodes:
162 ids[node] = f"n{len(ids)}"
163 text = "<br/>".join(label(node)).replace('"', "#quot;")
164 if isinstance(node, TableMeta):
165 lines.append(f' {ids[node]}[("{text}")]')
166 else:
167 lines.append(f' {ids[node]}["{text}"]')
169 for required, node in edges(graph, nodes):
170 lines.append(f" {ids[required]} --> {ids[node]}")
172 lines.append(f" classDef table {TABLE_STYLE}")
173 lines.append(f" classDef script {SCRIPT_STYLE}")
174 for kind, style in ((True, "table"), (False, "script")):
175 members = [ids[n] for n in nodes if isinstance(n, TableMeta) == kind]
176 if members:
177 lines.append(f" class {','.join(members)} {style}")
178 return "\n".join(lines) + "\n"
181def markdown(graph: DatabaseGraph, nodes: List, direction: str = "LR") -> str:
182 """Graph as a Markdown document, which GitLab and GitHub draw when it is
183 opened.
185 :param graph: Dependency graph.
186 :type graph: datakiln.metadata.graph.DatabaseGraph
188 :param nodes: Nodes shown, in order.
189 :type nodes: list
191 :param direction: Direction of the flow, LR or TB.
192 :type direction: str
194 :rtype: str
195 """
196 text = mermaid(graph, nodes, direction)
197 return f"# Dependency graph\n\n```mermaid\n{text}```\n"
200def html(graph: DatabaseGraph, nodes: List, direction: str = "LR") -> str:
201 """Graph as a web page, which any browser draws. The page loads Mermaid
202 from the internet.
204 :param graph: Dependency graph.
205 :type graph: datakiln.metadata.graph.DatabaseGraph
207 :param nodes: Nodes shown, in order.
208 :type nodes: list
210 :param direction: Direction of the flow, LR or TB.
211 :type direction: str
213 :rtype: str
214 """
215 text = escape(mermaid(graph, nodes, direction), quote=False)
216 return PAGE.format(graph=text, script=MERMAID_JS)
219# The writer of each extension of the output file, and Mermaid for the others
220FORMATS = {".md": markdown, ".html": html}
223def add_arguments(parser):
224 """Adds the options of the command.
226 :param parser: Parser of the command.
227 :type parser: argparse.ArgumentParser
228 """
229 add_directory(parser)
230 add_include(parser, "Show only these tables and scripts, and what "
231 "they need.")
232 parser.add_argument("--direction", choices=["LR", "TB"], default="LR",
233 help="Direction of the flow: left to right, or top "
234 "to bottom.")
235 parser.add_argument("--schemas", action="store_true",
236 help="Show the scripts that create schemas, and "
237 "the arrows from them.")
238 parser.add_argument("--output", metavar="file", type=str,
239 help="File to write the graph to, instead of "
240 "standard output. Its extension tells the "
241 "format: .md is a Markdown document, which "
242 "GitHub and GitLab draw when it is opened, "
243 "and .html is a web page, which a browser "
244 "draws. Anything else is Mermaid text.")
247def run(args):
248 """Shows the dependency graph of the tables and scripts.
250 :param args: Arguments of the command.
251 :type args: argparse.Namespace
252 """
254 directory = Path(args.directory).absolute()
255 name = DatabaseMeta.load_env(directory, required=False)["database"]
256 graph = DatabaseMeta(directory, name).graph
257 graph.sort()
259 nodes = select(graph, args.include, args.schemas)
260 write = FORMATS.get(Path(args.output or "").suffix, mermaid)
261 text = write(graph, nodes, args.direction)
262 if args.output:
263 Path(args.output).write_text(text, encoding="utf-8")
264 else:
265 print(text, end="")