Coverage for datakiln/show_graph.py: 0%

88 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2Shows the dependency graph of the tables and derive scripts. 

3""" 

4from html import escape 

5from operator import attrgetter 

6from pathlib import Path 

7from typing import Iterable, List 

8 

9import networkx as nx 

10 

11from datakiln.arguments import add_directory, add_include 

12from datakiln.metadata.database import DatabaseMeta 

13from datakiln.metadata.graph import DatabaseGraph 

14from datakiln.metadata.table import TableMeta 

15 

16 

17TABLE_STYLE = "fill:#e8f1fb,stroke:#4a78b5,color:#1f3b5c" 

18SCRIPT_STYLE = "fill:#fdf1dc,stroke:#c98a2b,color:#5c3d0e" 

19 

20# Without a layout, Mermaid draws stepped arrows, and arrows that go the same 

21# way run along each other. The ELK layout places the nodes so that fewer 

22# arrows cross, and runs the arrows between them at right angles. 

23HEADER = "---\nconfig:\n layout: elk\n---\n" 

24 

25# Mermaid 12.2 draws the ELK layout without a plugin 

26MERMAID_JS = ( 

27 "https://cdn.jsdelivr.net/npm/mermaid@12.2.0/dist/mermaid.min.js") 

28PAGE = """<!DOCTYPE html> 

29<html lang="en"> 

30<head> 

31<meta charset="utf-8"> 

32<title>Dependency graph</title> 

33<style>body {{ font-family: sans-serif; margin: 24px; }}</style> 

34</head> 

35<body> 

36<pre class="mermaid"> 

37{graph}</pre> 

38<script src="{script}"></script> 

39<script>mermaid.initialize({{ startOnLoad: true }});</script> 

40</body> 

41</html> 

42""" 

43 

44 

45def is_schema(node) -> bool: 

46 """Whether the node is a script that creates a schema. 

47 

48 :param node: Table or script. 

49 :type node: datakiln.metadata.table.TableMeta | 

50 datakiln.metadata.script.ScriptMeta 

51 

52 :rtype: bool 

53 """ 

54 return "." not in node.full_name and "(" not in node.full_name 

55 

56 

57def label(node) -> List[str]: 

58 """Lines to tell a node by: its name, and the file of a script if the 

59 name does not tell it. 

60 

61 :param node: Table or script. 

62 :type node: datakiln.metadata.table.TableMeta | 

63 datakiln.metadata.script.ScriptMeta 

64 

65 :rtype: list[str] 

66 """ 

67 lines = [node.full_name] 

68 if not isinstance(node, TableMeta): 

69 if node.path.stem != node.full_name.rpartition(".")[2]: 

70 lines.append(str(node.path)) 

71 return lines 

72 

73 

74def edges(graph: DatabaseGraph, nodes: Iterable) -> List[tuple]: 

75 """Pairs of two tables or two scripts where the second one needs the 

76 first. There are none between a table and a script. 

77 

78 An arrow is left out if the nodes are also linked through other nodes. If 

79 c needs b and b needs a, then c needs a through b, so the pairs are (a, b) 

80 and (b, c) but not (a, c). 

81 

82 :param graph: Dependency graph. 

83 :type graph: datakiln.metadata.graph.DatabaseGraph 

84 

85 :param nodes: Nodes that can be shown. 

86 :type nodes: Iterable 

87 

88 :rtype: list[tuple] 

89 """ 

90 candidates = set(nodes) 

91 shown = nx.DiGraph() 

92 for node in candidates: 

93 for required in graph.dependencies(node): 

94 if required in candidates and ( 

95 isinstance(required, TableMeta) == isinstance(node, TableMeta) 

96 ): 

97 shown.add_edge(required, node) 

98 

99 # Leave out the arrows already given by a longer path, as (a, c) is by 

100 # (a, b) and (b, c). Only the nodes that are shown are in the graph, so 

101 # if b is not shown, (a, c) stays 

102 pairs = list(nx.transitive_reduction(shown).edges) 

103 pairs.sort(key=lambda pair: (pair[1].full_name, pair[0].full_name)) 

104 return pairs 

105 

106 

107def select(graph: DatabaseGraph, include: Iterable[str] = None, 

108 schemas: bool = False) -> List: 

109 """Nodes to show, ordered by name: the ones that have an arrow, as the 

110 others would only be a row of boxes, and the ones asked for. 

111 

112 :param graph: Dependency graph. 

113 :type graph: datakiln.metadata.graph.DatabaseGraph 

114 

115 :param include: Names of the nodes to show, with the nodes they need. All 

116 by default. 

117 :type include: Iterable[str] 

118 

119 :param schemas: Whether to show the scripts that create schemas, which 

120 every script of the schema needs. 

121 :type schemas: bool 

122 

123 :raises ValueError: If any of the names is not a node. 

124 

125 :rtype: list 

126 """ 

127 nodes = set(graph.nodes.values()) 

128 asked = set() 

129 if include: 

130 asked = graph.find(include) 

131 nodes = graph.requirements(asked) 

132 

133 candidates = set() 

134 for node in nodes: 

135 if schemas or not is_schema(node): 

136 candidates.add(node) 

137 

138 connected = asked & candidates 

139 for required, node in edges(graph, candidates): 

140 connected.update((required, node)) 

141 return sorted(connected, key=attrgetter("full_name")) 

142 

143 

144def mermaid(graph: DatabaseGraph, nodes: List, direction: str = "LR") -> str: 

145 """Graph as a Mermaid flowchart, which GitLab and GitHub draw in a 

146 Markdown file. 

147 

148 :param graph: Dependency graph. 

149 :type graph: datakiln.metadata.graph.DatabaseGraph 

150 

151 :param nodes: Nodes shown, in order. 

152 :type nodes: list 

153 

154 :param direction: Direction of the flow, LR or TB. 

155 :type direction: str 

156 

157 :rtype: str 

158 """ 

159 ids = {} 

160 lines = [HEADER + f"flowchart {direction}"] 

161 for node in nodes: 

162 ids[node] = f"n{len(ids)}" 

163 text = "<br/>".join(label(node)).replace('"', "#quot;") 

164 if isinstance(node, TableMeta): 

165 lines.append(f' {ids[node]}[("{text}")]') 

166 else: 

167 lines.append(f' {ids[node]}["{text}"]') 

168 

169 for required, node in edges(graph, nodes): 

170 lines.append(f" {ids[required]} --> {ids[node]}") 

171 

172 lines.append(f" classDef table {TABLE_STYLE}") 

173 lines.append(f" classDef script {SCRIPT_STYLE}") 

174 for kind, style in ((True, "table"), (False, "script")): 

175 members = [ids[n] for n in nodes if isinstance(n, TableMeta) == kind] 

176 if members: 

177 lines.append(f" class {','.join(members)} {style}") 

178 return "\n".join(lines) + "\n" 

179 

180 

181def markdown(graph: DatabaseGraph, nodes: List, direction: str = "LR") -> str: 

182 """Graph as a Markdown document, which GitLab and GitHub draw when it is 

183 opened. 

184 

185 :param graph: Dependency graph. 

186 :type graph: datakiln.metadata.graph.DatabaseGraph 

187 

188 :param nodes: Nodes shown, in order. 

189 :type nodes: list 

190 

191 :param direction: Direction of the flow, LR or TB. 

192 :type direction: str 

193 

194 :rtype: str 

195 """ 

196 text = mermaid(graph, nodes, direction) 

197 return f"# Dependency graph\n\n```mermaid\n{text}```\n" 

198 

199 

200def html(graph: DatabaseGraph, nodes: List, direction: str = "LR") -> str: 

201 """Graph as a web page, which any browser draws. The page loads Mermaid 

202 from the internet. 

203 

204 :param graph: Dependency graph. 

205 :type graph: datakiln.metadata.graph.DatabaseGraph 

206 

207 :param nodes: Nodes shown, in order. 

208 :type nodes: list 

209 

210 :param direction: Direction of the flow, LR or TB. 

211 :type direction: str 

212 

213 :rtype: str 

214 """ 

215 text = escape(mermaid(graph, nodes, direction), quote=False) 

216 return PAGE.format(graph=text, script=MERMAID_JS) 

217 

218 

219# The writer of each extension of the output file, and Mermaid for the others 

220FORMATS = {".md": markdown, ".html": html} 

221 

222 

223def add_arguments(parser): 

224 """Adds the options of the command. 

225 

226 :param parser: Parser of the command. 

227 :type parser: argparse.ArgumentParser 

228 """ 

229 add_directory(parser) 

230 add_include(parser, "Show only these tables and scripts, and what " 

231 "they need.") 

232 parser.add_argument("--direction", choices=["LR", "TB"], default="LR", 

233 help="Direction of the flow: left to right, or top " 

234 "to bottom.") 

235 parser.add_argument("--schemas", action="store_true", 

236 help="Show the scripts that create schemas, and " 

237 "the arrows from them.") 

238 parser.add_argument("--output", metavar="file", type=str, 

239 help="File to write the graph to, instead of " 

240 "standard output. Its extension tells the " 

241 "format: .md is a Markdown document, which " 

242 "GitHub and GitLab draw when it is opened, " 

243 "and .html is a web page, which a browser " 

244 "draws. Anything else is Mermaid text.") 

245 

246 

247def run(args): 

248 """Shows the dependency graph of the tables and scripts. 

249 

250 :param args: Arguments of the command. 

251 :type args: argparse.Namespace 

252 """ 

253 

254 directory = Path(args.directory).absolute() 

255 name = DatabaseMeta.load_env(directory, required=False)["database"] 

256 graph = DatabaseMeta(directory, name).graph 

257 graph.sort() 

258 

259 nodes = select(graph, args.include, args.schemas) 

260 write = FORMATS.get(Path(args.output or "").suffix, mermaid) 

261 text = write(graph, nodes, args.direction) 

262 if args.output: 

263 Path(args.output).write_text(text, encoding="utf-8") 

264 else: 

265 print(text, end="")