Coverage for datakiln/build_dbms.py: 0%
56 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2Builds table metadata from DBMS.
3"""
4import logging
5from getpass import getuser
6from pathlib import Path
8from datakiln.metadata.database import DatabaseMeta, DRIVER
9from datakiln.arguments import (
10 add_directory, add_export, add_include, add_log
11)
12from datakiln.extraction.models import ExtractMeta, ExtractGraph
13from datakiln.extraction.strategies import (
14 StreamStrategy, TempTableStrategy, ParallelExecutor
15)
16from datakiln.extraction.exporters import MetadataExtractor
19logger = logging.getLogger(__name__)
22def add_arguments(parser):
23 """Adds the options of the command.
25 :param parser: Parser of the command.
26 :type parser: argparse.ArgumentParser
27 """
28 add_directory(parser)
29 add_log(parser)
30 add_include(parser, "Extract only these tables, and the ones they "
31 "need.")
32 add_export(parser)
33 parser.add_argument('--dialect', metavar="dialect", type=str,
34 required=True, choices=DRIVER.keys(),
35 help='Dialect of the source DBMS.')
36 parser.add_argument('--host', metavar="host", type=str,
37 required=True, help='Hostname of the source DBMS.')
38 parser.add_argument('--port', metavar="port", type=str,
39 required=True,
40 help='Port to use when connecting to the source '
41 'DBMS.')
42 parser.add_argument('--user', metavar="user", type=str,
43 required=False, default=getuser(),
44 help='Username to use when connecting to the source '
45 'DBMS.')
46 parser.add_argument('--password', metavar="password", type=str,
47 required=False,
48 help='Password to use when connecting to the source '
49 'DBMS.')
50 parser.add_argument('--database', metavar="database", type=str,
51 required=True,
52 help='Database to connect to in the source DBMS.')
53 parser.add_argument('--read-only', action='store_true', default=False,
54 help='Run in read-only mode (no temp tables).')
55 parser.add_argument('--parallel', type=int, default=4,
56 help='Number of parallel extraction jobs.')
59def run(args):
60 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
61 """Downloads data and creates sample metadata for tables.
63 :param args: Arguments of the command.
64 :type args: argparse.Namespace
65 """
66 directory = Path(args.directory).absolute()
68 # Create output directory
69 meta_dir = Path(directory) / "metadata"
70 meta_dir.mkdir(exist_ok=True)
72 # Connect parameters
73 db_params = {
74 "dialect": args.dialect,
75 "user": args.user,
76 "password": args.password,
77 "host": args.host,
78 "database": args.database,
79 "port": args.port
80 }
82 # Load all extraction metadata
83 tasks = []
84 for file in sorted(directory.glob("extract/*.json")):
85 tasks.append(ExtractMeta.from_json(file))
87 # Build dependency graph of the tables to export, and what they need
88 graph = ExtractGraph.select(tasks, args.include)
90 # Initialize execution strategy
91 if args.read_only:
92 strategy = StreamStrategy(db_params, directory)
93 else:
94 strategy = TempTableStrategy(db_params, directory)
96 # Track exported tables for metadata generation
97 exported_schemas = {}
98 for task in graph.sort():
99 if task in graph.export:
100 if task.schema not in exported_schemas:
101 exported_schemas[task.schema] = []
102 exported_schemas[task.schema].append(task.table)
104 if args.data:
105 # Create data directory if not exists
106 data_dir = directory / "data"
107 data_dir.mkdir(exist_ok=True)
109 # Initialize and run parallel executor
110 executor = ParallelExecutor(strategy, graph, max_workers=args.parallel)
111 try:
112 executor.execute(data_dir=data_dir)
113 finally:
114 strategy.cleanup()
116 # Skip metadata generation
117 if not args.metadata:
118 return
120 # Get target database params
121 db_params = DatabaseMeta.load_env(directory)
123 # Generate table metadata
124 with strategy.session() as session:
125 for schema, tables in exported_schemas.items():
126 # Retrieve metadata from DBMS
127 metadata = MetadataExtractor.get_metadata(session, schema, tables)
129 for _, table in metadata.tables.items():
130 # Generate table metadata
131 logger.info("Table '%s': Generating metadata.", table)
132 MetadataExtractor.generate_table_meta(
133 meta_dir,
134 table,
135 db_params["dialect"]
136 )