Coverage for datakiln/build_dbms.py: 0%

56 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2Builds table metadata from DBMS. 

3""" 

4import logging 

5from getpass import getuser 

6from pathlib import Path 

7 

8from datakiln.metadata.database import DatabaseMeta, DRIVER 

9from datakiln.arguments import ( 

10 add_directory, add_export, add_include, add_log 

11) 

12from datakiln.extraction.models import ExtractMeta, ExtractGraph 

13from datakiln.extraction.strategies import ( 

14 StreamStrategy, TempTableStrategy, ParallelExecutor 

15) 

16from datakiln.extraction.exporters import MetadataExtractor 

17 

18 

19logger = logging.getLogger(__name__) 

20 

21 

22def add_arguments(parser): 

23 """Adds the options of the command. 

24 

25 :param parser: Parser of the command. 

26 :type parser: argparse.ArgumentParser 

27 """ 

28 add_directory(parser) 

29 add_log(parser) 

30 add_include(parser, "Extract only these tables, and the ones they " 

31 "need.") 

32 add_export(parser) 

33 parser.add_argument('--dialect', metavar="dialect", type=str, 

34 required=True, choices=DRIVER.keys(), 

35 help='Dialect of the source DBMS.') 

36 parser.add_argument('--host', metavar="host", type=str, 

37 required=True, help='Hostname of the source DBMS.') 

38 parser.add_argument('--port', metavar="port", type=str, 

39 required=True, 

40 help='Port to use when connecting to the source ' 

41 'DBMS.') 

42 parser.add_argument('--user', metavar="user", type=str, 

43 required=False, default=getuser(), 

44 help='Username to use when connecting to the source ' 

45 'DBMS.') 

46 parser.add_argument('--password', metavar="password", type=str, 

47 required=False, 

48 help='Password to use when connecting to the source ' 

49 'DBMS.') 

50 parser.add_argument('--database', metavar="database", type=str, 

51 required=True, 

52 help='Database to connect to in the source DBMS.') 

53 parser.add_argument('--read-only', action='store_true', default=False, 

54 help='Run in read-only mode (no temp tables).') 

55 parser.add_argument('--parallel', type=int, default=4, 

56 help='Number of parallel extraction jobs.') 

57 

58 

59def run(args): 

60 # pylint: disable=too-many-locals, too-many-branches, too-many-statements 

61 """Downloads data and creates sample metadata for tables. 

62 

63 :param args: Arguments of the command. 

64 :type args: argparse.Namespace 

65 """ 

66 directory = Path(args.directory).absolute() 

67 

68 # Create output directory 

69 meta_dir = Path(directory) / "metadata" 

70 meta_dir.mkdir(exist_ok=True) 

71 

72 # Connect parameters 

73 db_params = { 

74 "dialect": args.dialect, 

75 "user": args.user, 

76 "password": args.password, 

77 "host": args.host, 

78 "database": args.database, 

79 "port": args.port 

80 } 

81 

82 # Load all extraction metadata 

83 tasks = [] 

84 for file in sorted(directory.glob("extract/*.json")): 

85 tasks.append(ExtractMeta.from_json(file)) 

86 

87 # Build dependency graph of the tables to export, and what they need 

88 graph = ExtractGraph.select(tasks, args.include) 

89 

90 # Initialize execution strategy 

91 if args.read_only: 

92 strategy = StreamStrategy(db_params, directory) 

93 else: 

94 strategy = TempTableStrategy(db_params, directory) 

95 

96 # Track exported tables for metadata generation 

97 exported_schemas = {} 

98 for task in graph.sort(): 

99 if task in graph.export: 

100 if task.schema not in exported_schemas: 

101 exported_schemas[task.schema] = [] 

102 exported_schemas[task.schema].append(task.table) 

103 

104 if args.data: 

105 # Create data directory if not exists 

106 data_dir = directory / "data" 

107 data_dir.mkdir(exist_ok=True) 

108 

109 # Initialize and run parallel executor 

110 executor = ParallelExecutor(strategy, graph, max_workers=args.parallel) 

111 try: 

112 executor.execute(data_dir=data_dir) 

113 finally: 

114 strategy.cleanup() 

115 

116 # Skip metadata generation 

117 if not args.metadata: 

118 return 

119 

120 # Get target database params 

121 db_params = DatabaseMeta.load_env(directory) 

122 

123 # Generate table metadata 

124 with strategy.session() as session: 

125 for schema, tables in exported_schemas.items(): 

126 # Retrieve metadata from DBMS 

127 metadata = MetadataExtractor.get_metadata(session, schema, tables) 

128 

129 for _, table in metadata.tables.items(): 

130 # Generate table metadata 

131 logger.info("Table '%s': Generating metadata.", table) 

132 MetadataExtractor.generate_table_meta( 

133 meta_dir, 

134 table, 

135 db_params["dialect"] 

136 )