Coverage for datakiln/build_database.py: 0%

51 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2Creates table and loads data. 

3""" 

4import logging 

5from pathlib import Path 

6 

7from datakiln.arguments import add_directory, add_include, add_log 

8from datakiln.metadata.database import DatabaseMeta, Session 

9 

10 

11logger = logging.getLogger(__name__) 

12 

13 

14def add_arguments(parser): 

15 """Adds the options of the command. 

16 

17 :param parser: Parser of the command. 

18 :type parser: argparse.ArgumentParser 

19 """ 

20 add_directory(parser) 

21 add_log(parser) 

22 add_include(parser, "Build only these tables and scripts, and " 

23 "what they need.") 

24 parser.add_argument("--grant", metavar="grant", type=str, nargs='+', 

25 help="Grant read permissions to specified users.") 

26 parser.add_argument("--rebuild-all", action="store_true", 

27 help="Rebuild all tables and scripts, even if " 

28 "up to date.") 

29 parser.add_argument("--prune", action="store_true", 

30 help="Drop the objects removed from the repo.") 

31 

32 

33def run(args): 

34 """Creates the tables, loads the data and runs the scripts. 

35 

36 :param args: Arguments of the command. 

37 :type args: argparse.Namespace 

38 """ 

39 

40 # Get absolute of specified path 

41 directory = Path(args.directory).absolute() 

42 

43 # Get database parameters 

44 db_params = DatabaseMeta.load_env(directory) 

45 

46 # Tables and scripts to build, with what they use 

47 database = DatabaseMeta(directory, db_params["database"]) 

48 include = None 

49 if args.include: 

50 selected = database.graph.find(args.include) 

51 include = database.graph.requirements(selected) 

52 

53 # Connect to database 

54 session = Session.create(**db_params) 

55 

56 # Create metadata 

57 database.create_metadata(session) 

58 database.load_stale(session) 

59 

60 # Create role for database read access 

61 database.create_role(session) 

62 

63 # Stage and flush to database 

64 updated = [] 

65 for table in database.graph.tables: 

66 if include is not None and table not in include: 

67 continue 

68 

69 # Skip the tables that did not change, unless all are rebuilt. A table 

70 # is also rebuilt if a table it references is, since that drops its 

71 # foreign keys 

72 if ( 

73 not args.rebuild_all 

74 and not table.needs_update(session) 

75 and database.graph.dependencies(table).isdisjoint(updated) 

76 ): 

77 logger.info("Table '%s'.'%s': Already up to date.", 

78 table.schema, table.name) 

79 continue 

80 table.pull() 

81 table.create(session) 

82 table.stage_meta.load(session) 

83 updated.append(table) 

84 

85 # Run transforms 

86 database.execute_directory("transform", session, max_statements=0) 

87 

88 # Load to final schema and flush to database 

89 for table in updated: 

90 table.resolve_columns(session) 

91 table.create_uniques(session) 

92 table.load(session) 

93 table.enforce_fks(session) 

94 table.create_indexes(session) 

95 table.create_metadata(session) 

96 database.forget_dependents(session, [table]) 

97 session.commit() 

98 

99 # Run views 

100 session.close() 

101 database.execute_derive( 

102 session, force=args.rebuild_all, include=include) 

103 

104 # Objects whose metadata or script was removed 

105 database.prune(session, drop=args.prune) 

106 

107 # Grant read permissions on entire database 

108 database.grant(session, args.grant) 

109 session.commit() 

110 session.close()