Coverage for datakiln/build_database.py: 0%
51 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2Creates table and loads data.
3"""
4import logging
5from pathlib import Path
7from datakiln.arguments import add_directory, add_include, add_log
8from datakiln.metadata.database import DatabaseMeta, Session
11logger = logging.getLogger(__name__)
14def add_arguments(parser):
15 """Adds the options of the command.
17 :param parser: Parser of the command.
18 :type parser: argparse.ArgumentParser
19 """
20 add_directory(parser)
21 add_log(parser)
22 add_include(parser, "Build only these tables and scripts, and "
23 "what they need.")
24 parser.add_argument("--grant", metavar="grant", type=str, nargs='+',
25 help="Grant read permissions to specified users.")
26 parser.add_argument("--rebuild-all", action="store_true",
27 help="Rebuild all tables and scripts, even if "
28 "up to date.")
29 parser.add_argument("--prune", action="store_true",
30 help="Drop the objects removed from the repo.")
33def run(args):
34 """Creates the tables, loads the data and runs the scripts.
36 :param args: Arguments of the command.
37 :type args: argparse.Namespace
38 """
40 # Get absolute of specified path
41 directory = Path(args.directory).absolute()
43 # Get database parameters
44 db_params = DatabaseMeta.load_env(directory)
46 # Tables and scripts to build, with what they use
47 database = DatabaseMeta(directory, db_params["database"])
48 include = None
49 if args.include:
50 selected = database.graph.find(args.include)
51 include = database.graph.requirements(selected)
53 # Connect to database
54 session = Session.create(**db_params)
56 # Create metadata
57 database.create_metadata(session)
58 database.load_stale(session)
60 # Create role for database read access
61 database.create_role(session)
63 # Stage and flush to database
64 updated = []
65 for table in database.graph.tables:
66 if include is not None and table not in include:
67 continue
69 # Skip the tables that did not change, unless all are rebuilt. A table
70 # is also rebuilt if a table it references is, since that drops its
71 # foreign keys
72 if (
73 not args.rebuild_all
74 and not table.needs_update(session)
75 and database.graph.dependencies(table).isdisjoint(updated)
76 ):
77 logger.info("Table '%s'.'%s': Already up to date.",
78 table.schema, table.name)
79 continue
80 table.pull()
81 table.create(session)
82 table.stage_meta.load(session)
83 updated.append(table)
85 # Run transforms
86 database.execute_directory("transform", session, max_statements=0)
88 # Load to final schema and flush to database
89 for table in updated:
90 table.resolve_columns(session)
91 table.create_uniques(session)
92 table.load(session)
93 table.enforce_fks(session)
94 table.create_indexes(session)
95 table.create_metadata(session)
96 database.forget_dependents(session, [table])
97 session.commit()
99 # Run views
100 session.close()
101 database.execute_derive(
102 session, force=args.rebuild_all, include=include)
104 # Objects whose metadata or script was removed
105 database.prune(session, drop=args.prune)
107 # Grant read permissions on entire database
108 database.grant(session, args.grant)
109 session.commit()
110 session.close()