Coverage for datakiln/build_metadata.py: 0%
56 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2Generate metadata from data file.
3"""
4import json
5import logging
6from pathlib import Path
7from itertools import starmap
8from functools import partial
10import pandas as pd
11from slugify import slugify
13from datakiln.arguments import add_directory, add_log
16logger = logging.getLogger(__name__)
18SUPPORTED_EXTENSIONS = {
19 ".csv.gz": partial(pd.read_csv, engine="python", nrows=10000),
20 ".csv": partial(pd.read_csv, engine="python", nrows=10000),
21 ".xlsx": partial(pd.read_excel, sheet_name=None, nrows=10000)
22}
25def dtype_to_psql(dtype):
26 """Define PostgreSQL type from pandas inferred type.
28 :param dtype: Type inferred by pandas.
29 :type dtype: type
31 :return: PostgreSQL data type.
32 :rtype: str
33 """
34 if dtype == object:
35 return "TEXT"
36 if dtype == int:
37 return "INTEGER"
38 if dtype == float:
39 return "DOUBLE PRECISION"
40 if dtype == bool:
41 return "BOOLEAN"
42 return None
45def dtype_to_meta(column, dtype):
46 """Generates metadata for column from tuple.
48 :param column: Column name as it is in data file.
49 :type column: str
51 :param dtype: Type inferred by pandas.
52 :type dtype: type
54 :return: Column metadata.
55 :rtype: dict
56 """
57 return {
58 "name": column,
59 "rename": slugify(column, separator="_"),
60 "dtype": {
61 "postgresql": dtype_to_psql(dtype)
62 }
63 }
66def generate_table_meta(meta_dir, table_name, data):
67 """Generates metadata for column from tuple.
69 :param meta_dir: Medatada directory.
70 :type meta_dir: Path
72 :param table_name: Name of the table.
73 :type table_name: str
75 :param data: DataFrame containing sample data for the table.
76 :type data: pandas.DataFrame
78 :return: None
79 """
80 # Ensure table name is database friendly
81 table_name = slugify(table_name, separator="_")
83 # Remove trailing white spaces from data
84 data = data.map(lambda x: x.strip() if isinstance(x, str) else x)
86 # Generate metadata
87 meta = {
88 "name": table_name,
89 "schema": "public",
90 "encoding": "utf-8",
91 "columns": list(starmap(dtype_to_meta, data.dtypes.items())),
92 "constraints": {
93 "primary_key": ["meta_row_id"]
94 }
95 }
97 # Write metadata to file
98 meta_path = meta_dir / f"{table_name}.json"
99 with open(meta_path, "w", encoding="utf-8") as file:
100 json.dump(meta, file, indent=4)
103def add_arguments(parser):
104 """Adds the options of the command.
106 :param parser: Parser of the command.
107 :type parser: argparse.ArgumentParser
108 """
109 add_directory(parser)
110 add_log(parser)
111 parser.add_argument('--delimiter', metavar="delimiter", type=str,
112 required=False,
113 help='Character to treat as delimiter.')
116def run(args):
117 """Generates metadata from the data files.
119 :param args: Arguments of the command.
120 :type args: argparse.Namespace
121 """
123 # Get absolute path for specified directory
124 directory = Path(args.directory).absolute()
126 # Ensure metadata directory exists
127 meta_dir = directory / "metadata"
128 meta_dir.mkdir(exist_ok=True)
130 # Generate metadata for each data file
131 for data_path in directory.glob("data/*.*"):
132 # Skip if the extension is not supported
133 extension = "".join(data_path.suffixes).lower()
134 if extension not in SUPPORTED_EXTENSIONS:
135 continue
136 logger.info("File '%s': Generating metadata.", data_path)
138 # Read file
139 params = {}
140 reader = SUPPORTED_EXTENSIONS[extension]
141 if args.delimiter and reader.func == pd.read_csv:
142 params["delimiter"] = args.delimiter
143 data = reader(data_path.absolute(), **params)
145 # Excel files can have more than one sheet
146 # so we handle the data as a dictionary
147 table_name = data_path.name[:-len(extension)]
148 if isinstance(data, pd.DataFrame):
149 data = {table_name: data}
150 # Only consider tables with header row
151 # when multiple tables are returned
152 elif isinstance(data, dict):
153 data = {
154 name: df
155 for name, df in data.items()
156 if not df.columns.str.startswith("Unnamed:").all()
157 }
159 # If only one table is left, use file name
160 if len(data) == 1:
161 data = {data_path.stem: list(data.values())[0]}
163 # Generate metadata
164 for table_name, df in data.items():
165 generate_table_meta(meta_dir, table_name, df)