Coverage for datakiln/build_metadata.py: 0%

56 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2Generate metadata from data file. 

3""" 

4import json 

5import logging 

6from pathlib import Path 

7from itertools import starmap 

8from functools import partial 

9 

10import pandas as pd 

11from slugify import slugify 

12 

13from datakiln.arguments import add_directory, add_log 

14 

15 

16logger = logging.getLogger(__name__) 

17 

18SUPPORTED_EXTENSIONS = { 

19 ".csv.gz": partial(pd.read_csv, engine="python", nrows=10000), 

20 ".csv": partial(pd.read_csv, engine="python", nrows=10000), 

21 ".xlsx": partial(pd.read_excel, sheet_name=None, nrows=10000) 

22} 

23 

24 

25def dtype_to_psql(dtype): 

26 """Define PostgreSQL type from pandas inferred type. 

27 

28 :param dtype: Type inferred by pandas. 

29 :type dtype: type 

30 

31 :return: PostgreSQL data type. 

32 :rtype: str 

33 """ 

34 if dtype == object: 

35 return "TEXT" 

36 if dtype == int: 

37 return "INTEGER" 

38 if dtype == float: 

39 return "DOUBLE PRECISION" 

40 if dtype == bool: 

41 return "BOOLEAN" 

42 return None 

43 

44 

45def dtype_to_meta(column, dtype): 

46 """Generates metadata for column from tuple. 

47 

48 :param column: Column name as it is in data file. 

49 :type column: str 

50 

51 :param dtype: Type inferred by pandas. 

52 :type dtype: type 

53 

54 :return: Column metadata. 

55 :rtype: dict 

56 """ 

57 return { 

58 "name": column, 

59 "rename": slugify(column, separator="_"), 

60 "dtype": { 

61 "postgresql": dtype_to_psql(dtype) 

62 } 

63 } 

64 

65 

66def generate_table_meta(meta_dir, table_name, data): 

67 """Generates metadata for column from tuple. 

68 

69 :param meta_dir: Medatada directory. 

70 :type meta_dir: Path 

71 

72 :param table_name: Name of the table. 

73 :type table_name: str 

74 

75 :param data: DataFrame containing sample data for the table. 

76 :type data: pandas.DataFrame 

77 

78 :return: None 

79 """ 

80 # Ensure table name is database friendly 

81 table_name = slugify(table_name, separator="_") 

82 

83 # Remove trailing white spaces from data 

84 data = data.map(lambda x: x.strip() if isinstance(x, str) else x) 

85 

86 # Generate metadata 

87 meta = { 

88 "name": table_name, 

89 "schema": "public", 

90 "encoding": "utf-8", 

91 "columns": list(starmap(dtype_to_meta, data.dtypes.items())), 

92 "constraints": { 

93 "primary_key": ["meta_row_id"] 

94 } 

95 } 

96 

97 # Write metadata to file 

98 meta_path = meta_dir / f"{table_name}.json" 

99 with open(meta_path, "w", encoding="utf-8") as file: 

100 json.dump(meta, file, indent=4) 

101 

102 

103def add_arguments(parser): 

104 """Adds the options of the command. 

105 

106 :param parser: Parser of the command. 

107 :type parser: argparse.ArgumentParser 

108 """ 

109 add_directory(parser) 

110 add_log(parser) 

111 parser.add_argument('--delimiter', metavar="delimiter", type=str, 

112 required=False, 

113 help='Character to treat as delimiter.') 

114 

115 

116def run(args): 

117 """Generates metadata from the data files. 

118 

119 :param args: Arguments of the command. 

120 :type args: argparse.Namespace 

121 """ 

122 

123 # Get absolute path for specified directory 

124 directory = Path(args.directory).absolute() 

125 

126 # Ensure metadata directory exists 

127 meta_dir = directory / "metadata" 

128 meta_dir.mkdir(exist_ok=True) 

129 

130 # Generate metadata for each data file 

131 for data_path in directory.glob("data/*.*"): 

132 # Skip if the extension is not supported 

133 extension = "".join(data_path.suffixes).lower() 

134 if extension not in SUPPORTED_EXTENSIONS: 

135 continue 

136 logger.info("File '%s': Generating metadata.", data_path) 

137 

138 # Read file 

139 params = {} 

140 reader = SUPPORTED_EXTENSIONS[extension] 

141 if args.delimiter and reader.func == pd.read_csv: 

142 params["delimiter"] = args.delimiter 

143 data = reader(data_path.absolute(), **params) 

144 

145 # Excel files can have more than one sheet 

146 # so we handle the data as a dictionary 

147 table_name = data_path.name[:-len(extension)] 

148 if isinstance(data, pd.DataFrame): 

149 data = {table_name: data} 

150 # Only consider tables with header row 

151 # when multiple tables are returned 

152 elif isinstance(data, dict): 

153 data = { 

154 name: df 

155 for name, df in data.items() 

156 if not df.columns.str.startswith("Unnamed:").all() 

157 } 

158 

159 # If only one table is left, use file name 

160 if len(data) == 1: 

161 data = {data_path.stem: list(data.values())[0]} 

162 

163 # Generate metadata 

164 for table_name, df in data.items(): 

165 generate_table_meta(meta_dir, table_name, df)