Coverage for datakiln/extraction/renderers.py: 0%

39 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2SQL rendering strategies for the extraction pipeline. 

3""" 

4from threading import Lock 

5from typing import List, TYPE_CHECKING 

6import itertools 

7 

8from sqlalchemy import ( 

9 select, Table, MetaData, literal_column, and_, or_ 

10) 

11from sqlalchemy.sql import Select 

12from sqlalchemy.sql.elements import ClauseElement 

13 

14from datakiln.metadata.database import Session 

15from datakiln.extraction.models import ExtractMeta 

16 

17if TYPE_CHECKING: 

18 from datakiln.extraction.strategies import ExecutionStrategy 

19 

20 

21class SQLGenerator: 

22 # pylint: disable=too-few-public-methods 

23 """Base SQL generation utility providing column resolution and joins.""" 

24 _lock = Lock() 

25 

26 @staticmethod 

27 def base_select(schema: str, table: str) -> Select: 

28 """Return a base SELECT * for a given schema-qualified table. 

29 

30 :param schema: Database schema name. 

31 :type schema: str 

32 

33 :param table: Table name. 

34 :type table: str 

35 

36 :return: SA Select statement. 

37 :rtype: Select 

38 """ 

39 src = Table(table, MetaData(), schema=schema).alias("src") 

40 return select(literal_column("*")).select_from(src) 

41 

42 

43class QueryDirector: 

44 # pylint: disable=too-few-public-methods 

45 """Orchestrates query construction across strategies and filters.""" 

46 

47 @staticmethod 

48 # pylint: disable=too-many-locals 

49 def build_queries( 

50 session: Session, 

51 task: ExtractMeta, 

52 strategy: 'ExecutionStrategy' 

53 ) -> Select: 

54 """Build the runnable SA Select statement for a task. 

55 

56 :param session: Active SQLAlchemy session. 

57 :type session: Session 

58 

59 :param task: Extraction task metadata. 

60 :type task: ExtractMeta 

61 

62 :param strategy: Execution strategy instance. 

63 :type strategy: ExecutionStrategy 

64 

65 :return: SA Select statement ready for execution. 

66 :rtype: Select 

67 """ 

68 filters = [] 

69 for filter_group in task.filters: 

70 group_filters = [] 

71 

72 for filter_config in filter_group: 

73 group_filters.append(strategy.process_filter( 

74 filter_config, session, task 

75 )) 

76 filters.append(group_filters) 

77 

78 # Build base statement 

79 stmt = SQLGenerator.base_select(task.schema, task.table) 

80 

81 # If no filters, return the base statement 

82 if not filters: 

83 return stmt 

84 

85 # Build chunked queries 

86 return QueryDirector._chunked_queries( 

87 stmt, 

88 groups=filters 

89 ) 

90 

91 @staticmethod 

92 def _chunked_queries( 

93 base_stmt: Select, 

94 groups: List[List[List[ClauseElement]]] 

95 ) -> Select: 

96 """Decompose primary cohort sets into a chunked SA Select statement. 

97 

98 :param base_stmt: Base SA Select to extend with WHERE conditions. 

99 :type base_stmt: Select 

100 

101 :param groups: A 3D list where the outermost list represents OR groups, 

102 the middle list represents AND conditions within an OR 

103 group, and the innermost list contains the chunked SA 

104 ClauseElements for a specific condition. 

105 :type groups: List[List[List[ClauseElement]]] 

106 

107 :return: SA Select statement representing all chunks. 

108 :rtype: Select 

109 """ 

110 or_clauses = [] 

111 

112 for or_group in groups: 

113 # Iterate over the cartesian product of all filter chunks 

114 # in this OR group. 

115 for chunk_combo in itertools.product(*or_group): 

116 clauses = list(chunk_combo) 

117 

118 if not clauses: 

119 continue 

120 

121 # Combine clauses since they are part of the same group 

122 or_clauses.append( 

123 and_(*clauses) if len(clauses) > 1 else clauses[0] 

124 ) 

125 

126 if not or_clauses: 

127 return base_stmt 

128 

129 return base_stmt.where( 

130 or_(*or_clauses) if len(or_clauses) > 1 else or_clauses[0] 

131 )