Coverage for datakiln/extraction/renderers.py: 0%
39 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2SQL rendering strategies for the extraction pipeline.
3"""
4from threading import Lock
5from typing import List, TYPE_CHECKING
6import itertools
8from sqlalchemy import (
9 select, Table, MetaData, literal_column, and_, or_
10)
11from sqlalchemy.sql import Select
12from sqlalchemy.sql.elements import ClauseElement
14from datakiln.metadata.database import Session
15from datakiln.extraction.models import ExtractMeta
17if TYPE_CHECKING:
18 from datakiln.extraction.strategies import ExecutionStrategy
21class SQLGenerator:
22 # pylint: disable=too-few-public-methods
23 """Base SQL generation utility providing column resolution and joins."""
24 _lock = Lock()
26 @staticmethod
27 def base_select(schema: str, table: str) -> Select:
28 """Return a base SELECT * for a given schema-qualified table.
30 :param schema: Database schema name.
31 :type schema: str
33 :param table: Table name.
34 :type table: str
36 :return: SA Select statement.
37 :rtype: Select
38 """
39 src = Table(table, MetaData(), schema=schema).alias("src")
40 return select(literal_column("*")).select_from(src)
43class QueryDirector:
44 # pylint: disable=too-few-public-methods
45 """Orchestrates query construction across strategies and filters."""
47 @staticmethod
48 # pylint: disable=too-many-locals
49 def build_queries(
50 session: Session,
51 task: ExtractMeta,
52 strategy: 'ExecutionStrategy'
53 ) -> Select:
54 """Build the runnable SA Select statement for a task.
56 :param session: Active SQLAlchemy session.
57 :type session: Session
59 :param task: Extraction task metadata.
60 :type task: ExtractMeta
62 :param strategy: Execution strategy instance.
63 :type strategy: ExecutionStrategy
65 :return: SA Select statement ready for execution.
66 :rtype: Select
67 """
68 filters = []
69 for filter_group in task.filters:
70 group_filters = []
72 for filter_config in filter_group:
73 group_filters.append(strategy.process_filter(
74 filter_config, session, task
75 ))
76 filters.append(group_filters)
78 # Build base statement
79 stmt = SQLGenerator.base_select(task.schema, task.table)
81 # If no filters, return the base statement
82 if not filters:
83 return stmt
85 # Build chunked queries
86 return QueryDirector._chunked_queries(
87 stmt,
88 groups=filters
89 )
91 @staticmethod
92 def _chunked_queries(
93 base_stmt: Select,
94 groups: List[List[List[ClauseElement]]]
95 ) -> Select:
96 """Decompose primary cohort sets into a chunked SA Select statement.
98 :param base_stmt: Base SA Select to extend with WHERE conditions.
99 :type base_stmt: Select
101 :param groups: A 3D list where the outermost list represents OR groups,
102 the middle list represents AND conditions within an OR
103 group, and the innermost list contains the chunked SA
104 ClauseElements for a specific condition.
105 :type groups: List[List[List[ClauseElement]]]
107 :return: SA Select statement representing all chunks.
108 :rtype: Select
109 """
110 or_clauses = []
112 for or_group in groups:
113 # Iterate over the cartesian product of all filter chunks
114 # in this OR group.
115 for chunk_combo in itertools.product(*or_group):
116 clauses = list(chunk_combo)
118 if not clauses:
119 continue
121 # Combine clauses since they are part of the same group
122 or_clauses.append(
123 and_(*clauses) if len(clauses) > 1 else clauses[0]
124 )
126 if not or_clauses:
127 return base_stmt
129 return base_stmt.where(
130 or_(*or_clauses) if len(or_clauses) > 1 else or_clauses[0]
131 )