Coverage for datakiln/extraction/strategies.py: 0%

449 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2Execution strategies and parallel orchestration for the extraction pipeline. 

3""" 

4import json 

5import logging 

6import hashlib 

7import threading 

8import signal 

9import os 

10from itertools import chain 

11from pathlib import Path 

12from textwrap import dedent 

13from abc import ABC, abstractmethod 

14from contextlib import contextmanager 

15from functools import cached_property 

16from threading import RLock 

17from typing import List, Dict, Optional, Iterator, Any, Tuple, Union 

18from concurrent.futures import ThreadPoolExecutor, as_completed 

19 

20import pandas as pd 

21from sqlalchemy import ( 

22 MetaData, Table, Column, select, literal, and_, text, 

23 values, column as sa_column, bindparam, String, Float, 

24 inspect, Index 

25) 

26from sqlalchemy.types import UserDefinedType 

27from sqlalchemy.schema import DropTable 

28from sqlalchemy.exc import SQLAlchemyError 

29from sqlalchemy.sql import Select 

30from sqlalchemy.ext.compiler import compiles 

31from sqlalchemy.sql.selectable import Values 

32 

33from datakiln.metadata.database import DatabaseMeta, Session 

34from datakiln.metadata.graph import Unresolved 

35from datakiln.extraction.models import ( 

36 UUID, ExtractMeta, ExtractGraph, FilterConfig 

37) 

38from datakiln.extraction.renderers import QueryDirector 

39from datakiln.extraction.exporters import MetadataExtractor, DataExporter 

40 

41logger = logging.getLogger(__name__) 

42 

43 

44@compiles(Float, "oracle") 

45def compile_oracle_float(type_, compiler, **kwargs): 

46 # pylint: disable=unused-argument 

47 """Compile generic SQLAlchemy Float for Oracle dialect. 

48 

49 Pandas to_sql maps float64 columns to Float(precision=53). 

50 Oracle FLOAT types use binary precision instead of decimal precision, 

51 causing SQLAlchemy 2.0+ to raise an ArgumentError. Defaulting to 

52 plain 'FLOAT' allows Oracle to use its default binary precision (126). 

53 """ 

54 return "FLOAT" 

55 

56 

57class _OracleCLOB(UserDefinedType): # pylint: disable=abstract-method 

58 """Minimal SQLAlchemy type that binds a Python str as Oracle DB_TYPE_CLOB. 

59 

60 Text() and oracle.CLOB() both resolve to DB_TYPE_NVARCHAR via 

61 setinputsizes, which still has a size limit. This type returns 

62 DB_TYPE_CLOB directly so oracledb sends the value as a true LOB, 

63 allowing the JSON seed list to exceed Oracle's VARCHAR2 4000-byte 

64 cap without ORA-01460. 

65 """ 

66 cache_ok = True 

67 

68 def get_col_spec(self, **kwargs): # pylint: disable=unused-argument 

69 """ 

70 Get the column specification for Oracle. 

71 

72 :param kwargs: Keyword arguments. 

73 :type kwargs: dict 

74 

75 :return: Column specification. 

76 :rtype: str 

77 """ 

78 return "CLOB" 

79 

80 def get_dbapi_type(self, dbapi): 

81 """ 

82 Get the database API type for Oracle. 

83 

84 :param dbapi: Database API. 

85 :type dbapi: oracledb 

86 

87 :return: Database API type. 

88 :rtype: oracledb.DB_TYPE_CLOB 

89 """ 

90 return dbapi.DB_TYPE_CLOB 

91 

92 

93@compiles(Values, "oracle") 

94def compile_oracle_values(element, compiler, **kwargs): 

95 """Translate a Values clause to Oracle JSON_TABLE query.""" 

96 columns = element.columns 

97 raw_data = element._data[0] if element._data else [] # pylint: disable=protected-access # noqa: E501 

98 json_str = json.dumps(raw_data) 

99 

100 bind_name = f"seed_list_{id(element)}" 

101 bind_param = bindparam(bind_name, value=json_str, type_=_OracleCLOB()) 

102 bind_placeholder = compiler.process(bind_param, **kwargs) 

103 

104 columns_spec = [] 

105 for col_idx, col in enumerate(columns): 

106 if isinstance(col.type, String) and not col.type.length: 

107 col_type = "VARCHAR2(4000)" 

108 else: 

109 col_type = compiler.dialect.type_compiler.process(col.type) 

110 columns_spec.append(f"{col.name} {col_type} PATH '$[{col_idx}]'") 

111 

112 return dedent(f""" 

113 ( 

114 SELECT jt.* 

115 FROM JSON_TABLE({bind_placeholder}, '$[*]' 

116 COLUMNS ({', '.join(columns_spec)})) jt 

117 ) {compiler.preparer.quote(element.name)} 

118 """).strip() 

119 

120 

121class HashingReader: 

122 # pylint: disable=too-few-public-methods 

123 """File-like wrapper that calculates MD5 hash on the fly while reading.""" 

124 

125 def __init__(self, fp): 

126 self.fp = fp 

127 self.hasher = hashlib.md5() # noqa: DUO130 

128 

129 def read(self, size=-1): 

130 """Read data and update hasher.""" 

131 chunk = self.fp.read(size) 

132 self.hasher.update(chunk) 

133 return chunk 

134 

135 def seek(self, offset, whence=0): 

136 """Delegate seek to the underlying file pointer.""" 

137 return self.fp.seek(offset, whence) 

138 

139 def tell(self): 

140 """Delegate tell to the underlying file pointer.""" 

141 return self.fp.tell() 

142 

143 

144# pylint: disable=too-many-instance-attributes 

145class ExecutionStrategy(ABC): 

146 """Abstract base class for extraction strategies.""" 

147 

148 def __init__(self, db_params: Dict, directory: Path): 

149 self.db_params = db_params 

150 self.directory = directory 

151 self.cohorts = {} 

152 self.created_tables = set() 

153 self.lock = threading.Lock() 

154 self._local = threading.local() 

155 self.csv_hashes = {} 

156 self.graph = None 

157 self.abort = threading.Event() 

158 

159 @cached_property 

160 def engine(self): 

161 """Returns the shared database engine.""" 

162 with self.session(bind=False) as session: 

163 return session.get_bind() 

164 

165 @contextmanager 

166 def session(self, bind: Optional[bool] = True) -> Iterator[Session]: 

167 """Returns a managed database session. 

168 

169 If a session is already active for this thread, it is yielded. 

170 Otherwise, a new pooled session is created and registered as active. 

171 

172 :param bind: Whether to use the shared engine. If False, a new 

173 engine is created using Session.create(). 

174 """ 

175 if not bind: 

176 with Session.create(**self.db_params) as session: 

177 yield session 

178 return 

179 

180 # Return thread-local session if active 

181 if hasattr(self._local, "session"): 

182 yield self._local.session 

183 return 

184 

185 # Create a new managed session for this thread 

186 with Session(bind=self.engine, autoflush=True) as session: 

187 self._local.session = session 

188 try: 

189 yield session 

190 finally: 

191 if hasattr(self._local, "session"): 

192 del self._local.session 

193 

194 def build_queries(self, task: ExtractMeta) -> Select: 

195 """Build the extraction SA Select statement. 

196 

197 :param task: Extraction task metadata. 

198 :type task: ExtractMeta 

199 

200 :return: SA Select statement. 

201 :rtype: Select 

202 """ 

203 with self.session() as session: 

204 return QueryDirector.build_queries(session, task, strategy=self) 

205 

206 @property 

207 def is_read_only(self) -> bool: 

208 """Returns whether the strategy is read-only.""" 

209 return False 

210 

211 @abstractmethod 

212 def process_filter( 

213 self, 

214 filter_config: FilterConfig, 

215 session: Session, 

216 task: ExtractMeta 

217 ) -> List[Any]: 

218 """Determine how to process a filter for this strategy.""" 

219 return [filter_config.to_exists(self)] 

220 

221 @abstractmethod 

222 def execute(self, task: ExtractMeta, export_to: Optional[Path] = None): 

223 """Execute the extraction task.""" 

224 

225 @abstractmethod 

226 def get_cohort(self, cohort: str) -> str: 

227 """Retrieve or create a cohort representation.""" 

228 

229 @abstractmethod 

230 def drop_cohort(self, cohort: str): 

231 """Drop a specific cohort table.""" 

232 

233 @property 

234 def tmp_dir(self) -> Path: 

235 """Folder of the files that are not exported.""" 

236 return self.directory / "data" / ".tmp" 

237 

238 def drop(self, name: str): 

239 """Drop the temporary data of the task with this temporary name, 

240 which is its file.""" 

241 table_name = name[4:-(len(UUID) + 1)] 

242 (self.tmp_dir / f"{table_name}.csv.gz").unlink(missing_ok=True) 

243 self.created_tables.discard(name) 

244 

245 def cleanup(self): 

246 """Drop all remaining temporary data and cohorts, and the folder of 

247 the files that are not exported.""" 

248 for name in list(self.created_tables): 

249 self.drop(name) 

250 self.drop_cohorts() 

251 

252 try: 

253 self.tmp_dir.rmdir() 

254 except OSError: 

255 pass 

256 

257 def drop_cohorts(self): 

258 """Drop all remaining cohort tables.""" 

259 for cohort in list(self.cohorts.keys()): 

260 self.drop_cohort(cohort) 

261 

262 

263class TempTableStrategy(ExecutionStrategy): 

264 """Strategy that uses temporary tables (requires write permissions).""" 

265 

266 def has_index( 

267 self, 

268 session: Session, 

269 schema: Optional[str], 

270 table: str, 

271 columns: List[str] 

272 ) -> bool: 

273 """Evaluate the existence of an index with the specified columns.""" 

274 insp = inspect(session.bind) 

275 for idx in insp.get_indexes(table, schema=schema): 

276 if idx["column_names"] == columns: 

277 return True 

278 return False 

279 

280 def ensure_index( 

281 self, 

282 session: Session, 

283 table_parts: List[str], 

284 columns: List[str] 

285 ): 

286 """Create a covering index on the reference table if non-existent.""" 

287 if not columns: 

288 return 

289 

290 columns_hash = hashlib.md5("_".join(columns).encode()).hexdigest()[:8] # noqa: DUO130, E501 

291 schema = table_parts[0] if len(table_parts) == 2 else None 

292 tbl_name = table_parts[-1] 

293 

294 with self.lock: 

295 if not self.has_index(session, schema, tbl_name, columns): 

296 try: 

297 name = f"{tbl_name[:49]}_idx_{columns_hash}" 

298 sa_table = Table( 

299 tbl_name, MetaData(), schema=schema, 

300 autoload_with=session.bind 

301 ) 

302 sa_cols = [sa_table.c[c] for c in columns] 

303 idx = Index(name, *sa_cols) 

304 idx.create(bind=session.bind) 

305 except SQLAlchemyError as err: 

306 logger.warning( 

307 "Table '%s': Failed to create index (%s).", 

308 ".".join(table_parts), err 

309 ) 

310 

311 def process_filter( 

312 self, 

313 filter_config: FilterConfig, 

314 session: Session, 

315 task: ExtractMeta 

316 ) -> List[Any]: 

317 if filter_config.resolved_cohort is not None: 

318 actual_table = self.cohorts.get( 

319 filter_config.resolved_cohort, 

320 filter_config.resolved_cohort 

321 ).split(".") 

322 self.ensure_index( 

323 session, 

324 actual_table, 

325 filter_config.cohort_columns 

326 ) 

327 return super().process_filter(filter_config, session, task) 

328 

329 def execute(self, task: ExtractMeta, export_to: Optional[Path] = None): 

330 # pylint: disable=too-many-locals 

331 logger.info("Table '%s': Extracting.", task.full_name) 

332 

333 with self.session() as session: 

334 # TempTableStrategy is never read-only, but multiple OR groups 

335 # will still produce multiple statements. 

336 query = self.build_queries(task) 

337 temp_table = task.temp_name 

338 

339 # Reflect source table to get column names and types 

340 source_table = Table( 

341 task.table, 

342 MetaData(), 

343 schema=task.schema, 

344 autoload_with=session.bind 

345 ) 

346 

347 # Initialize destination table 

348 meta = MetaData() 

349 dest_table = Table(temp_table, meta) 

350 

351 # Add columns to destination table 

352 col_names = [] 

353 for col in source_table.columns: 

354 dest_table.append_column(Column(col.name, col.type)) 

355 col_names.append(col.name) 

356 

357 # Create destination table 

358 self.created_tables.add(temp_table) 

359 self.cohorts[task.full_name] = temp_table 

360 dest_table.create(session.bind) 

361 

362 # Fallback to .tmp dir if export_to is not specified 

363 target_dir = export_to or self.tmp_dir 

364 target_dir.mkdir(parents=True, exist_ok=True) 

365 export_path = target_dir / f"{task.table}.csv.gz" 

366 

367 # Write to CSV and inserting simultaneously, deleting the file if 

368 # it is not complete 

369 try: 

370 self.csv_hashes[task.full_name] = DataExporter.stream_to_csv( 

371 session, 

372 query, 

373 export_path, 

374 table_name=task.table, 

375 insert_to=dest_table, 

376 chunk_size=task.chunk_size, 

377 abort=self.abort 

378 ) 

379 except BaseException: 

380 export_path.unlink(missing_ok=True) 

381 raise 

382 if self.abort.is_set(): 

383 export_path.unlink(missing_ok=True) 

384 

385 def drop(self, name: str): 

386 """Drop a temporary table, and its file.""" 

387 super().drop(name) 

388 with self.session() as session: 

389 try: 

390 sa_table = Table(name, MetaData()) 

391 session.execute(DropTable(sa_table, if_exists=True)) 

392 session.commit() 

393 except SQLAlchemyError as err: 

394 logger.warning("Table '%s': Could not drop (%s).", name, err) 

395 

396 # pylint: disable=too-many-locals 

397 def get_cohort(self, cohort: str) -> str: 

398 """Retrieve or create a cohort table.""" 

399 if cohort in self.cohorts: 

400 return self.cohorts[cohort] 

401 

402 logger.info("Cohort '%s': Generating data.", cohort) 

403 

404 name = cohort.replace(".sql", "").replace(".", "_") 

405 name = f'tmp_{name}_{UUID}' 

406 

407 if not cohort.endswith(".sql"): 

408 db_params = DatabaseMeta.load_env(self.directory) 

409 source_engine = Session.create(**db_params).get_bind() 

410 schema, table = cohort.split(".") 

411 

412 # Initialize source table 

413 source_meta = MetaData() 

414 source_table = Table( 

415 table, 

416 source_meta, 

417 schema=schema, 

418 autoload_with=source_engine 

419 ) 

420 

421 dest_columns = [] 

422 dest_engine = self.engine 

423 for column in source_table.columns: 

424 # Get column metadata 

425 dialect = dest_engine.dialect.name 

426 col_meta = MetadataExtractor.generate_column_meta( 

427 column, dialect=dialect, compile_type=False 

428 ) 

429 dest_columns.append(Column( 

430 col_meta["name"], col_meta["dtype"][dialect] 

431 )) 

432 

433 # Create destination table 

434 dest_meta = MetaData() 

435 dest_table = Table(name, dest_meta, *dest_columns) 

436 self.cohorts[cohort] = name 

437 dest_table.create(dest_engine) 

438 

439 # Move data from source to destination 

440 cohort_data = pd.read_sql_table(table, source_engine, schema) 

441 cohort_data.to_sql( 

442 name, dest_engine, index=False, if_exists='append' 

443 ) 

444 

445 return name 

446 

447 # Load cohort query 

448 cohort_path = self.directory / 'extract' / cohort 

449 with open(cohort_path, encoding="utf-8") as file: 

450 query = text(file.read().strip()) 

451 

452 # Write cohort data to temp table 

453 self.cohorts[cohort] = name 

454 self.created_tables.add(name) 

455 with self.session() as session: 

456 cohort_data = pd.read_sql_query(query, session.bind) 

457 cohort_data.to_sql( 

458 name, session.bind, index=False, if_exists='fail' 

459 ) 

460 

461 return name 

462 

463 def drop_cohort(self, cohort: str): 

464 """Drop a specific cohort table.""" 

465 if cohort in self.cohorts: 

466 logger.info("Cohort '%s': Dropping data.", cohort) 

467 try: 

468 with self.session() as session: 

469 sa_table = Table(self.cohorts[cohort], MetaData()) 

470 session.execute(DropTable(sa_table, if_exists=True)) 

471 session.commit() 

472 except SQLAlchemyError as err: 

473 logger.warning( 

474 "Cohort '%s': Could not drop (%s).", cohort, err) 

475 finally: 

476 del self.cohorts[cohort] 

477 

478 

479class JoinChain: 

480 """Join chain for filtering on read-only mode.""" 

481 

482 def __init__(self, filter_config: 'FilterConfig', engine): 

483 self._engine = engine 

484 self._cohort_cols = filter_config.cohort_columns 

485 self._cohort_alias = filter_config.reference.alias 

486 self._target_cols = [] 

487 for c in filter_config.reference.columns: 

488 self._target_cols.append(text(c)) 

489 self._cohort_types = {} 

490 self._seed_joins = [] 

491 self._where_clauses = [] 

492 self._from_clause = None 

493 

494 @classmethod 

495 def build( 

496 cls, 

497 filter_config: 'FilterConfig', 

498 engine, 

499 task: 'ExtractMeta' 

500 ) -> "JoinChain": 

501 """Build and return a fully resolved chain for *filter_config*. 

502 

503 :param filter_config: Filter configuration describing the join chain. 

504 :type filter_config: FilterConfig 

505 

506 :param engine: SQLAlchemy engine used to reflect remote tables. 

507 

508 :param task: Extraction task metadata. 

509 :type task: ExtractMeta 

510 

511 :return: Populated :class:`JoinChain` ready to execute seed chunks. 

512 :rtype: JoinChain 

513 """ 

514 # Initialize join chain 

515 chain_ = cls(filter_config, engine) 

516 

517 # Create aliased source table for the chain 

518 src = Table( 

519 task.table, 

520 MetaData(), 

521 schema=task.schema, 

522 autoload_with=engine 

523 ).alias("src") 

524 chain_._from_clause = src 

525 

526 # Resolve column types and seed joins from the filter config 

527 for name in filter_config.cohort_columns: 

528 col = src.c.get(name, src.c.get(name.upper())) 

529 if col is None: 

530 continue 

531 chain_._cohort_types[name] = col.type 

532 chain_._seed_joins.append((col, name)) 

533 

534 # Extend join chain with subsequent joins 

535 for j_conf in filter_config.join: 

536 chain_._extend(j_conf) 

537 

538 return chain_ 

539 

540 def _extend(self, j_conf): 

541 """Reflect one join config and extend the SA chain.""" 

542 ref = j_conf.reference 

543 sa_table = Table( 

544 ref.table, MetaData(), schema=ref.schema, 

545 autoload_with=self._engine, 

546 ).alias(ref.alias) 

547 

548 # Retrieve column types and seed joins from the filter config 

549 for pair in j_conf.seed_pairs(): 

550 col = sa_table.c[pair.remote] 

551 self._cohort_types[pair.local] = col.type 

552 self._seed_joins.append((col, pair.local)) 

553 

554 # Retrieve constant filters from the filter config 

555 for p in j_conf.const_filters(): 

556 where = sa_table.c[p.remote] == literal(p.value) 

557 self._where_clauses.append(where) 

558 

559 # Build the FROM clause for the join chain 

560 if self._from_clause is None: 

561 self._from_clause = sa_table 

562 else: 

563 on_clauses = [] 

564 for p in j_conf.cross_pairs(): 

565 on_clauses.append(text(f"{p.left} = {p.right}")) 

566 onclause = and_(*on_clauses) if on_clauses else text("1=1") 

567 self._from_clause = self._from_clause.join(sa_table, onclause) 

568 

569 def execute(self, chunk: List, session) -> List: 

570 """Execute a seed chunk against the built chain. 

571 

572 :param chunk: Batch of seed tuples to look up. 

573 :type chunk: list 

574 

575 :param session: Active SQLAlchemy session. 

576 :type session: Session 

577 

578 :return: All matching rows from the remote tables. 

579 :rtype: list 

580 """ 

581 # Create list of column objects for VALUES clause 

582 val_cols = [] 

583 for name in self._cohort_cols: 

584 column = sa_column(name, self._cohort_types.get(name, String())) 

585 val_cols.append(column) 

586 v = values(*val_cols, name=self._cohort_alias).data(chunk) 

587 

588 # Build join ON clauses from seed pairs 

589 seed_on = [] 

590 for col, name in self._seed_joins: 

591 seed_on.append(col == v.c[name]) 

592 

593 # Build the full statement 

594 stmt = select(*self._target_cols).select_from( 

595 self._from_clause.join(v, and_(*seed_on)) 

596 ).distinct() 

597 if self._where_clauses: 

598 stmt = stmt.where(and_(*self._where_clauses)) 

599 return session.execute( 

600 stmt, 

601 execution_options={ 

602 "skip_formatting": True 

603 } 

604 ).all() 

605 

606 

607class StreamStrategy(ExecutionStrategy): 

608 """Strategy that streams data directly from source (read-only).""" 

609 

610 @property 

611 def is_read_only(self) -> bool: 

612 return True 

613 

614 def process_filter( 

615 self, 

616 filter_config: FilterConfig, 

617 session: Session, 

618 task: ExtractMeta 

619 ) -> List[Any]: 

620 if filter_config.resolved_cohort and not filter_config.is_sql_cohort: 

621 cohort_data = self.get_cohort_set(filter_config, task=task) 

622 return [ 

623 filter_config.to_where(cohort_data[i:i + task.chunk_size]) 

624 for i in range(0, max(1, len(cohort_data)), task.chunk_size) 

625 ] 

626 return super().process_filter(filter_config, session, task) 

627 

628 def execute(self, task: ExtractMeta, export_to: Optional[Path] = None): 

629 """Execute extraction by streaming and filtering in memory.""" 

630 target_dir = export_to or self.tmp_dir 

631 target_dir.mkdir(parents=True, exist_ok=True) 

632 

633 logger.info("Table '%s': Extracting.", task.full_name) 

634 

635 # The file is known before it is written, to delete it if it is not 

636 # complete 

637 if export_to is None: 

638 self.created_tables.add(task.temp_name) 

639 

640 with self.session() as session: 

641 # Export to a CSV file 

642 query = self.build_queries(task) 

643 export_path = target_dir / f"{task.table}.csv.gz" 

644 try: 

645 self.csv_hashes[task.full_name] = DataExporter.stream_to_csv( 

646 session, 

647 query, 

648 export_path, 

649 table_name=task.table, 

650 chunk_size=task.chunk_size, 

651 abort=self.abort 

652 ) 

653 except BaseException: 

654 export_path.unlink(missing_ok=True) 

655 raise 

656 if self.abort.is_set(): 

657 export_path.unlink(missing_ok=True) 

658 

659 def drop_cohort(self, cohort: str): 

660 """Clear cohort from memory (no DDL in stream mode).""" 

661 if cohort in self.cohorts: 

662 logger.info("Cohort '%s': Dropping data.", cohort) 

663 del self.cohorts[cohort] 

664 

665 def get_cohort(self, cohort: str) -> Any: 

666 """Register the cohort but do not load data yet.""" 

667 if cohort in self.cohorts: 

668 return self.cohorts[cohort] 

669 

670 if cohort.endswith(".sql"): 

671 cohort_path = self.directory / 'extract' / cohort 

672 name = cohort.replace(".sql", "") 

673 with open(cohort_path, encoding="utf-8") as file: 

674 query = file.read().strip() 

675 subquery = f"(\n {query}\n) {name}" 

676 self.cohorts[cohort] = subquery 

677 return subquery 

678 

679 self.cohorts[cohort] = cohort 

680 return cohort 

681 

682 def get_cohort_set( 

683 self, 

684 filter_config: FilterConfig, 

685 task: 'ExtractMeta' 

686 ) -> List[Tuple[str, ...]]: 

687 """Load cohort data into memory. 

688 

689 :param filter_config: Filter configuration. 

690 :type filter_config: FilterConfig 

691 

692 :param task: Extraction task metadata. 

693 :type task: ExtractMeta 

694 

695 :return: List of cohort records. 

696 :rtype: list[tuple[str, ...]] 

697 """ 

698 seed_chunks = self._fetch_local_seeds( 

699 filter_config, 

700 chunk_size=task.chunk_size 

701 ) 

702 

703 # If no joins, no need to query the database 

704 if not filter_config.join: 

705 return list(dict.fromkeys(chain.from_iterable(seed_chunks))) 

706 

707 chain_obj = JoinChain.build(filter_config, self.engine, task=task) 

708 final_tuples = set() 

709 for chunk in seed_chunks: 

710 with self.session() as session: 

711 for row in chain_obj.execute(chunk, session): 

712 str_vals = [] 

713 for val in row: 

714 str_vals.append(str(val)) 

715 final_tuples.add(tuple(str_vals)) 

716 return list(final_tuples) 

717 

718 def _fetch_local_seeds( 

719 self, 

720 filter_config: FilterConfig, 

721 chunk_size: int = 10000 

722 ) -> Iterator[List[Tuple[str, ...]]]: 

723 # pylint: disable=too-many-locals 

724 """Fetch the initial set of IDs from the local cohort table or CSV.""" 

725 fetch_cols = filter_config.cohort_columns 

726 cohort = filter_config.resolved_cohort 

727 

728 # Check if the cohort is an extraction task in the graph 

729 if self.graph and cohort in self.graph.tasks: 

730 # Determine the file path 

731 table = self.graph.tasks[cohort].table 

732 temp_name = f"tmp_{table}_{UUID}" 

733 if temp_name in self.created_tables: 

734 csv_path = self.directory / "data" / ".tmp" / f"{table}.csv.gz" 

735 else: 

736 csv_path = self.directory / "data" / f"{table}.csv.gz" 

737 

738 if not csv_path.exists(): 

739 raise FileNotFoundError(f"Missing file for '{cohort}'") 

740 

741 # Read and hash the compressed file on-the-fly 

742 with open(csv_path, 'rb') as f: 

743 reader = HashingReader(f) 

744 chunks = pd.read_csv( 

745 reader, 

746 compression='gzip', 

747 usecols=fetch_cols, 

748 dtype=str, 

749 chunksize=chunk_size 

750 ) 

751 

752 # Load seeds in chunks 

753 for chunk in chunks: 

754 chunk = chunk[fetch_cols] 

755 unique = chunk.drop_duplicates() 

756 yield list(unique.itertuples(index=False, name=None)) 

757 

758 # Validate hash to prevent corrupted reads 

759 if reader.hasher.hexdigest() != self.csv_hashes[cohort]: 

760 raise ValueError(f"Hash mismatch for file '{csv_path.name}'") 

761 

762 else: 

763 # Fallback to local cohort database query 

764 db_params = DatabaseMeta.load_env(self.directory) 

765 with Session.create(**db_params) as session: 

766 schema, table_name = cohort.split(".") 

767 sa_table = Table( 

768 table_name, 

769 MetaData(), 

770 schema=schema, 

771 autoload_with=session.bind 

772 ) 

773 cols = [sa_table.c[c] for c in fetch_cols] 

774 stmt = select(*cols).distinct() 

775 res = session.execute(stmt) 

776 for partition in res.partitions(chunk_size): 

777 yield [tuple(str(v) for v in row) for row in partition] 

778 

779 

780class ParallelExecutor: 

781 # pylint: disable=too-few-public-methods,too-many-instance-attributes 

782 """Orchestrates parallel execution with reference counting for cleanup.""" 

783 

784 def __init__( 

785 self, 

786 strategy: 'ExecutionStrategy', 

787 graph: 'ExtractGraph', 

788 max_workers: int = 4 

789 ): 

790 self.strategy = strategy 

791 self.graph = graph 

792 self.strategy.graph = graph 

793 self.max_workers = max_workers 

794 self.lock = RLock() 

795 

796 # Tracking dependencies 

797 self.ref_counts = dict(graph.digraph.out_degree()) 

798 

799 # graphlib topological sorter for parallel orchestration 

800 self.ts = graph.topological_sorter 

801 self.live_tasks = set() 

802 

803 # Register signal handlers for clean abort 

804 self._register_signal_handlers() 

805 

806 def _register_signal_handlers(self): 

807 """Register signal handlers to guarantee cleanup on abort.""" 

808 def _handler(signum, _frame): 

809 logger.error("Pipeline: Received signal %s, aborting.", signum) 

810 self.strategy.abort.set() 

811 self.strategy.cleanup() 

812 os._exit(1) 

813 

814 # Only register if running in the main thread 

815 if threading.current_thread() is threading.main_thread(): 

816 signal.signal(signal.SIGINT, _handler) 

817 signal.signal(signal.SIGTERM, _handler) 

818 

819 def execute( 

820 self, 

821 data_dir: Path 

822 ): 

823 """Run tasks in parallel using TopologicalSorter interactive mode. 

824 

825 :param data_dir: Directory of the data of the tasks to export. 

826 :type data_dir: Path 

827 """ 

828 with ThreadPoolExecutor(max_workers=self.max_workers) as executor: 

829 futures = {} 

830 

831 while self.ts.is_active(): 

832 # Submit all ready tasks 

833 for node in self.ts.get_ready(): 

834 task = executor.submit( 

835 self._run_task, 

836 node, 

837 data_dir 

838 ) 

839 futures[task] = node 

840 

841 if not futures: 

842 # This shouldn't happen if is_active is true, 

843 # unless there's a deadlock or we're waiting for futures. 

844 if any(f.running() for f in futures): 

845 pass 

846 else: 

847 break 

848 

849 # Wait for at least one task to complete 

850 for future in as_completed(futures): 

851 node = futures.pop(future) 

852 try: 

853 future.result() 

854 with self.lock: 

855 self.ts.done(node) 

856 # Trigger cleanup and get new ready tasks 

857 self._on_task_complete(node) 

858 break # Go back to the loop to submit new ready tasks 

859 except (SQLAlchemyError, OSError) as e: 

860 logger.error( 

861 "Task '%s': Failed (%s).", node.full_name, e) 

862 self.strategy.cleanup() 

863 os._exit(1) 

864 

865 def _run_task( 

866 self, 

867 node: Union[ExtractMeta, Unresolved], 

868 data_dir: Path 

869 ): 

870 """Execute a task (table extraction or cohort initialization).""" 

871 # Start a managed session for this task 

872 with self.strategy.session(): 

873 # Table extraction task 

874 if isinstance(node, ExtractMeta): 

875 export_to = data_dir if node in self.graph.export else None 

876 

877 with self.lock: 

878 self.live_tasks.add(node) 

879 

880 self.strategy.execute(node, export_to=export_to) 

881 # Cohort node 

882 else: 

883 self.strategy.get_cohort(node.full_name) 

884 

885 def _on_task_complete(self, node: Union[ExtractMeta, Unresolved]): 

886 """Perform cleanup for dependencies no longer needed.""" 

887 to_drop_cohorts = [] 

888 to_drop = [] 

889 

890 with self.lock: 

891 # Cohorts currently have no prerequisites in the graph 

892 for dep in self.graph.dependencies(node): 

893 # Decrement ref counts of dependencies used by THIS node 

894 self.ref_counts[dep] -= 1 

895 

896 # If no other nodes depend on this dependency, drop it 

897 if self.ref_counts[dep] == 0: 

898 if dep.full_name in self.strategy.cohorts: 

899 to_drop_cohorts.append(dep.full_name) 

900 elif dep in self.live_tasks: 

901 to_drop.append(dep.temp_name) 

902 self.live_tasks.remove(dep) 

903 

904 # Check if THIS node itself is no longer needed by anyone else 

905 if self.ref_counts.get(node, 0) == 0: 

906 if isinstance(node, ExtractMeta): 

907 to_drop.append(node.temp_name) 

908 self.live_tasks.discard(node) 

909 elif node.full_name in self.strategy.cohorts: 

910 to_drop_cohorts.append(node.full_name) 

911 

912 # Execute drops outside of the lock 

913 for dep in to_drop_cohorts: 

914 self.strategy.drop_cohort(dep) 

915 for name in to_drop: 

916 self.strategy.drop(name)