Coverage for datakiln/extraction/strategies.py: 0%
449 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-16 23:23 +0000
1"""
2Execution strategies and parallel orchestration for the extraction pipeline.
3"""
4import json
5import logging
6import hashlib
7import threading
8import signal
9import os
10from itertools import chain
11from pathlib import Path
12from textwrap import dedent
13from abc import ABC, abstractmethod
14from contextlib import contextmanager
15from functools import cached_property
16from threading import RLock
17from typing import List, Dict, Optional, Iterator, Any, Tuple, Union
18from concurrent.futures import ThreadPoolExecutor, as_completed
20import pandas as pd
21from sqlalchemy import (
22 MetaData, Table, Column, select, literal, and_, text,
23 values, column as sa_column, bindparam, String, Float,
24 inspect, Index
25)
26from sqlalchemy.types import UserDefinedType
27from sqlalchemy.schema import DropTable
28from sqlalchemy.exc import SQLAlchemyError
29from sqlalchemy.sql import Select
30from sqlalchemy.ext.compiler import compiles
31from sqlalchemy.sql.selectable import Values
33from datakiln.metadata.database import DatabaseMeta, Session
34from datakiln.metadata.graph import Unresolved
35from datakiln.extraction.models import (
36 UUID, ExtractMeta, ExtractGraph, FilterConfig
37)
38from datakiln.extraction.renderers import QueryDirector
39from datakiln.extraction.exporters import MetadataExtractor, DataExporter
41logger = logging.getLogger(__name__)
44@compiles(Float, "oracle")
45def compile_oracle_float(type_, compiler, **kwargs):
46 # pylint: disable=unused-argument
47 """Compile generic SQLAlchemy Float for Oracle dialect.
49 Pandas to_sql maps float64 columns to Float(precision=53).
50 Oracle FLOAT types use binary precision instead of decimal precision,
51 causing SQLAlchemy 2.0+ to raise an ArgumentError. Defaulting to
52 plain 'FLOAT' allows Oracle to use its default binary precision (126).
53 """
54 return "FLOAT"
57class _OracleCLOB(UserDefinedType): # pylint: disable=abstract-method
58 """Minimal SQLAlchemy type that binds a Python str as Oracle DB_TYPE_CLOB.
60 Text() and oracle.CLOB() both resolve to DB_TYPE_NVARCHAR via
61 setinputsizes, which still has a size limit. This type returns
62 DB_TYPE_CLOB directly so oracledb sends the value as a true LOB,
63 allowing the JSON seed list to exceed Oracle's VARCHAR2 4000-byte
64 cap without ORA-01460.
65 """
66 cache_ok = True
68 def get_col_spec(self, **kwargs): # pylint: disable=unused-argument
69 """
70 Get the column specification for Oracle.
72 :param kwargs: Keyword arguments.
73 :type kwargs: dict
75 :return: Column specification.
76 :rtype: str
77 """
78 return "CLOB"
80 def get_dbapi_type(self, dbapi):
81 """
82 Get the database API type for Oracle.
84 :param dbapi: Database API.
85 :type dbapi: oracledb
87 :return: Database API type.
88 :rtype: oracledb.DB_TYPE_CLOB
89 """
90 return dbapi.DB_TYPE_CLOB
93@compiles(Values, "oracle")
94def compile_oracle_values(element, compiler, **kwargs):
95 """Translate a Values clause to Oracle JSON_TABLE query."""
96 columns = element.columns
97 raw_data = element._data[0] if element._data else [] # pylint: disable=protected-access # noqa: E501
98 json_str = json.dumps(raw_data)
100 bind_name = f"seed_list_{id(element)}"
101 bind_param = bindparam(bind_name, value=json_str, type_=_OracleCLOB())
102 bind_placeholder = compiler.process(bind_param, **kwargs)
104 columns_spec = []
105 for col_idx, col in enumerate(columns):
106 if isinstance(col.type, String) and not col.type.length:
107 col_type = "VARCHAR2(4000)"
108 else:
109 col_type = compiler.dialect.type_compiler.process(col.type)
110 columns_spec.append(f"{col.name} {col_type} PATH '$[{col_idx}]'")
112 return dedent(f"""
113 (
114 SELECT jt.*
115 FROM JSON_TABLE({bind_placeholder}, '$[*]'
116 COLUMNS ({', '.join(columns_spec)})) jt
117 ) {compiler.preparer.quote(element.name)}
118 """).strip()
121class HashingReader:
122 # pylint: disable=too-few-public-methods
123 """File-like wrapper that calculates MD5 hash on the fly while reading."""
125 def __init__(self, fp):
126 self.fp = fp
127 self.hasher = hashlib.md5() # noqa: DUO130
129 def read(self, size=-1):
130 """Read data and update hasher."""
131 chunk = self.fp.read(size)
132 self.hasher.update(chunk)
133 return chunk
135 def seek(self, offset, whence=0):
136 """Delegate seek to the underlying file pointer."""
137 return self.fp.seek(offset, whence)
139 def tell(self):
140 """Delegate tell to the underlying file pointer."""
141 return self.fp.tell()
144# pylint: disable=too-many-instance-attributes
145class ExecutionStrategy(ABC):
146 """Abstract base class for extraction strategies."""
148 def __init__(self, db_params: Dict, directory: Path):
149 self.db_params = db_params
150 self.directory = directory
151 self.cohorts = {}
152 self.created_tables = set()
153 self.lock = threading.Lock()
154 self._local = threading.local()
155 self.csv_hashes = {}
156 self.graph = None
157 self.abort = threading.Event()
159 @cached_property
160 def engine(self):
161 """Returns the shared database engine."""
162 with self.session(bind=False) as session:
163 return session.get_bind()
165 @contextmanager
166 def session(self, bind: Optional[bool] = True) -> Iterator[Session]:
167 """Returns a managed database session.
169 If a session is already active for this thread, it is yielded.
170 Otherwise, a new pooled session is created and registered as active.
172 :param bind: Whether to use the shared engine. If False, a new
173 engine is created using Session.create().
174 """
175 if not bind:
176 with Session.create(**self.db_params) as session:
177 yield session
178 return
180 # Return thread-local session if active
181 if hasattr(self._local, "session"):
182 yield self._local.session
183 return
185 # Create a new managed session for this thread
186 with Session(bind=self.engine, autoflush=True) as session:
187 self._local.session = session
188 try:
189 yield session
190 finally:
191 if hasattr(self._local, "session"):
192 del self._local.session
194 def build_queries(self, task: ExtractMeta) -> Select:
195 """Build the extraction SA Select statement.
197 :param task: Extraction task metadata.
198 :type task: ExtractMeta
200 :return: SA Select statement.
201 :rtype: Select
202 """
203 with self.session() as session:
204 return QueryDirector.build_queries(session, task, strategy=self)
206 @property
207 def is_read_only(self) -> bool:
208 """Returns whether the strategy is read-only."""
209 return False
211 @abstractmethod
212 def process_filter(
213 self,
214 filter_config: FilterConfig,
215 session: Session,
216 task: ExtractMeta
217 ) -> List[Any]:
218 """Determine how to process a filter for this strategy."""
219 return [filter_config.to_exists(self)]
221 @abstractmethod
222 def execute(self, task: ExtractMeta, export_to: Optional[Path] = None):
223 """Execute the extraction task."""
225 @abstractmethod
226 def get_cohort(self, cohort: str) -> str:
227 """Retrieve or create a cohort representation."""
229 @abstractmethod
230 def drop_cohort(self, cohort: str):
231 """Drop a specific cohort table."""
233 @property
234 def tmp_dir(self) -> Path:
235 """Folder of the files that are not exported."""
236 return self.directory / "data" / ".tmp"
238 def drop(self, name: str):
239 """Drop the temporary data of the task with this temporary name,
240 which is its file."""
241 table_name = name[4:-(len(UUID) + 1)]
242 (self.tmp_dir / f"{table_name}.csv.gz").unlink(missing_ok=True)
243 self.created_tables.discard(name)
245 def cleanup(self):
246 """Drop all remaining temporary data and cohorts, and the folder of
247 the files that are not exported."""
248 for name in list(self.created_tables):
249 self.drop(name)
250 self.drop_cohorts()
252 try:
253 self.tmp_dir.rmdir()
254 except OSError:
255 pass
257 def drop_cohorts(self):
258 """Drop all remaining cohort tables."""
259 for cohort in list(self.cohorts.keys()):
260 self.drop_cohort(cohort)
263class TempTableStrategy(ExecutionStrategy):
264 """Strategy that uses temporary tables (requires write permissions)."""
266 def has_index(
267 self,
268 session: Session,
269 schema: Optional[str],
270 table: str,
271 columns: List[str]
272 ) -> bool:
273 """Evaluate the existence of an index with the specified columns."""
274 insp = inspect(session.bind)
275 for idx in insp.get_indexes(table, schema=schema):
276 if idx["column_names"] == columns:
277 return True
278 return False
280 def ensure_index(
281 self,
282 session: Session,
283 table_parts: List[str],
284 columns: List[str]
285 ):
286 """Create a covering index on the reference table if non-existent."""
287 if not columns:
288 return
290 columns_hash = hashlib.md5("_".join(columns).encode()).hexdigest()[:8] # noqa: DUO130, E501
291 schema = table_parts[0] if len(table_parts) == 2 else None
292 tbl_name = table_parts[-1]
294 with self.lock:
295 if not self.has_index(session, schema, tbl_name, columns):
296 try:
297 name = f"{tbl_name[:49]}_idx_{columns_hash}"
298 sa_table = Table(
299 tbl_name, MetaData(), schema=schema,
300 autoload_with=session.bind
301 )
302 sa_cols = [sa_table.c[c] for c in columns]
303 idx = Index(name, *sa_cols)
304 idx.create(bind=session.bind)
305 except SQLAlchemyError as err:
306 logger.warning(
307 "Table '%s': Failed to create index (%s).",
308 ".".join(table_parts), err
309 )
311 def process_filter(
312 self,
313 filter_config: FilterConfig,
314 session: Session,
315 task: ExtractMeta
316 ) -> List[Any]:
317 if filter_config.resolved_cohort is not None:
318 actual_table = self.cohorts.get(
319 filter_config.resolved_cohort,
320 filter_config.resolved_cohort
321 ).split(".")
322 self.ensure_index(
323 session,
324 actual_table,
325 filter_config.cohort_columns
326 )
327 return super().process_filter(filter_config, session, task)
329 def execute(self, task: ExtractMeta, export_to: Optional[Path] = None):
330 # pylint: disable=too-many-locals
331 logger.info("Table '%s': Extracting.", task.full_name)
333 with self.session() as session:
334 # TempTableStrategy is never read-only, but multiple OR groups
335 # will still produce multiple statements.
336 query = self.build_queries(task)
337 temp_table = task.temp_name
339 # Reflect source table to get column names and types
340 source_table = Table(
341 task.table,
342 MetaData(),
343 schema=task.schema,
344 autoload_with=session.bind
345 )
347 # Initialize destination table
348 meta = MetaData()
349 dest_table = Table(temp_table, meta)
351 # Add columns to destination table
352 col_names = []
353 for col in source_table.columns:
354 dest_table.append_column(Column(col.name, col.type))
355 col_names.append(col.name)
357 # Create destination table
358 self.created_tables.add(temp_table)
359 self.cohorts[task.full_name] = temp_table
360 dest_table.create(session.bind)
362 # Fallback to .tmp dir if export_to is not specified
363 target_dir = export_to or self.tmp_dir
364 target_dir.mkdir(parents=True, exist_ok=True)
365 export_path = target_dir / f"{task.table}.csv.gz"
367 # Write to CSV and inserting simultaneously, deleting the file if
368 # it is not complete
369 try:
370 self.csv_hashes[task.full_name] = DataExporter.stream_to_csv(
371 session,
372 query,
373 export_path,
374 table_name=task.table,
375 insert_to=dest_table,
376 chunk_size=task.chunk_size,
377 abort=self.abort
378 )
379 except BaseException:
380 export_path.unlink(missing_ok=True)
381 raise
382 if self.abort.is_set():
383 export_path.unlink(missing_ok=True)
385 def drop(self, name: str):
386 """Drop a temporary table, and its file."""
387 super().drop(name)
388 with self.session() as session:
389 try:
390 sa_table = Table(name, MetaData())
391 session.execute(DropTable(sa_table, if_exists=True))
392 session.commit()
393 except SQLAlchemyError as err:
394 logger.warning("Table '%s': Could not drop (%s).", name, err)
396 # pylint: disable=too-many-locals
397 def get_cohort(self, cohort: str) -> str:
398 """Retrieve or create a cohort table."""
399 if cohort in self.cohorts:
400 return self.cohorts[cohort]
402 logger.info("Cohort '%s': Generating data.", cohort)
404 name = cohort.replace(".sql", "").replace(".", "_")
405 name = f'tmp_{name}_{UUID}'
407 if not cohort.endswith(".sql"):
408 db_params = DatabaseMeta.load_env(self.directory)
409 source_engine = Session.create(**db_params).get_bind()
410 schema, table = cohort.split(".")
412 # Initialize source table
413 source_meta = MetaData()
414 source_table = Table(
415 table,
416 source_meta,
417 schema=schema,
418 autoload_with=source_engine
419 )
421 dest_columns = []
422 dest_engine = self.engine
423 for column in source_table.columns:
424 # Get column metadata
425 dialect = dest_engine.dialect.name
426 col_meta = MetadataExtractor.generate_column_meta(
427 column, dialect=dialect, compile_type=False
428 )
429 dest_columns.append(Column(
430 col_meta["name"], col_meta["dtype"][dialect]
431 ))
433 # Create destination table
434 dest_meta = MetaData()
435 dest_table = Table(name, dest_meta, *dest_columns)
436 self.cohorts[cohort] = name
437 dest_table.create(dest_engine)
439 # Move data from source to destination
440 cohort_data = pd.read_sql_table(table, source_engine, schema)
441 cohort_data.to_sql(
442 name, dest_engine, index=False, if_exists='append'
443 )
445 return name
447 # Load cohort query
448 cohort_path = self.directory / 'extract' / cohort
449 with open(cohort_path, encoding="utf-8") as file:
450 query = text(file.read().strip())
452 # Write cohort data to temp table
453 self.cohorts[cohort] = name
454 self.created_tables.add(name)
455 with self.session() as session:
456 cohort_data = pd.read_sql_query(query, session.bind)
457 cohort_data.to_sql(
458 name, session.bind, index=False, if_exists='fail'
459 )
461 return name
463 def drop_cohort(self, cohort: str):
464 """Drop a specific cohort table."""
465 if cohort in self.cohorts:
466 logger.info("Cohort '%s': Dropping data.", cohort)
467 try:
468 with self.session() as session:
469 sa_table = Table(self.cohorts[cohort], MetaData())
470 session.execute(DropTable(sa_table, if_exists=True))
471 session.commit()
472 except SQLAlchemyError as err:
473 logger.warning(
474 "Cohort '%s': Could not drop (%s).", cohort, err)
475 finally:
476 del self.cohorts[cohort]
479class JoinChain:
480 """Join chain for filtering on read-only mode."""
482 def __init__(self, filter_config: 'FilterConfig', engine):
483 self._engine = engine
484 self._cohort_cols = filter_config.cohort_columns
485 self._cohort_alias = filter_config.reference.alias
486 self._target_cols = []
487 for c in filter_config.reference.columns:
488 self._target_cols.append(text(c))
489 self._cohort_types = {}
490 self._seed_joins = []
491 self._where_clauses = []
492 self._from_clause = None
494 @classmethod
495 def build(
496 cls,
497 filter_config: 'FilterConfig',
498 engine,
499 task: 'ExtractMeta'
500 ) -> "JoinChain":
501 """Build and return a fully resolved chain for *filter_config*.
503 :param filter_config: Filter configuration describing the join chain.
504 :type filter_config: FilterConfig
506 :param engine: SQLAlchemy engine used to reflect remote tables.
508 :param task: Extraction task metadata.
509 :type task: ExtractMeta
511 :return: Populated :class:`JoinChain` ready to execute seed chunks.
512 :rtype: JoinChain
513 """
514 # Initialize join chain
515 chain_ = cls(filter_config, engine)
517 # Create aliased source table for the chain
518 src = Table(
519 task.table,
520 MetaData(),
521 schema=task.schema,
522 autoload_with=engine
523 ).alias("src")
524 chain_._from_clause = src
526 # Resolve column types and seed joins from the filter config
527 for name in filter_config.cohort_columns:
528 col = src.c.get(name, src.c.get(name.upper()))
529 if col is None:
530 continue
531 chain_._cohort_types[name] = col.type
532 chain_._seed_joins.append((col, name))
534 # Extend join chain with subsequent joins
535 for j_conf in filter_config.join:
536 chain_._extend(j_conf)
538 return chain_
540 def _extend(self, j_conf):
541 """Reflect one join config and extend the SA chain."""
542 ref = j_conf.reference
543 sa_table = Table(
544 ref.table, MetaData(), schema=ref.schema,
545 autoload_with=self._engine,
546 ).alias(ref.alias)
548 # Retrieve column types and seed joins from the filter config
549 for pair in j_conf.seed_pairs():
550 col = sa_table.c[pair.remote]
551 self._cohort_types[pair.local] = col.type
552 self._seed_joins.append((col, pair.local))
554 # Retrieve constant filters from the filter config
555 for p in j_conf.const_filters():
556 where = sa_table.c[p.remote] == literal(p.value)
557 self._where_clauses.append(where)
559 # Build the FROM clause for the join chain
560 if self._from_clause is None:
561 self._from_clause = sa_table
562 else:
563 on_clauses = []
564 for p in j_conf.cross_pairs():
565 on_clauses.append(text(f"{p.left} = {p.right}"))
566 onclause = and_(*on_clauses) if on_clauses else text("1=1")
567 self._from_clause = self._from_clause.join(sa_table, onclause)
569 def execute(self, chunk: List, session) -> List:
570 """Execute a seed chunk against the built chain.
572 :param chunk: Batch of seed tuples to look up.
573 :type chunk: list
575 :param session: Active SQLAlchemy session.
576 :type session: Session
578 :return: All matching rows from the remote tables.
579 :rtype: list
580 """
581 # Create list of column objects for VALUES clause
582 val_cols = []
583 for name in self._cohort_cols:
584 column = sa_column(name, self._cohort_types.get(name, String()))
585 val_cols.append(column)
586 v = values(*val_cols, name=self._cohort_alias).data(chunk)
588 # Build join ON clauses from seed pairs
589 seed_on = []
590 for col, name in self._seed_joins:
591 seed_on.append(col == v.c[name])
593 # Build the full statement
594 stmt = select(*self._target_cols).select_from(
595 self._from_clause.join(v, and_(*seed_on))
596 ).distinct()
597 if self._where_clauses:
598 stmt = stmt.where(and_(*self._where_clauses))
599 return session.execute(
600 stmt,
601 execution_options={
602 "skip_formatting": True
603 }
604 ).all()
607class StreamStrategy(ExecutionStrategy):
608 """Strategy that streams data directly from source (read-only)."""
610 @property
611 def is_read_only(self) -> bool:
612 return True
614 def process_filter(
615 self,
616 filter_config: FilterConfig,
617 session: Session,
618 task: ExtractMeta
619 ) -> List[Any]:
620 if filter_config.resolved_cohort and not filter_config.is_sql_cohort:
621 cohort_data = self.get_cohort_set(filter_config, task=task)
622 return [
623 filter_config.to_where(cohort_data[i:i + task.chunk_size])
624 for i in range(0, max(1, len(cohort_data)), task.chunk_size)
625 ]
626 return super().process_filter(filter_config, session, task)
628 def execute(self, task: ExtractMeta, export_to: Optional[Path] = None):
629 """Execute extraction by streaming and filtering in memory."""
630 target_dir = export_to or self.tmp_dir
631 target_dir.mkdir(parents=True, exist_ok=True)
633 logger.info("Table '%s': Extracting.", task.full_name)
635 # The file is known before it is written, to delete it if it is not
636 # complete
637 if export_to is None:
638 self.created_tables.add(task.temp_name)
640 with self.session() as session:
641 # Export to a CSV file
642 query = self.build_queries(task)
643 export_path = target_dir / f"{task.table}.csv.gz"
644 try:
645 self.csv_hashes[task.full_name] = DataExporter.stream_to_csv(
646 session,
647 query,
648 export_path,
649 table_name=task.table,
650 chunk_size=task.chunk_size,
651 abort=self.abort
652 )
653 except BaseException:
654 export_path.unlink(missing_ok=True)
655 raise
656 if self.abort.is_set():
657 export_path.unlink(missing_ok=True)
659 def drop_cohort(self, cohort: str):
660 """Clear cohort from memory (no DDL in stream mode)."""
661 if cohort in self.cohorts:
662 logger.info("Cohort '%s': Dropping data.", cohort)
663 del self.cohorts[cohort]
665 def get_cohort(self, cohort: str) -> Any:
666 """Register the cohort but do not load data yet."""
667 if cohort in self.cohorts:
668 return self.cohorts[cohort]
670 if cohort.endswith(".sql"):
671 cohort_path = self.directory / 'extract' / cohort
672 name = cohort.replace(".sql", "")
673 with open(cohort_path, encoding="utf-8") as file:
674 query = file.read().strip()
675 subquery = f"(\n {query}\n) {name}"
676 self.cohorts[cohort] = subquery
677 return subquery
679 self.cohorts[cohort] = cohort
680 return cohort
682 def get_cohort_set(
683 self,
684 filter_config: FilterConfig,
685 task: 'ExtractMeta'
686 ) -> List[Tuple[str, ...]]:
687 """Load cohort data into memory.
689 :param filter_config: Filter configuration.
690 :type filter_config: FilterConfig
692 :param task: Extraction task metadata.
693 :type task: ExtractMeta
695 :return: List of cohort records.
696 :rtype: list[tuple[str, ...]]
697 """
698 seed_chunks = self._fetch_local_seeds(
699 filter_config,
700 chunk_size=task.chunk_size
701 )
703 # If no joins, no need to query the database
704 if not filter_config.join:
705 return list(dict.fromkeys(chain.from_iterable(seed_chunks)))
707 chain_obj = JoinChain.build(filter_config, self.engine, task=task)
708 final_tuples = set()
709 for chunk in seed_chunks:
710 with self.session() as session:
711 for row in chain_obj.execute(chunk, session):
712 str_vals = []
713 for val in row:
714 str_vals.append(str(val))
715 final_tuples.add(tuple(str_vals))
716 return list(final_tuples)
718 def _fetch_local_seeds(
719 self,
720 filter_config: FilterConfig,
721 chunk_size: int = 10000
722 ) -> Iterator[List[Tuple[str, ...]]]:
723 # pylint: disable=too-many-locals
724 """Fetch the initial set of IDs from the local cohort table or CSV."""
725 fetch_cols = filter_config.cohort_columns
726 cohort = filter_config.resolved_cohort
728 # Check if the cohort is an extraction task in the graph
729 if self.graph and cohort in self.graph.tasks:
730 # Determine the file path
731 table = self.graph.tasks[cohort].table
732 temp_name = f"tmp_{table}_{UUID}"
733 if temp_name in self.created_tables:
734 csv_path = self.directory / "data" / ".tmp" / f"{table}.csv.gz"
735 else:
736 csv_path = self.directory / "data" / f"{table}.csv.gz"
738 if not csv_path.exists():
739 raise FileNotFoundError(f"Missing file for '{cohort}'")
741 # Read and hash the compressed file on-the-fly
742 with open(csv_path, 'rb') as f:
743 reader = HashingReader(f)
744 chunks = pd.read_csv(
745 reader,
746 compression='gzip',
747 usecols=fetch_cols,
748 dtype=str,
749 chunksize=chunk_size
750 )
752 # Load seeds in chunks
753 for chunk in chunks:
754 chunk = chunk[fetch_cols]
755 unique = chunk.drop_duplicates()
756 yield list(unique.itertuples(index=False, name=None))
758 # Validate hash to prevent corrupted reads
759 if reader.hasher.hexdigest() != self.csv_hashes[cohort]:
760 raise ValueError(f"Hash mismatch for file '{csv_path.name}'")
762 else:
763 # Fallback to local cohort database query
764 db_params = DatabaseMeta.load_env(self.directory)
765 with Session.create(**db_params) as session:
766 schema, table_name = cohort.split(".")
767 sa_table = Table(
768 table_name,
769 MetaData(),
770 schema=schema,
771 autoload_with=session.bind
772 )
773 cols = [sa_table.c[c] for c in fetch_cols]
774 stmt = select(*cols).distinct()
775 res = session.execute(stmt)
776 for partition in res.partitions(chunk_size):
777 yield [tuple(str(v) for v in row) for row in partition]
780class ParallelExecutor:
781 # pylint: disable=too-few-public-methods,too-many-instance-attributes
782 """Orchestrates parallel execution with reference counting for cleanup."""
784 def __init__(
785 self,
786 strategy: 'ExecutionStrategy',
787 graph: 'ExtractGraph',
788 max_workers: int = 4
789 ):
790 self.strategy = strategy
791 self.graph = graph
792 self.strategy.graph = graph
793 self.max_workers = max_workers
794 self.lock = RLock()
796 # Tracking dependencies
797 self.ref_counts = dict(graph.digraph.out_degree())
799 # graphlib topological sorter for parallel orchestration
800 self.ts = graph.topological_sorter
801 self.live_tasks = set()
803 # Register signal handlers for clean abort
804 self._register_signal_handlers()
806 def _register_signal_handlers(self):
807 """Register signal handlers to guarantee cleanup on abort."""
808 def _handler(signum, _frame):
809 logger.error("Pipeline: Received signal %s, aborting.", signum)
810 self.strategy.abort.set()
811 self.strategy.cleanup()
812 os._exit(1)
814 # Only register if running in the main thread
815 if threading.current_thread() is threading.main_thread():
816 signal.signal(signal.SIGINT, _handler)
817 signal.signal(signal.SIGTERM, _handler)
819 def execute(
820 self,
821 data_dir: Path
822 ):
823 """Run tasks in parallel using TopologicalSorter interactive mode.
825 :param data_dir: Directory of the data of the tasks to export.
826 :type data_dir: Path
827 """
828 with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
829 futures = {}
831 while self.ts.is_active():
832 # Submit all ready tasks
833 for node in self.ts.get_ready():
834 task = executor.submit(
835 self._run_task,
836 node,
837 data_dir
838 )
839 futures[task] = node
841 if not futures:
842 # This shouldn't happen if is_active is true,
843 # unless there's a deadlock or we're waiting for futures.
844 if any(f.running() for f in futures):
845 pass
846 else:
847 break
849 # Wait for at least one task to complete
850 for future in as_completed(futures):
851 node = futures.pop(future)
852 try:
853 future.result()
854 with self.lock:
855 self.ts.done(node)
856 # Trigger cleanup and get new ready tasks
857 self._on_task_complete(node)
858 break # Go back to the loop to submit new ready tasks
859 except (SQLAlchemyError, OSError) as e:
860 logger.error(
861 "Task '%s': Failed (%s).", node.full_name, e)
862 self.strategy.cleanup()
863 os._exit(1)
865 def _run_task(
866 self,
867 node: Union[ExtractMeta, Unresolved],
868 data_dir: Path
869 ):
870 """Execute a task (table extraction or cohort initialization)."""
871 # Start a managed session for this task
872 with self.strategy.session():
873 # Table extraction task
874 if isinstance(node, ExtractMeta):
875 export_to = data_dir if node in self.graph.export else None
877 with self.lock:
878 self.live_tasks.add(node)
880 self.strategy.execute(node, export_to=export_to)
881 # Cohort node
882 else:
883 self.strategy.get_cohort(node.full_name)
885 def _on_task_complete(self, node: Union[ExtractMeta, Unresolved]):
886 """Perform cleanup for dependencies no longer needed."""
887 to_drop_cohorts = []
888 to_drop = []
890 with self.lock:
891 # Cohorts currently have no prerequisites in the graph
892 for dep in self.graph.dependencies(node):
893 # Decrement ref counts of dependencies used by THIS node
894 self.ref_counts[dep] -= 1
896 # If no other nodes depend on this dependency, drop it
897 if self.ref_counts[dep] == 0:
898 if dep.full_name in self.strategy.cohorts:
899 to_drop_cohorts.append(dep.full_name)
900 elif dep in self.live_tasks:
901 to_drop.append(dep.temp_name)
902 self.live_tasks.remove(dep)
904 # Check if THIS node itself is no longer needed by anyone else
905 if self.ref_counts.get(node, 0) == 0:
906 if isinstance(node, ExtractMeta):
907 to_drop.append(node.temp_name)
908 self.live_tasks.discard(node)
909 elif node.full_name in self.strategy.cohorts:
910 to_drop_cohorts.append(node.full_name)
912 # Execute drops outside of the lock
913 for dep in to_drop_cohorts:
914 self.strategy.drop_cohort(dep)
915 for name in to_drop:
916 self.strategy.drop(name)