"""Per-group task queues and subprocess execution.""" from __future__ import annotations import json import os import signal import shutil import subprocess import threading import time from collections.abc import Mapping from dataclasses import replace from datetime import datetime from pathlib import Path from .config import Settings from .discovery import BackgroundTask, Group, Task, TaskCatalog from .inputs import PendingUpload from .store import Run, RunStore from .task_runtime import PAUSED_EXIT_CODE, atomic_json ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json" ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400 REVIEW_UPLOAD_RETENTION_SECONDS = 86400 class TaskConflictError(RuntimeError): def __init__(self, run: Run): self.run = run super().__init__(f"Run {run.id} is using resources required by this task.") def kill_remaining_group(process: subprocess.Popen) -> None: # The leader can already be gone while a transfer subprocess is still alive. try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass class RunManager: def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None: self.settings = settings self.catalog = catalog self.store = store self._wake_events: dict[str, threading.Event] = {} self._workers: dict[str, threading.Thread] = {} self._processes: dict[int, subprocess.Popen[bytes]] = {} self._background_processes: set[subprocess.Popen[bytes]] = set() self._lock = threading.Lock() self._cleanup_lock = threading.Lock() self._closing = threading.Event() def run_task( self, task: Task, input_data: dict[str, str | int | list[str]] | None = None, uploads: Mapping[str, PendingUpload] | None = None, ) -> tuple[Run, bool]: sensitive_names = {field.name for field in task.inputs if field.sensitive} stored_input = self._redact_input(input_data or {}, sensitive_names) with self._lock: if self._closing.is_set(): raise RuntimeError("The application is shutting down.") blocker = self.task_status(task)["blocked_by"] if blocker: raise TaskConflictError(blocker) run, created = self.store.enqueue( target_kind="task", target_path=task.id, queue_group=task.group_id, input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",), ) if created: try: uploaded_inputs = self._save_uploads(run.id, uploads or {}) execution_input = {**(input_data or {}), **uploaded_inputs} run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names)) self._save_work_input(run, execution_input) except (OSError, ValueError) as error: self._delete_uploads(run.id) self._discard_work_input(run) return self.store.finish( run.id, status="failed", message=f"Could not save task input: {error}", ), True if created: self._wake_worker(task.group_id) return run, created def run_group(self, group: Group) -> tuple[Run, bool]: resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources)) resources = resources or (f"group:{group.id}",) with self._lock: if self._closing.is_set(): raise RuntimeError("The application is shutting down.") for active in self.store.active_runs(): if active.target_kind == "group" and active.target_path == group.id: return active, False if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)): raise TaskConflictError(active) run, created = self.store.enqueue( target_kind="group", target_path=group.id, queue_group=group.id, resources=resources, ) if created: self._wake_worker(group.id) return run, created def startup(self) -> None: """Recover only runs with the durable input/context written by this runner.""" self.store.mark_interrupted() tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks} for old in self.store.interrupted_without_recovery(): work = self._work_root(old) if not (work / "input.json").is_file(): continue task = tasks.get(old.target_path) if task is None: continue if old.started_at and not task.recoverable: continue run, created = self.store.enqueue( target_kind="task", target_path=task.id, queue_group=task.group_id, input_data=json.loads(old.input_json), resources=task.resources, recovery_of_run_id=old.id, ) if created: self._wake_worker(run.queue_group) for run in self.store.active_runs(): if run.parent_run_id is None: self._wake_worker(run.queue_group) def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict: active = active_runs if active_runs is not None else self.store.active_runs() own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None) resources = set(task.resources or (f"group:{task.group_id}",)) blocker = next((run for run in active if run is not own and run.parent_run_id is None and resources.intersection(json.loads(run.resources_json))), None) return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""} def progress(self, run: Run) -> str: try: data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text()) except FileNotFoundError: return "" return str(data.get("text", "")) def shutdown(self, timeout: float | None = None) -> None: self._closing.set() deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout) with self._lock: for event in self._wake_events.values(): event.set() processes = [*self._processes.values(), *self._background_processes] for process in processes: if process.poll() is None: try: os.kill(process.pid, signal.SIGUSR1) except ProcessLookupError: pass while any(process.poll() is None for process in processes) and time.monotonic() < deadline: time.sleep(.05) for process in processes: kill_remaining_group(process) for process in processes: process.wait(timeout=5) for worker in self._workers.values(): worker.join(timeout=max(0, deadline - time.monotonic())) def _work_root(self, run: Run) -> Path: while run.recovery_of_run_id is not None: previous = self.store.get(run.recovery_of_run_id) if previous is None: raise RuntimeError("The original recovery run no longer exists") run = previous return self.settings.state_root / "runs" / f"{run.id}.work" def _save_work_input(self, run: Run, values: dict) -> None: atomic_json(self._work_root(run) / "input.json", values) def _discard_work_input(self, run: Run) -> None: for name in ("input.json", "input.json.tmp"): (self._work_root(run) / name).unlink(missing_ok=True) def request_cancel(self, run_id: int) -> Run | None: run = self.store.request_cancel(run_id) if run is None: return None if run.parent_run_id is None and run.target_kind == "group": child = self.store.active_child(run.id) if child is not None: self.request_cancel(child.id) self._terminate_active_process(run.id) completed = self.store.get(run.id) if completed is not None and completed.status == "cancelled": self._delete_uploads(run.id) self._discard_work_input(completed) return completed def artifacts(self, run_id: int) -> list[str]: root = self._artifact_root(run_id) if not root.is_dir(): return [] return [ path.name for path in sorted(root.iterdir(), key=lambda item: item.name) if path.is_file() and not path.is_symlink() ] def artifact_path(self, run_id: int, filename: str) -> Path | None: path = self._artifact_root(run_id) / filename if Path(filename).name != filename or path.is_symlink() or not path.is_file(): return None return path def delete_artifact(self, run_id: int, filename: str) -> None: path = self.artifact_path(run_id, filename) if path is None: return path.unlink(missing_ok=True) try: path.parent.rmdir() except OSError: pass def review_upload_paths(self, run: Run, task: Task) -> dict[str, Path] | None: """Return reusable uploads, or None when a run cannot be executed again.""" if run.status != "succeeded" or run.target_kind != "task" or task.execution is None: return None if any(field.sensitive for field in task.inputs): return None try: values = json.loads(run.input_json) except json.JSONDecodeError: return None if not isinstance(values, dict) or values.get(task.execution.field.name) != task.execution.dry_run_value: return None root = self._upload_root(run.id) paths = {} for field in task.inputs: if field.type != "file": continue value = values.get(field.name) if not value and not field.required: continue if not isinstance(value, str) or not run.finished_at: return None if time.time() - datetime.fromisoformat(run.finished_at).timestamp() >= REVIEW_UPLOAD_RETENTION_SECONDS: return None path = Path(value) if root.is_symlink() or path.parent != root or path.is_symlink() or not path.is_file(): return None if path.name != field.name + path.suffix or (field.accept and path.suffix.lower() not in field.accept): return None paths[field.name] = path return paths def discard_review_uploads(self, run_id: int) -> None: self._delete_uploads(run_id) def cleanup_transient_files( self, *, artifact_max_age_seconds: int = 86_400, orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS, ) -> None: """Remove transient run data and state for task groups removed long ago.""" with self._cleanup_lock: self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds) def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None: runs_root = self.settings.state_root / "runs" cutoff = time.time() - artifact_max_age_seconds if runs_root.is_dir(): lifetimes = self.store.run_lifetimes() protected = self._recovery_context_ids() for path in runs_root.iterdir(): run_id_text, separator, kind = path.name.partition(".") if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink(): continue run_id = int(run_id_text) if run_id not in lifetimes: # A worker may have created this run after the bulk snapshot. run = self.store.get(run_id) lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None) status, finished_at = lifetimes[run_id] if status in {"queued", "running"} or run_id in protected: continue if kind in {"input.json", "execution-input.json"} and path.is_file(): path.unlink(missing_ok=True) elif kind == "uploads" and path.is_dir(): run = self.store.get(run_id) task = self.catalog.task(run.target_path) if run and run.target_kind == "task" else None if task is None or not self.review_upload_paths(run, task): shutil.rmtree(path, ignore_errors=True) elif kind == "artifacts" and path.is_dir(): if status in {"failed", "interrupted", "cancelled"}: self._keep_diagnostic_reports(path) try: completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime except FileNotFoundError: continue if completed_at < cutoff: shutil.rmtree(path, ignore_errors=True) elif kind == "work" and path.is_dir(): (path / "input.json").unlink(missing_ok=True) if path.stat().st_mtime < time.time() - 14 * 86400: shutil.rmtree(path) elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400: path.unlink() self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds) def _recovery_context_ids(self) -> set[int]: protected = set() tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks} for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]: task = tasks.get(run.target_path) if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()): continue while True: protected.add(run.id) if run.recovery_of_run_id is None: break run = self.store.get(run.recovery_of_run_id) if run is None: break return protected def _promote_review_uploads(self, run: Run, task: Task) -> Run: if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs): return run values = json.loads(run.input_json) if values.get(task.execution.field.name) != task.execution.dry_run_value: return run origin = self._work_root(run).name.split(".", 1)[0] old_root = self._upload_root(int(origin)) for field in task.inputs: if field.type != "file" or not values.get(field.name): continue source = Path(values[field.name]) if source.parent != old_root or source.is_symlink() or not source.is_file(): raise RuntimeError("The original uploaded review input is no longer available") destination = self._upload_root(run.id) / source.name destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) shutil.copy2(source, destination) values[field.name] = str(destination) return self.store.set_input(run.id, values) @staticmethod def _keep_diagnostic_reports(root: Path) -> None: try: paths = list(root.iterdir()) except FileNotFoundError: return for path in paths: if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json": try: json.loads(path.read_text(encoding="utf-8")) continue except FileNotFoundError: continue except ValueError: pass if path.is_dir() and not path.is_symlink(): shutil.rmtree(path) else: path.unlink(missing_ok=True) def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None: """Delete state only after a group remains absent for a grace period.""" if retention_seconds < 0 or not self.settings.task_root.is_dir(): return root = self.settings.state_root / "tasks" if not root.is_dir(): return marker_path = root / ORPHANED_TASK_STATE_FILENAME try: markers = json.loads(marker_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): markers = {} if not isinstance(markers, dict): markers = {} active_groups = {group.id for group in self.catalog.groups()} current_time = time.time() updated: dict[str, float] = {} for path in root.iterdir(): if path.is_symlink() or not path.is_dir(): continue if path.name in active_groups: continue first_missing_at = markers.get(path.name, current_time) if not isinstance(first_missing_at, (int, float)): first_missing_at = current_time if current_time - float(first_missing_at) >= retention_seconds: shutil.rmtree(path, ignore_errors=True) else: updated[path.name] = float(first_missing_at) if updated: temporary = marker_path.with_suffix(marker_path.suffix + ".tmp") temporary.write_text(json.dumps(updated, sort_keys=True) + "\n", encoding="utf-8") temporary.replace(marker_path) else: marker_path.unlink(missing_ok=True) def run_background_task(self, task: BackgroundTask) -> None: """Run a declared internal task without creating high-volume run history.""" if self._closing.is_set(): return log_path = self.settings.state_root / "background" / f"{task.id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) environment = self._environment(task.group_id) environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work") try: with log_path.open("wb") as log_file: with self._lock: if self._closing.is_set(): return process = subprocess.Popen( [self.settings.python_executable, "-m", "app.worker", str(task.path)], cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file, stderr=subprocess.STDOUT, env=environment, start_new_session=True, ) self._background_processes.add(process) try: exit_code = process.wait() if exit_code not in (0, PAUSED_EXIT_CODE): log_file.write(f"Background task exited with status {exit_code}.\n".encode()) finally: kill_remaining_group(process) with self._lock: self._background_processes.discard(process) except OSError as error: log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8") def _wake_worker(self, group_id: str) -> None: with self._lock: if self._closing.is_set(): return event = self._wake_events.setdefault(group_id, threading.Event()) worker = self._workers.get(group_id) if worker is None or not worker.is_alive(): worker = threading.Thread( target=self._worker_loop, args=(group_id, event), name=f"task-group-{group_id}", daemon=True, ) self._workers[group_id] = worker worker.start() event.set() def _worker_loop(self, group_id: str, event: threading.Event) -> None: while not self._closing.is_set(): event.clear() # A queued run is claimable only after its private input and uploads are saved. with self._lock: if self._closing.is_set(): return run = self.store.claim_next(group_id) if run is None: event.wait() continue try: self._execute_root_run(run) finally: with self._lock: for waiting in self._wake_events.values(): waiting.set() def _execute_root_run(self, run: Run) -> None: if run.target_kind == "task": task = self.catalog.task(run.target_path) if task is None: self.store.finish( run.id, status="failed", message="Task is no longer available.", ) return self._execute_task(run, task) return group = self.catalog.group(run.target_path) if group is None: self.store.finish( run.id, status="failed", message="Group is no longer available.", ) return self._execute_group(run, group) def _execute_group(self, parent_run: Run, group: Group) -> None: for task in group.tasks: if self.store.is_cancel_requested(parent_run.id): self.store.finish( parent_run.id, status="cancelled", message="Group cancellation requested.", ) return child = self.store.create_child(parent_run, task.id) completed_child = self._execute_task(child, task, parent_run_id=parent_run.id) if completed_child.status != "succeeded": parent_status = ( "cancelled" if completed_child.status == "cancelled" or self.store.is_cancel_requested(parent_run.id) else "failed" ) self.store.finish( parent_run.id, status=parent_status, message=f"Stopped after {task.id}: {completed_child.status}.", ) return self.store.finish(parent_run.id, status="succeeded") def _execute_task( self, run: Run, task: Task, *, parent_run_id: int | None = None, ) -> Run: if self._closing.is_set(): return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.") if self.store.is_cancel_requested(run.id) or ( parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) ): return self.store.finish(run.id, status="cancelled", message="Cancelled before execution.") log_path = self.settings.state_root / "runs" / f"{run.id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) artifact_root = self._artifact_root(run.id) artifact_root.mkdir(parents=True, exist_ok=True) try: work = self._work_root(run) input_path = work / "input.json" if not input_path.is_file(): if run.recovery_of_run_id is not None: raise FileNotFoundError("The original recovery input is no longer available") self._save_work_input(run, json.loads(run.input_json)) except OSError as error: self._delete_uploads(run.id) self._discard_work_input(run) shutil.rmtree(artifact_root, ignore_errors=True) return self.store.finish( run.id, status="failed", message=f"Could not prepare task input: {error}", ) environment = self._environment(task.group_id) environment["SERVER_MAINTENANCE_INPUT"] = str(input_path) environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root) environment["SERVER_MAINTENANCE_WORK"] = str(work) environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json") if run.recovery_of_run_id is not None: old = self.store.get(run.recovery_of_run_id) if old and old.started_at: environment["SERVER_MAINTENANCE_RECOVER"] = "1" self.store.set_log_path(run.id, str(log_path)) try: with log_path.open("wb") as log_file: with self._lock: if self._closing.is_set(): return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.") process = subprocess.Popen( [self.settings.python_executable, "-m", "app.worker", str(task.path)], cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file, stderr=subprocess.STDOUT, env=environment, start_new_session=True, ) self._processes[run.id] = process if self.store.is_cancel_requested(run.id) or ( parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) ): self._terminate_active_process(run.id) exit_code = process.wait() except OSError as error: self._delete_uploads(run.id) self._discard_work_input(run) shutil.rmtree(artifact_root, ignore_errors=True) return self.store.finish( run.id, status="failed", message=f"Could not start task: {error}", log_path=str(log_path), ) finally: with self._lock: owned = self._processes.pop(run.id, None) if owned is not None: kill_remaining_group(owned) if self.store.is_cancel_requested(run.id) or ( parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) ): self._delete_uploads(run.id) self._discard_work_input(run) self._keep_diagnostic_reports(artifact_root) return self.store.finish( run.id, status="cancelled", exit_code=exit_code, log_path=str(log_path), ) if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0): return self.store.finish( run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path), message="Application shutdown; saved state will be checked by recovery after restart.", ) status = "succeeded" if exit_code == 0 else "failed" message = None if status == "succeeded": try: run = self._promote_review_uploads(run, task) except (RuntimeError, OSError) as error: status, message = "failed", f"Could not preserve review input: {error}" candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat()) if not self.review_upload_paths(candidate, task): self._delete_uploads(run.id) if status != "succeeded": self._keep_diagnostic_reports(artifact_root) else: try: artifact_root.rmdir() except OSError: pass completed = self.store.finish(run.id, status=status, exit_code=exit_code, message=message, log_path=str(log_path)) self._discard_work_input(run) self.cleanup_transient_files() return completed def _environment(self, group_id: str) -> dict[str, str]: task_state = self.settings.state_root / "tasks" / group_id task_state.mkdir(parents=True, exist_ok=True) environment = { "PATH": "/usr/local/bin:/usr/bin:/bin", "PYTHONUNBUFFERED": "1", "PYTHONPATH": str(Path(__file__).resolve().parent.parent), "SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root), "SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root), "SERVER_MAINTENANCE_STATE": str(task_state), } if self.settings.timezone_name: environment["TZ"] = self.settings.timezone_name return environment def _upload_root(self, run_id: int) -> Path: return self.settings.state_root / "runs" / f"{run_id}.uploads" def _artifact_root(self, run_id: int) -> Path: return self.settings.state_root / "runs" / f"{run_id}.artifacts" @staticmethod def _redact_input( input_data: Mapping[str, str | int | list[str]], sensitive_names: set[str], ) -> dict[str, str | int | list[str]]: return { name: "[redacted]" if name in sensitive_names else value for name, value in input_data.items() } def _save_uploads( self, run_id: int, uploads: Mapping[str, PendingUpload], ) -> dict[str, str]: if not uploads: return {} root = self._upload_root(run_id) root.mkdir(parents=True, exist_ok=False) saved = {} try: for field_name, upload in uploads.items(): suffix = upload.extension or ".upload" destination = root / f"{field_name}{suffix}" temporary = destination.with_suffix(destination.suffix + ".part") total = 0 with temporary.open("xb") as output: while chunk := upload.stream.read(64 * 1024): total += len(chunk) if upload.maximum_bytes is not None and total > upload.maximum_bytes: raise ValueError(f"{field_name} exceeds its size limit") output.write(chunk) temporary.replace(destination) destination.chmod(0o600) saved[field_name] = str(destination) except Exception: shutil.rmtree(root, ignore_errors=True) raise return saved def _delete_uploads(self, run_id: int) -> None: shutil.rmtree(self._upload_root(run_id), ignore_errors=True) def _terminate_active_process(self, run_id: int) -> None: with self._lock: process = self._processes.get(run_id) if process is None or process.poll() is not None: return try: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: return threading.Thread( target=self._kill_after_grace, args=(process,), name=f"task-cancel-{run_id}", daemon=True, ).start() def _kill_after_grace(self, process: subprocess.Popen[bytes]) -> None: try: process.wait(timeout=self.settings.cancel_grace_seconds) except subprocess.TimeoutExpired: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass