Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-10-29 22:12:04 +00:00
1 parent 034fecbd51
commit 3bfd199b9c
28 files changed
+2356 -172

No files matched your search

+245 -47
View File
@@ -10,6 +10,7 @@ import subprocess
import threading
import time
from collections.abc import Mapping
from dataclasses import replace
from datetime import datetime
from pathlib import Path
@@ -17,6 +18,7 @@ from .config import Settings
from .discovery import BackgroundTask, Group, Task, TaskCatalog
from .inputs import PendingUpload
from .store import Run, RunStore
from .task_runtime import PAUSED_EXIT_CODE, atomic_json
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
@@ -24,6 +26,20 @@ ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
class TaskConflictError(RuntimeError):
def __init__(self, run: Run):
self.run = run
super().__init__(f"Run {run.id} is using resources required by this task.")
def kill_remaining_group(process: subprocess.Popen) -> None:
# The leader can already be gone while a transfer subprocess is still alive.
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
class RunManager:
def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None:
self.settings = settings
@@ -32,7 +48,9 @@ class RunManager:
self._wake_events: dict[str, threading.Event] = {}
self._workers: dict[str, threading.Thread] = {}
self._processes: dict[int, subprocess.Popen[bytes]] = {}
self._background_processes: set[subprocess.Popen[bytes]] = set()
self._lock = threading.Lock()
self._closing = threading.Event()
def run_task(
self,
@@ -40,14 +58,20 @@ class RunManager:
input_data: dict[str, str | int | list[str]] | None = None,
uploads: Mapping[str, PendingUpload] | None = None,
) -> tuple[Run, bool]:
if self._closing.is_set():
raise RuntimeError("The application is shutting down; try again after it restarts.")
sensitive_names = {field.name for field in task.inputs if field.sensitive}
stored_input = self._redact_input(input_data or {}, sensitive_names)
run, created = self.store.enqueue(
target_kind="task",
target_path=task.id,
queue_group=task.group_id,
input_data=stored_input,
)
with self._lock:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
blocker = self.task_status(task)["blocked_by"]
if blocker:
raise TaskConflictError(blocker)
run, created = self.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
)
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
@@ -56,9 +80,11 @@ class RunManager:
run = self.store.set_input(run.id, stored_input)
if sensitive_names:
self._write_execution_input(run.id, execution_input)
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id,
status="failed",
@@ -68,15 +94,99 @@ class RunManager:
return run, created
def run_group(self, group: Group) -> tuple[Run, bool]:
run, created = self.store.enqueue(
target_kind="group",
target_path=group.id,
queue_group=group.id,
)
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
resources = resources or (f"group:{group.id}",)
with self._lock:
for active in self.store.active_runs():
if active.target_kind == "group" and active.target_path == group.id:
return active, False
if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)):
raise TaskConflictError(active)
run, created = self.store.enqueue(
target_kind="group", target_path=group.id, queue_group=group.id, resources=resources,
)
if created:
self._wake_worker(group.id)
return run, created
def startup(self) -> None:
"""Recover only runs with the durable input/context written by this runner."""
self.store.mark_interrupted()
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
for old in self.store.interrupted_without_recovery():
work = self._work_root(old)
if not (work / "input.json").is_file():
continue
task = tasks.get(old.target_path)
if task is None:
continue
if old.started_at and not task.recoverable:
continue
run, created = self.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=json.loads(old.input_json), resources=task.resources,
recovery_of_run_id=old.id,
)
if created:
self._wake_worker(run.queue_group)
for run in self.store.active_runs():
if run.parent_run_id is None:
self._wake_worker(run.queue_group)
def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict:
active = active_runs if active_runs is not None else self.store.active_runs()
own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None)
resources = set(task.resources or (f"group:{task.group_id}",))
blocker = next((run for run in active if run is not own and run.parent_run_id is None
and resources.intersection(json.loads(run.resources_json))), None)
return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""}
def progress(self, run: Run) -> str:
try:
data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text())
except FileNotFoundError:
return ""
return str(data.get("text", ""))
def shutdown(self, timeout: float | None = None) -> None:
self._closing.set()
for event in self._wake_events.values():
event.set()
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
with self._lock:
processes = [*self._processes.values(), *self._background_processes]
for process in processes:
if process.poll() is None:
try:
os.kill(process.pid, signal.SIGUSR1)
except ProcessLookupError:
pass
while any(process.poll() is None for process in processes) and time.monotonic() < deadline:
time.sleep(.05)
for process in processes:
kill_remaining_group(process)
for process in processes:
process.wait(timeout=5)
for worker in self._workers.values():
worker.join(timeout=max(0, deadline - time.monotonic()))
def _work_root(self, run: Run) -> Path:
while run.recovery_of_run_id is not None:
previous = self.store.get(run.recovery_of_run_id)
if previous is None:
raise RuntimeError("The original recovery run no longer exists")
run = previous
return self.settings.state_root / "runs" / f"{run.id}.work"
def _save_work_input(self, run: Run, values: dict) -> None:
atomic_json(self._work_root(run) / "input.json", values)
def _discard_work_input(self, run: Run) -> None:
for name in ("input.json", "input.json.tmp"):
(self._work_root(run) / name).unlink(missing_ok=True)
def request_cancel(self, run_id: int) -> Run | None:
run = self.store.request_cancel(run_id)
if run is None:
@@ -90,6 +200,7 @@ class RunManager:
if completed is not None and completed.status == "cancelled":
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(completed)
return completed
def artifacts(self, run_id: int) -> list[str]:
@@ -165,9 +276,10 @@ class RunManager:
cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir():
lifetimes = self.store.run_lifetimes()
protected = self._recovery_context_ids()
for path in runs_root.iterdir():
run_id_text, separator, kind = path.name.partition(".")
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink():
continue
run_id = int(run_id_text)
if run_id not in lifetimes:
@@ -175,7 +287,7 @@ class RunManager:
run = self.store.get(run_id)
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
status, finished_at = lifetimes[run_id]
if status in {"queued", "running"}:
if status in {"queued", "running"} or run_id in protected:
continue
if kind in {"input.json", "execution-input.json"} and path.is_file():
path.unlink(missing_ok=True)
@@ -193,8 +305,50 @@ class RunManager:
continue
if completed_at < cutoff:
shutil.rmtree(path, ignore_errors=True)
elif kind == "work" and path.is_dir():
(path / "input.json").unlink(missing_ok=True)
if path.stat().st_mtime < time.time() - 14 * 86400:
shutil.rmtree(path)
elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400:
path.unlink()
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
def _recovery_context_ids(self) -> set[int]:
protected = set()
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]:
task = tasks.get(run.target_path)
if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()):
continue
while True:
protected.add(run.id)
if run.recovery_of_run_id is None:
break
run = self.store.get(run.recovery_of_run_id)
if run is None:
break
return protected
def _promote_review_uploads(self, run: Run, task: Task) -> Run:
if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs):
return run
values = json.loads(run.input_json)
if values.get(task.execution.field.name) != task.execution.dry_run_value:
return run
origin = self._work_root(run).name.split(".", 1)[0]
old_root = self._upload_root(int(origin))
for field in task.inputs:
if field.type != "file" or not values.get(field.name):
continue
source = Path(values[field.name])
if source.parent != old_root or source.is_symlink() or not source.is_file():
raise RuntimeError("The original uploaded review input is no longer available")
destination = self._upload_root(run.id) / source.name
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
shutil.copy2(source, destination)
values[field.name] = str(destination)
return self.store.set_input(run.id, values)
@staticmethod
def _keep_diagnostic_reports(root: Path) -> None:
try:
@@ -254,24 +408,31 @@ class RunManager:
def run_background_task(self, task: BackgroundTask) -> None:
"""Run a declared internal task without creating high-volume run history."""
if self._closing.is_set():
return
log_path = self.settings.state_root / "background" / f"{task.id}.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work")
try:
with log_path.open("wb") as log_file:
process = subprocess.Popen(
[self.settings.python_executable, str(task.path)],
cwd=task.path.parent,
stdin=subprocess.DEVNULL,
stdout=log_file,
stderr=subprocess.STDOUT,
env=environment,
start_new_session=True,
)
exit_code = process.wait()
if exit_code:
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
with self._lock:
if self._closing.is_set():
return
process = subprocess.Popen(
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
)
self._background_processes.add(process)
try:
exit_code = process.wait()
if exit_code not in (0, PAUSED_EXIT_CODE):
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
finally:
kill_remaining_group(process)
with self._lock:
self._background_processes.discard(process)
except OSError as error:
log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8")
@@ -291,13 +452,17 @@ class RunManager:
event.set()
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
while True:
while not self._closing.is_set():
event.clear()
run = self.store.claim_next(group_id)
if run is None:
event.wait()
continue
self._execute_root_run(run)
try:
self._execute_root_run(run)
finally:
for waiting in self._wake_events.values():
waiting.set()
def _execute_root_run(self, run: Run) -> None:
if run.target_kind == "task":
@@ -355,6 +520,8 @@ class RunManager:
*,
parent_run_id: int | None = None,
) -> Run:
if self._closing.is_set():
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
@@ -366,12 +533,20 @@ class RunManager:
artifact_root = self._artifact_root(run.id)
artifact_root.mkdir(parents=True, exist_ok=True)
try:
if not input_path.exists():
work = self._work_root(run)
saved_input = work / "input.json"
if run.recovery_of_run_id is not None:
input_path = saved_input
elif not input_path.exists():
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
input_path.write_text(run.input_json + "\n", encoding="utf-8")
input_path.chmod(0o600)
if not saved_input.exists():
self._save_work_input(run, json.loads(input_path.read_text()))
except OSError as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
run.id,
@@ -381,24 +556,33 @@ class RunManager:
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
environment["SERVER_MAINTENANCE_WORK"] = str(work)
environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json")
if run.recovery_of_run_id is not None:
old = self.store.get(run.recovery_of_run_id)
if old and old.started_at:
environment["SERVER_MAINTENANCE_RECOVER"] = "1"
self.store.set_log_path(run.id, str(log_path))
try:
with log_path.open("wb") as log_file:
process = subprocess.Popen(
[self.settings.python_executable, str(task.path)],
cwd=task.path.parent,
stdin=subprocess.DEVNULL,
stdout=log_file,
stderr=subprocess.STDOUT,
env=environment,
start_new_session=True,
)
with self._lock:
if self._closing.is_set():
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
process = subprocess.Popen(
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
)
self._processes[run.id] = process
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._terminate_active_process(run.id)
exit_code = process.wait()
except OSError as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
run.id,
@@ -408,7 +592,9 @@ class RunManager:
)
finally:
with self._lock:
self._processes.pop(run.id, None)
owned = self._processes.pop(run.id, None)
if owned is not None:
kill_remaining_group(owned)
self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
@@ -416,6 +602,7 @@ class RunManager:
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._delete_uploads(run.id)
self._discard_work_input(run)
self._keep_diagnostic_reports(artifact_root)
return self.store.finish(
run.id,
@@ -423,21 +610,31 @@ class RunManager:
exit_code=exit_code,
log_path=str(log_path),
)
completed = self.store.finish(
run.id,
status="succeeded" if exit_code == 0 else "failed",
exit_code=exit_code,
log_path=str(log_path),
)
if not self.review_upload_paths(completed, task):
if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0):
return self.store.finish(
run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path),
message="Application shutdown; saved state will be checked by recovery after restart.",
)
status = "succeeded" if exit_code == 0 else "failed"
message = None
if status == "succeeded":
try:
run = self._promote_review_uploads(run, task)
except (RuntimeError, OSError) as error:
status, message = "failed", f"Could not preserve review input: {error}"
candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat())
if not self.review_upload_paths(candidate, task):
self._delete_uploads(run.id)
if completed.status != "succeeded":
if status != "succeeded":
self._keep_diagnostic_reports(artifact_root)
else:
try:
artifact_root.rmdir()
except OSError:
pass
completed = self.store.finish(run.id, status=status, exit_code=exit_code,
message=message, log_path=str(log_path))
self._discard_work_input(run)
self.cleanup_transient_files()
return completed
@@ -447,6 +644,7 @@ class RunManager:
environment = {
"PATH": "/usr/local/bin:/usr/bin:/bin",
"PYTHONUNBUFFERED": "1",
"PYTHONPATH": str(Path(__file__).resolve().parent.parent),
"SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root),
"SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root),
"SERVER_MAINTENANCE_STATE": str(task_state),