Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-10-29 22:12:04 +00:00
1 parent 034fecbd51
commit 3bfd199b9c
28 files changed
+2356 -172

No files matched your search

+41 -6
View File
@@ -3,15 +3,50 @@
from __future__ import annotations
import os
import signal
import threading
import time
from waitress import serve
from waitress import create_server
from .web import create_app
def main() -> None:
stopping = False
requested_at = None
def request_shutdown(_signal, _frame):
nonlocal stopping, requested_at
stopping = True
if requested_at is None:
requested_at = time.monotonic()
signal.signal(signal.SIGTERM, request_shutdown)
signal.signal(signal.SIGINT, request_shutdown)
app, server = None, None
try:
app = create_app()
server = create_server(
app,
host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"),
port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")),
)
thread = threading.Thread(target=server.run, name="http-server", daemon=True)
thread.start()
while not stopping and thread.is_alive():
time.sleep(.1)
finally:
print("Shutdown requested; pausing tasks at safe checkpoints.", flush=True)
if app is not None:
app.extensions["background_scheduler"].stop()
manager = app.extensions["run_manager"]
elapsed = time.monotonic() - requested_at if requested_at is not None else 0
manager.shutdown(timeout=max(0, manager.settings.shutdown_grace_seconds - elapsed))
if server is not None:
server.task_dispatcher.shutdown(timeout=5)
server.close()
if __name__ == "__main__":
serve(
create_app(),
host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"),
port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")),
)
main()
+2
View File
@@ -17,6 +17,7 @@ class Settings:
timezone_name: str | None
allowed_proxy_ips: frozenset[str]
cancel_grace_seconds: float
shutdown_grace_seconds: float = 110
@property
def database_path(self) -> Path:
@@ -55,4 +56,5 @@ class Settings:
cancel_grace_seconds=float(
os.environ.get("SERVER_MAINTENANCE_CANCEL_GRACE_SECONDS", "10")
),
shutdown_grace_seconds=float(os.environ.get("SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS", "110")),
)
+22
View File
@@ -39,6 +39,8 @@ class Task:
wait_for_result: bool = False
download_artifacts: bool = False
execution: "ExecutionMode | None" = None
recoverable: bool = False
resources: tuple[str, ...] = ()
@dataclass(frozen=True)
@@ -190,6 +192,7 @@ class TaskCatalog:
for filename in ordered_filenames:
path = paths_by_filename[filename]
declaration = declarations_by_filename.get(path.name)
recoverable, resources = read_runtime_declaration(path)
tasks.append(
Task(
id=f"{group_path.name}/{path.name}",
@@ -210,6 +213,8 @@ class TaskCatalog:
wait_for_result=declaration.wait_for_result if declaration else False,
download_artifacts=declaration.download_artifacts if declaration else False,
execution=declaration.execution if declaration else None,
recoverable=recoverable,
resources=(f"group:{group_path.name}", *resources),
)
)
return tasks
@@ -294,6 +299,23 @@ def read_description(path: Path) -> str | None:
)
def read_runtime_declaration(path: Path) -> tuple[bool, tuple[str, ...]]:
"""Discover the opt-in hook and literal resource names without importing code."""
try:
module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
except (OSError, SyntaxError, UnicodeDecodeError):
return False, ()
recoverable = any(isinstance(node, ast.FunctionDef) and node.name == "recover" for node in module.body)
resources = ()
for node in module.body:
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "RESOURCES" for t in node.targets):
value = ast.literal_eval(node.value)
if not isinstance(value, (tuple, list)) or any(not isinstance(x, str) or not re.fullmatch(r"[a-z][a-z0-9_-]*", x) for x in value):
raise ValueError(f"RESOURCES in {path} must be literal resource names")
resources = tuple(dict.fromkeys(value))
return recoverable, resources
@dataclass(frozen=True)
class GroupManifest:
title: str | None
+245 -47
View File
@@ -10,6 +10,7 @@ import subprocess
import threading
import time
from collections.abc import Mapping
from dataclasses import replace
from datetime import datetime
from pathlib import Path
@@ -17,6 +18,7 @@ from .config import Settings
from .discovery import BackgroundTask, Group, Task, TaskCatalog
from .inputs import PendingUpload
from .store import Run, RunStore
from .task_runtime import PAUSED_EXIT_CODE, atomic_json
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
@@ -24,6 +26,20 @@ ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
class TaskConflictError(RuntimeError):
def __init__(self, run: Run):
self.run = run
super().__init__(f"Run {run.id} is using resources required by this task.")
def kill_remaining_group(process: subprocess.Popen) -> None:
# The leader can already be gone while a transfer subprocess is still alive.
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
class RunManager:
def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None:
self.settings = settings
@@ -32,7 +48,9 @@ class RunManager:
self._wake_events: dict[str, threading.Event] = {}
self._workers: dict[str, threading.Thread] = {}
self._processes: dict[int, subprocess.Popen[bytes]] = {}
self._background_processes: set[subprocess.Popen[bytes]] = set()
self._lock = threading.Lock()
self._closing = threading.Event()
def run_task(
self,
@@ -40,14 +58,20 @@ class RunManager:
input_data: dict[str, str | int | list[str]] | None = None,
uploads: Mapping[str, PendingUpload] | None = None,
) -> tuple[Run, bool]:
if self._closing.is_set():
raise RuntimeError("The application is shutting down; try again after it restarts.")
sensitive_names = {field.name for field in task.inputs if field.sensitive}
stored_input = self._redact_input(input_data or {}, sensitive_names)
run, created = self.store.enqueue(
target_kind="task",
target_path=task.id,
queue_group=task.group_id,
input_data=stored_input,
)
with self._lock:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
blocker = self.task_status(task)["blocked_by"]
if blocker:
raise TaskConflictError(blocker)
run, created = self.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
)
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
@@ -56,9 +80,11 @@ class RunManager:
run = self.store.set_input(run.id, stored_input)
if sensitive_names:
self._write_execution_input(run.id, execution_input)
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id,
status="failed",
@@ -68,15 +94,99 @@ class RunManager:
return run, created
def run_group(self, group: Group) -> tuple[Run, bool]:
run, created = self.store.enqueue(
target_kind="group",
target_path=group.id,
queue_group=group.id,
)
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
resources = resources or (f"group:{group.id}",)
with self._lock:
for active in self.store.active_runs():
if active.target_kind == "group" and active.target_path == group.id:
return active, False
if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)):
raise TaskConflictError(active)
run, created = self.store.enqueue(
target_kind="group", target_path=group.id, queue_group=group.id, resources=resources,
)
if created:
self._wake_worker(group.id)
return run, created
def startup(self) -> None:
"""Recover only runs with the durable input/context written by this runner."""
self.store.mark_interrupted()
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
for old in self.store.interrupted_without_recovery():
work = self._work_root(old)
if not (work / "input.json").is_file():
continue
task = tasks.get(old.target_path)
if task is None:
continue
if old.started_at and not task.recoverable:
continue
run, created = self.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=json.loads(old.input_json), resources=task.resources,
recovery_of_run_id=old.id,
)
if created:
self._wake_worker(run.queue_group)
for run in self.store.active_runs():
if run.parent_run_id is None:
self._wake_worker(run.queue_group)
def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict:
active = active_runs if active_runs is not None else self.store.active_runs()
own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None)
resources = set(task.resources or (f"group:{task.group_id}",))
blocker = next((run for run in active if run is not own and run.parent_run_id is None
and resources.intersection(json.loads(run.resources_json))), None)
return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""}
def progress(self, run: Run) -> str:
try:
data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text())
except FileNotFoundError:
return ""
return str(data.get("text", ""))
def shutdown(self, timeout: float | None = None) -> None:
self._closing.set()
for event in self._wake_events.values():
event.set()
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
with self._lock:
processes = [*self._processes.values(), *self._background_processes]
for process in processes:
if process.poll() is None:
try:
os.kill(process.pid, signal.SIGUSR1)
except ProcessLookupError:
pass
while any(process.poll() is None for process in processes) and time.monotonic() < deadline:
time.sleep(.05)
for process in processes:
kill_remaining_group(process)
for process in processes:
process.wait(timeout=5)
for worker in self._workers.values():
worker.join(timeout=max(0, deadline - time.monotonic()))
def _work_root(self, run: Run) -> Path:
while run.recovery_of_run_id is not None:
previous = self.store.get(run.recovery_of_run_id)
if previous is None:
raise RuntimeError("The original recovery run no longer exists")
run = previous
return self.settings.state_root / "runs" / f"{run.id}.work"
def _save_work_input(self, run: Run, values: dict) -> None:
atomic_json(self._work_root(run) / "input.json", values)
def _discard_work_input(self, run: Run) -> None:
for name in ("input.json", "input.json.tmp"):
(self._work_root(run) / name).unlink(missing_ok=True)
def request_cancel(self, run_id: int) -> Run | None:
run = self.store.request_cancel(run_id)
if run is None:
@@ -90,6 +200,7 @@ class RunManager:
if completed is not None and completed.status == "cancelled":
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(completed)
return completed
def artifacts(self, run_id: int) -> list[str]:
@@ -165,9 +276,10 @@ class RunManager:
cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir():
lifetimes = self.store.run_lifetimes()
protected = self._recovery_context_ids()
for path in runs_root.iterdir():
run_id_text, separator, kind = path.name.partition(".")
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink():
continue
run_id = int(run_id_text)
if run_id not in lifetimes:
@@ -175,7 +287,7 @@ class RunManager:
run = self.store.get(run_id)
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
status, finished_at = lifetimes[run_id]
if status in {"queued", "running"}:
if status in {"queued", "running"} or run_id in protected:
continue
if kind in {"input.json", "execution-input.json"} and path.is_file():
path.unlink(missing_ok=True)
@@ -193,8 +305,50 @@ class RunManager:
continue
if completed_at < cutoff:
shutil.rmtree(path, ignore_errors=True)
elif kind == "work" and path.is_dir():
(path / "input.json").unlink(missing_ok=True)
if path.stat().st_mtime < time.time() - 14 * 86400:
shutil.rmtree(path)
elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400:
path.unlink()
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
def _recovery_context_ids(self) -> set[int]:
protected = set()
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]:
task = tasks.get(run.target_path)
if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()):
continue
while True:
protected.add(run.id)
if run.recovery_of_run_id is None:
break
run = self.store.get(run.recovery_of_run_id)
if run is None:
break
return protected
def _promote_review_uploads(self, run: Run, task: Task) -> Run:
if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs):
return run
values = json.loads(run.input_json)
if values.get(task.execution.field.name) != task.execution.dry_run_value:
return run
origin = self._work_root(run).name.split(".", 1)[0]
old_root = self._upload_root(int(origin))
for field in task.inputs:
if field.type != "file" or not values.get(field.name):
continue
source = Path(values[field.name])
if source.parent != old_root or source.is_symlink() or not source.is_file():
raise RuntimeError("The original uploaded review input is no longer available")
destination = self._upload_root(run.id) / source.name
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
shutil.copy2(source, destination)
values[field.name] = str(destination)
return self.store.set_input(run.id, values)
@staticmethod
def _keep_diagnostic_reports(root: Path) -> None:
try:
@@ -254,24 +408,31 @@ class RunManager:
def run_background_task(self, task: BackgroundTask) -> None:
"""Run a declared internal task without creating high-volume run history."""
if self._closing.is_set():
return
log_path = self.settings.state_root / "background" / f"{task.id}.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work")
try:
with log_path.open("wb") as log_file:
process = subprocess.Popen(
[self.settings.python_executable, str(task.path)],
cwd=task.path.parent,
stdin=subprocess.DEVNULL,
stdout=log_file,
stderr=subprocess.STDOUT,
env=environment,
start_new_session=True,
)
exit_code = process.wait()
if exit_code:
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
with self._lock:
if self._closing.is_set():
return
process = subprocess.Popen(
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
)
self._background_processes.add(process)
try:
exit_code = process.wait()
if exit_code not in (0, PAUSED_EXIT_CODE):
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
finally:
kill_remaining_group(process)
with self._lock:
self._background_processes.discard(process)
except OSError as error:
log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8")
@@ -291,13 +452,17 @@ class RunManager:
event.set()
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
while True:
while not self._closing.is_set():
event.clear()
run = self.store.claim_next(group_id)
if run is None:
event.wait()
continue
self._execute_root_run(run)
try:
self._execute_root_run(run)
finally:
for waiting in self._wake_events.values():
waiting.set()
def _execute_root_run(self, run: Run) -> None:
if run.target_kind == "task":
@@ -355,6 +520,8 @@ class RunManager:
*,
parent_run_id: int | None = None,
) -> Run:
if self._closing.is_set():
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
@@ -366,12 +533,20 @@ class RunManager:
artifact_root = self._artifact_root(run.id)
artifact_root.mkdir(parents=True, exist_ok=True)
try:
if not input_path.exists():
work = self._work_root(run)
saved_input = work / "input.json"
if run.recovery_of_run_id is not None:
input_path = saved_input
elif not input_path.exists():
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
input_path.write_text(run.input_json + "\n", encoding="utf-8")
input_path.chmod(0o600)
if not saved_input.exists():
self._save_work_input(run, json.loads(input_path.read_text()))
except OSError as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
run.id,
@@ -381,24 +556,33 @@ class RunManager:
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
environment["SERVER_MAINTENANCE_WORK"] = str(work)
environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json")
if run.recovery_of_run_id is not None:
old = self.store.get(run.recovery_of_run_id)
if old and old.started_at:
environment["SERVER_MAINTENANCE_RECOVER"] = "1"
self.store.set_log_path(run.id, str(log_path))
try:
with log_path.open("wb") as log_file:
process = subprocess.Popen(
[self.settings.python_executable, str(task.path)],
cwd=task.path.parent,
stdin=subprocess.DEVNULL,
stdout=log_file,
stderr=subprocess.STDOUT,
env=environment,
start_new_session=True,
)
with self._lock:
if self._closing.is_set():
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
process = subprocess.Popen(
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
)
self._processes[run.id] = process
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._terminate_active_process(run.id)
exit_code = process.wait()
except OSError as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
run.id,
@@ -408,7 +592,9 @@ class RunManager:
)
finally:
with self._lock:
self._processes.pop(run.id, None)
owned = self._processes.pop(run.id, None)
if owned is not None:
kill_remaining_group(owned)
self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
@@ -416,6 +602,7 @@ class RunManager:
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._delete_uploads(run.id)
self._discard_work_input(run)
self._keep_diagnostic_reports(artifact_root)
return self.store.finish(
run.id,
@@ -423,21 +610,31 @@ class RunManager:
exit_code=exit_code,
log_path=str(log_path),
)
completed = self.store.finish(
run.id,
status="succeeded" if exit_code == 0 else "failed",
exit_code=exit_code,
log_path=str(log_path),
)
if not self.review_upload_paths(completed, task):
if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0):
return self.store.finish(
run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path),
message="Application shutdown; saved state will be checked by recovery after restart.",
)
status = "succeeded" if exit_code == 0 else "failed"
message = None
if status == "succeeded":
try:
run = self._promote_review_uploads(run, task)
except (RuntimeError, OSError) as error:
status, message = "failed", f"Could not preserve review input: {error}"
candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat())
if not self.review_upload_paths(candidate, task):
self._delete_uploads(run.id)
if completed.status != "succeeded":
if status != "succeeded":
self._keep_diagnostic_reports(artifact_root)
else:
try:
artifact_root.rmdir()
except OSError:
pass
completed = self.store.finish(run.id, status=status, exit_code=exit_code,
message=message, log_path=str(log_path))
self._discard_work_input(run)
self.cleanup_transient_files()
return completed
@@ -447,6 +644,7 @@ class RunManager:
environment = {
"PATH": "/usr/local/bin:/usr/bin:/bin",
"PYTHONUNBUFFERED": "1",
"PYTHONPATH": str(Path(__file__).resolve().parent.parent),
"SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root),
"SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root),
"SERVER_MAINTENANCE_STATE": str(task_state),
+37 -3
View File
@@ -30,6 +30,8 @@ class Run:
log_path: str | None
message: str | None
input_json: str
recovery_of_run_id: int | None = None
resources_json: str = "[]"
def now() -> str:
@@ -72,6 +74,10 @@ class RunStore:
connection.execute(
"ALTER TABLE runs ADD COLUMN input_json TEXT NOT NULL DEFAULT '{}'"
)
if "recovery_of_run_id" not in columns:
connection.execute("ALTER TABLE runs ADD COLUMN recovery_of_run_id INTEGER REFERENCES runs(id)")
if "resources_json" not in columns:
connection.execute("ALTER TABLE runs ADD COLUMN resources_json TEXT NOT NULL DEFAULT '[]'")
connection.execute(
"""
CREATE INDEX IF NOT EXISTS runs_queue_index
@@ -106,6 +112,8 @@ class RunStore:
queue_group: str,
trigger: str = "manual",
input_data: Mapping[str, str | int | list[str]] | None = None,
resources: tuple[str, ...] = (),
recovery_of_run_id: int | None = None,
) -> tuple[Run, bool]:
input_json = json.dumps(input_data or {}, sort_keys=True, separators=(",", ":"))
with self._connect() as connection:
@@ -126,10 +134,12 @@ class RunStore:
cursor = connection.execute(
"""
INSERT INTO runs (
target_kind, target_path, queue_group, trigger, status, created_at, input_json
) VALUES (?, ?, ?, ?, 'queued', ?, ?)
target_kind, target_path, queue_group, trigger, status, created_at, input_json,
resources_json, recovery_of_run_id
) VALUES (?, ?, ?, ?, 'queued', ?, ?, ?, ?)
""",
(target_kind, target_path, queue_group, trigger, now(), input_json),
(target_kind, target_path, queue_group, trigger, now(), input_json,
json.dumps(resources), recovery_of_run_id),
)
return self.get(cursor.lastrowid, connection=connection), True
@@ -162,6 +172,10 @@ class RunStore:
).fetchone()
if row is None:
return None
requested = set(json.loads(row["resources_json"]))
for active in connection.execute("SELECT resources_json FROM runs WHERE status='running' AND parent_run_id IS NULL"):
if requested & set(json.loads(active["resources_json"])):
return None
run_id = row["id"]
connection.execute(
"UPDATE runs SET status = 'running', started_at = ? WHERE id = ?",
@@ -263,6 +277,26 @@ class RunStore:
for row in connection.execute("SELECT id, status, finished_at FROM runs")
}
def active_runs(self) -> list[Run]:
with self._connect() as connection:
return [self._run(row) for row in connection.execute(
"SELECT * FROM runs WHERE status IN ('queued','running') ORDER BY id"
)]
def interrupted_without_recovery(self) -> list[Run]:
with self._connect() as connection:
return [self._run(row) for row in connection.execute(
"""SELECT r.* FROM runs r WHERE r.status='interrupted' AND r.target_kind='task'
AND r.cancel_requested_at IS NULL
AND NOT EXISTS (SELECT 1 FROM runs child WHERE child.recovery_of_run_id=r.id)
ORDER BY r.id"""
)]
def recovery_for(self, run_id: int) -> Run | None:
with self._connect() as connection:
row = connection.execute("SELECT * FROM runs WHERE recovery_of_run_id=? ORDER BY id DESC LIMIT 1", (run_id,)).fetchone()
return self._run(row) if row else None
def set_log_path(self, run_id: int, path: str) -> None:
with self._connect() as connection:
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id))
+161
View File
@@ -0,0 +1,161 @@
"""Cooperative task checkpoints and optional plain-text progress."""
from __future__ import annotations
import json
import os
import signal
import time
from contextlib import contextmanager
from pathlib import Path
PAUSED_EXIT_CODE = 75
CHECKPOINT_SECONDS = 60
_pause_requested = False
_defer = 0
_state: dict = {}
_last_save = 0.0
_last_progress = 0.0
_previous_clean = False
class TaskPaused(BaseException):
"""Unwind at a safe boundary without treating a pause as an ordinary failure."""
def work_directory() -> Path | None:
value = os.environ.get("SERVER_MAINTENANCE_WORK")
return Path(value) if value else None
def atomic_json(path: Path, value: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("w", encoding="utf-8") as stream:
temporary.chmod(0o600)
json.dump(value, stream, ensure_ascii=False)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def initialize() -> None:
global _state, _previous_clean, _pause_requested, _last_save
_pause_requested = False
root = work_directory()
if root is not None:
try:
saved = json.loads((root / "checkpoint.json").read_text())
except FileNotFoundError:
saved = {}
_state = saved.get("data", {})
_previous_clean = bool(saved.get("clean_shutdown"))
_write_checkpoint(False)
_last_save = time.monotonic()
signal.signal(signal.SIGUSR1, request_pause)
def request_pause(_signal=None, _frame=None) -> None:
global _pause_requested
_pause_requested = True
def pause_requested() -> bool:
return _pause_requested and work_directory() is not None
def checkpoint_data() -> dict:
return _state if work_directory() is not None else {}
def was_clean_shutdown() -> bool:
return _previous_clean
def _write_checkpoint(clean: bool) -> None:
root = work_directory()
if root is not None:
atomic_json(root / "checkpoint.json", {
"version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(),
})
def checkpoint(data: dict | None = None, *, force: bool = False) -> None:
"""Keep the latest state in memory; persist ordinary repeatable work once a minute."""
global _state, _last_save
if data is not None:
_state = data
if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS:
_write_checkpoint(False)
_last_save = time.monotonic()
check_stop()
def check_stop() -> None:
if _pause_requested and work_directory() is not None and not _defer:
_write_checkpoint(True)
raise TaskPaused()
@contextmanager
def critical_section():
"""Finish a small consistency-sensitive operation before observing a pause."""
global _defer
check_stop()
_defer += 1
try:
yield
finally:
_defer -= 1
check_stop()
def sleep(seconds: float) -> None:
deadline = time.monotonic() + seconds
while (remaining := deadline - time.monotonic()) > 0:
check_stop()
time.sleep(min(remaining, 0.25))
check_stop()
def progress(text: str, *, force: bool = False) -> None:
global _last_progress
root = work_directory()
if root is not None and (force or time.monotonic() - _last_progress >= 1):
path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json")))
atomic_json(path, {"text": text, "updated_at": time.time()})
_last_progress = time.monotonic()
class Progress:
"""Item-count estimates, independently usable for several named steps."""
def __init__(self, totals: dict[str, int]):
self.totals = totals
self.done = dict.fromkeys(totals, 0)
self.started: dict[str, float] = {}
self.baseline: dict[str, int] = {}
def update(self, label: str, done: int, *, force: bool = False) -> None:
self.started.setdefault(label, time.monotonic())
self.baseline.setdefault(label, done)
self.done[label] = done
lines = []
for name, total in self.totals.items():
count = self.done[name]
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
line = f"{name}: {percent}%"
processed = count - self.baseline.get(name, 0)
if processed > 0 and count < total and name in self.started:
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
line += f", ETA {estimate}"
lines.append(line)
progress("\n".join(lines), force=force)
+105 -15
View File
@@ -14,7 +14,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u
from .config import Settings
from .discovery import TaskCatalog
from .inputs import PendingUpload, validate_inputs, validate_uploads
from .runner import RunManager
from .runner import RunManager, TaskConflictError
from .scheduler import BackgroundScheduler
from .store import RunStore
@@ -24,11 +24,8 @@ def create_app(settings: Settings | None = None) -> Flask:
catalog = TaskCatalog(settings.task_root)
store = RunStore(settings.database_path)
store.initialize()
store.mark_interrupted()
manager = RunManager(settings, catalog, store)
manager.cleanup_transient_files()
scheduler = BackgroundScheduler(catalog, manager)
scheduler.start()
project_root = Path(__file__).resolve().parent.parent
app = Flask(
@@ -62,6 +59,66 @@ def create_app(settings: Settings | None = None) -> Flask:
return "—"
return datetime.fromisoformat(value).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
def run_links(run):
if run is None:
return None
display_status = run.status
if run.recovery_of_run_id is not None and run.status in {"queued", "running"}:
display_status = "Recovery queued" if run.status == "queued" else "recovering"
return {
"run_id": run.id,
"status": run.status,
"display_status": display_status,
"recovery_of_run_id": run.recovery_of_run_id,
"artifacts_url": url_for("run_artifacts", run_id=run.id),
"detail_url": url_for("run_detail", run_id=run.id),
"output_url": url_for("run_output", run_id=run.id),
"state_url": url_for("run_state", run_id=run.id),
}
def task_state(task, active_runs=None):
state = manager.task_status(task, active_runs=active_runs)
return {
"active_run": run_links(state["active_run"]),
"blocked_by": run_links(state["blocked_by"]),
"progress": state["progress"],
}
def live_states(groups, latest):
active = store.active_runs()
tasks = {task.id: task_state(task, active) for group in groups for task in group.tasks}
group_states = {}
for group in groups:
run = latest[("group", group.id)]
active = run if run and run.status in {"queued", "running"} else None
blocker = next((
tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
for task in group.tasks
if tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
), None)
group_states[group.id] = {
"active_run": run_links(active),
"blocked_by": None if active else blocker,
"progress": manager.progress(active) if active else "",
}
return {"tasks": tasks, "groups": group_states}
def unavailable(error):
if isinstance(error, TaskConflictError):
return task_conflict(error)
message = str(error)
if request.accept_mimetypes.best == "application/json":
return {"error": message}, 503
return render_template("unavailable.html", message=message, blocking_run=None), 503
@app.errorhandler(TaskConflictError)
def task_conflict(error):
blocking_run = run_links(error.run)
message = f"Task blocked by run #{error.run.id} ({blocking_run['display_status']})."
if request.accept_mimetypes.best == "application/json":
return {"error": message, "blocked_by": blocking_run}, 409
return render_template("unavailable.html", message=message, blocking_run=blocking_run), 409
def render_index(
*,
form_values: dict[str, dict[str, list[str]]] | None = None,
@@ -80,6 +137,7 @@ def create_app(settings: Settings | None = None) -> Flask:
"index.html",
groups=groups,
latest=latest,
live_states=live_states(groups, latest),
form_values=form_values or {},
input_errors=input_errors or {},
),
@@ -90,6 +148,14 @@ def create_app(settings: Settings | None = None) -> Flask:
def index():
return render_index()
@app.get("/tasks/state")
def tasks_state():
groups = catalog.groups()
latest = {("group", group.id): store.last_run("group", group.id) for group in groups}
response = app.json.response(live_states(groups, latest))
response.headers["Cache-Control"] = "no-store"
return response
@app.post("/tasks/<path:task_id>/run")
def run_task(task_id: str):
task = catalog.task(task_id)
@@ -114,15 +180,12 @@ def create_app(settings: Settings | None = None) -> Flask:
input_errors={task.id: errors},
status=400,
)
run, _ = manager.run_task(task, input_data, uploads)
try:
run, _ = manager.run_task(task, input_data, uploads)
except RuntimeError as error:
return unavailable(error)
if request.accept_mimetypes.best == "application/json":
return {
"artifacts_url": url_for("run_artifacts", run_id=run.id),
"detail_url": url_for("run_detail", run_id=run.id),
"output_url": url_for("run_output", run_id=run.id),
"run_id": run.id,
"state_url": url_for("run_state", run_id=run.id),
}, 202
return run_links(run), 202
return redirect(url_for("run_detail", run_id=run.id))
@app.post("/groups/<path:group_id>/run")
@@ -130,7 +193,12 @@ def create_app(settings: Settings | None = None) -> Flask:
group = catalog.group(group_id)
if group is None or not group.can_run_as_group:
abort(404)
run, _ = manager.run_group(group)
try:
run, _ = manager.run_group(group)
except RuntimeError as error:
return unavailable(error)
if request.accept_mimetypes.best == "application/json":
return run_links(run), 202
return redirect(url_for("run_detail", run_id=run.id))
@app.get("/runs/<int:run_id>")
@@ -147,6 +215,10 @@ def create_app(settings: Settings | None = None) -> Flask:
output=output,
output_offset=offset,
execution_task=execution_task,
recovery_run=run_links(store.recovery_for(run.id)),
progress=manager.progress(run),
display_status=run_links(run)["display_status"],
live_states={"tasks": {execution_task.id: task_state(execution_task)} if execution_task else {}, "groups": {}},
)
def execution_replay_task(run):
@@ -190,9 +262,14 @@ def create_app(settings: Settings | None = None) -> Flask:
except OSError:
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream)
executed, created = manager.run_task(task, validated, uploads)
try:
executed, created = manager.run_task(task, validated, uploads)
except RuntimeError as error:
return unavailable(error)
if created and executed.status == "queued":
manager.discard_review_uploads(run.id)
if request.accept_mimetypes.best == "application/json":
return run_links(executed), 202
return redirect(url_for("run_detail", run_id=executed.id))
@app.post("/runs/<int:run_id>/cancel")
@@ -216,6 +293,9 @@ def create_app(settings: Settings | None = None) -> Flask:
return {
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
"status": run.status,
"display_status": run_links(run)["display_status"],
"recovery_of_run_id": run.recovery_of_run_id,
"progress": manager.progress(run),
}
def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
@@ -299,8 +379,10 @@ def create_app(settings: Settings | None = None) -> Flask:
if run is None:
abort(404)
return {
"status": run.status,
**run_links(run),
"progress": manager.progress(run),
"finished_at": run.finished_at,
"recovery_run": run_links(store.recovery_for(run.id)),
"exit_code": run.exit_code,
"cancel_requested_at": run.cancel_requested_at,
}
@@ -316,4 +398,12 @@ def create_app(settings: Settings | None = None) -> Flask:
runs=store.history(target_kind, target_path),
)
try:
manager.startup()
manager.cleanup_transient_files()
scheduler.start()
except BaseException:
scheduler.stop()
manager.shutdown(timeout=5)
raise
return app
+65
View File
@@ -0,0 +1,65 @@
"""Execute task code in isolation, optionally entering its embedded recovery hook."""
import os
import runpy
import sys
from pathlib import Path
from urllib.error import HTTPError, URLError
from . import task_runtime as runtime
def service_unavailable(error: Exception) -> bool:
"""Retry connection failures, not authentication, invalid data or task bugs."""
while error is not None:
if isinstance(error, HTTPError):
return error.code in {408, 502, 503, 504}
if isinstance(error, (ConnectionError, TimeoutError)):
return True
if isinstance(error, URLError):
return isinstance(error.reason, (OSError, TimeoutError))
sqlstate = getattr(error, "sqlstate", None)
if sqlstate and (sqlstate.startswith("08") or sqlstate in {"57P01", "57P03"}):
return True
if type(error).__module__.startswith("psycopg") and type(error).__name__ == "OperationalError":
text = str(error).casefold()
return any(reason in text for reason in ("connection refused", "timeout expired", "could not translate host name", "server closed the connection"))
error = error.__cause__
return False
def main() -> None:
path = Path(sys.argv[1]).resolve()
sys.path.insert(0, str(path.parent))
sys.argv = [str(path)]
runtime.initialize()
try:
if os.environ.get("SERVER_MAINTENANCE_RECOVER") == "1":
namespace = runpy.run_path(str(path), run_name="__task_recovery__")
recover = namespace.get("recover")
if not callable(recover):
raise RuntimeError("This task no longer provides its recovery function")
print("Recovering interrupted task; verifying current state.", flush=True)
attempts = 0
while True:
try:
recover()
break
except Exception as error:
if not service_unavailable(error):
raise
attempts += 1
delay = min(30, 2 ** min(attempts, 5))
message = f"Recovery waiting for an unavailable service; retry in {delay} seconds."
runtime.progress(message, force=True)
print(message, flush=True)
runtime.sleep(delay)
else:
runpy.run_path(str(path), run_name="__main__")
except runtime.TaskPaused:
print("Paused at a safe checkpoint for recovery after restart.", flush=True)
raise SystemExit(runtime.PAUSED_EXIT_CODE)
if __name__ == "__main__":
main()