Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -3,15 +3,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
|
||||
from waitress import serve
|
||||
from waitress import create_server
|
||||
|
||||
from .web import create_app
|
||||
|
||||
|
||||
def main() -> None:
|
||||
stopping = False
|
||||
requested_at = None
|
||||
|
||||
def request_shutdown(_signal, _frame):
|
||||
nonlocal stopping, requested_at
|
||||
stopping = True
|
||||
if requested_at is None:
|
||||
requested_at = time.monotonic()
|
||||
|
||||
signal.signal(signal.SIGTERM, request_shutdown)
|
||||
signal.signal(signal.SIGINT, request_shutdown)
|
||||
app, server = None, None
|
||||
try:
|
||||
app = create_app()
|
||||
server = create_server(
|
||||
app,
|
||||
host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")),
|
||||
)
|
||||
thread = threading.Thread(target=server.run, name="http-server", daemon=True)
|
||||
thread.start()
|
||||
while not stopping and thread.is_alive():
|
||||
time.sleep(.1)
|
||||
finally:
|
||||
print("Shutdown requested; pausing tasks at safe checkpoints.", flush=True)
|
||||
if app is not None:
|
||||
app.extensions["background_scheduler"].stop()
|
||||
manager = app.extensions["run_manager"]
|
||||
elapsed = time.monotonic() - requested_at if requested_at is not None else 0
|
||||
manager.shutdown(timeout=max(0, manager.settings.shutdown_grace_seconds - elapsed))
|
||||
if server is not None:
|
||||
server.task_dispatcher.shutdown(timeout=5)
|
||||
server.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve(
|
||||
create_app(),
|
||||
host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")),
|
||||
)
|
||||
main()
|
||||
@@ -17,6 +17,7 @@ class Settings:
|
||||
timezone_name: str | None
|
||||
allowed_proxy_ips: frozenset[str]
|
||||
cancel_grace_seconds: float
|
||||
shutdown_grace_seconds: float = 110
|
||||
|
||||
@property
|
||||
def database_path(self) -> Path:
|
||||
@@ -55,4 +56,5 @@ class Settings:
|
||||
cancel_grace_seconds=float(
|
||||
os.environ.get("SERVER_MAINTENANCE_CANCEL_GRACE_SECONDS", "10")
|
||||
),
|
||||
shutdown_grace_seconds=float(os.environ.get("SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS", "110")),
|
||||
)
|
||||
@@ -39,6 +39,8 @@ class Task:
|
||||
wait_for_result: bool = False
|
||||
download_artifacts: bool = False
|
||||
execution: "ExecutionMode | None" = None
|
||||
recoverable: bool = False
|
||||
resources: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -190,6 +192,7 @@ class TaskCatalog:
|
||||
for filename in ordered_filenames:
|
||||
path = paths_by_filename[filename]
|
||||
declaration = declarations_by_filename.get(path.name)
|
||||
recoverable, resources = read_runtime_declaration(path)
|
||||
tasks.append(
|
||||
Task(
|
||||
id=f"{group_path.name}/{path.name}",
|
||||
@@ -210,6 +213,8 @@ class TaskCatalog:
|
||||
wait_for_result=declaration.wait_for_result if declaration else False,
|
||||
download_artifacts=declaration.download_artifacts if declaration else False,
|
||||
execution=declaration.execution if declaration else None,
|
||||
recoverable=recoverable,
|
||||
resources=(f"group:{group_path.name}", *resources),
|
||||
)
|
||||
)
|
||||
return tasks
|
||||
@@ -294,6 +299,23 @@ def read_description(path: Path) -> str | None:
|
||||
)
|
||||
|
||||
|
||||
def read_runtime_declaration(path: Path) -> tuple[bool, tuple[str, ...]]:
|
||||
"""Discover the opt-in hook and literal resource names without importing code."""
|
||||
try:
|
||||
module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
except (OSError, SyntaxError, UnicodeDecodeError):
|
||||
return False, ()
|
||||
recoverable = any(isinstance(node, ast.FunctionDef) and node.name == "recover" for node in module.body)
|
||||
resources = ()
|
||||
for node in module.body:
|
||||
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "RESOURCES" for t in node.targets):
|
||||
value = ast.literal_eval(node.value)
|
||||
if not isinstance(value, (tuple, list)) or any(not isinstance(x, str) or not re.fullmatch(r"[a-z][a-z0-9_-]*", x) for x in value):
|
||||
raise ValueError(f"RESOURCES in {path} must be literal resource names")
|
||||
resources = tuple(dict.fromkeys(value))
|
||||
return recoverable, resources
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GroupManifest:
|
||||
title: str | None
|
||||
|
||||
+245
-47
@@ -10,6 +10,7 @@ import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -17,6 +18,7 @@ from .config import Settings
|
||||
from .discovery import BackgroundTask, Group, Task, TaskCatalog
|
||||
from .inputs import PendingUpload
|
||||
from .store import Run, RunStore
|
||||
from .task_runtime import PAUSED_EXIT_CODE, atomic_json
|
||||
|
||||
|
||||
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
|
||||
@@ -24,6 +26,20 @@ ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
|
||||
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
|
||||
|
||||
|
||||
class TaskConflictError(RuntimeError):
|
||||
def __init__(self, run: Run):
|
||||
self.run = run
|
||||
super().__init__(f"Run {run.id} is using resources required by this task.")
|
||||
|
||||
|
||||
def kill_remaining_group(process: subprocess.Popen) -> None:
|
||||
# The leader can already be gone while a transfer subprocess is still alive.
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
class RunManager:
|
||||
def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None:
|
||||
self.settings = settings
|
||||
@@ -32,7 +48,9 @@ class RunManager:
|
||||
self._wake_events: dict[str, threading.Event] = {}
|
||||
self._workers: dict[str, threading.Thread] = {}
|
||||
self._processes: dict[int, subprocess.Popen[bytes]] = {}
|
||||
self._background_processes: set[subprocess.Popen[bytes]] = set()
|
||||
self._lock = threading.Lock()
|
||||
self._closing = threading.Event()
|
||||
|
||||
def run_task(
|
||||
self,
|
||||
@@ -40,14 +58,20 @@ class RunManager:
|
||||
input_data: dict[str, str | int | list[str]] | None = None,
|
||||
uploads: Mapping[str, PendingUpload] | None = None,
|
||||
) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down; try again after it restarts.")
|
||||
sensitive_names = {field.name for field in task.inputs if field.sensitive}
|
||||
stored_input = self._redact_input(input_data or {}, sensitive_names)
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="task",
|
||||
target_path=task.id,
|
||||
queue_group=task.group_id,
|
||||
input_data=stored_input,
|
||||
)
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
blocker = self.task_status(task)["blocked_by"]
|
||||
if blocker:
|
||||
raise TaskConflictError(blocker)
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
|
||||
)
|
||||
if created:
|
||||
try:
|
||||
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
||||
@@ -56,9 +80,11 @@ class RunManager:
|
||||
run = self.store.set_input(run.id, stored_input)
|
||||
if sensitive_names:
|
||||
self._write_execution_input(run.id, execution_input)
|
||||
self._save_work_input(run, execution_input)
|
||||
except (OSError, ValueError) as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
status="failed",
|
||||
@@ -68,15 +94,99 @@ class RunManager:
|
||||
return run, created
|
||||
|
||||
def run_group(self, group: Group) -> tuple[Run, bool]:
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="group",
|
||||
target_path=group.id,
|
||||
queue_group=group.id,
|
||||
)
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
|
||||
resources = resources or (f"group:{group.id}",)
|
||||
with self._lock:
|
||||
for active in self.store.active_runs():
|
||||
if active.target_kind == "group" and active.target_path == group.id:
|
||||
return active, False
|
||||
if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)):
|
||||
raise TaskConflictError(active)
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="group", target_path=group.id, queue_group=group.id, resources=resources,
|
||||
)
|
||||
if created:
|
||||
self._wake_worker(group.id)
|
||||
return run, created
|
||||
|
||||
def startup(self) -> None:
|
||||
"""Recover only runs with the durable input/context written by this runner."""
|
||||
self.store.mark_interrupted()
|
||||
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
|
||||
for old in self.store.interrupted_without_recovery():
|
||||
work = self._work_root(old)
|
||||
if not (work / "input.json").is_file():
|
||||
continue
|
||||
task = tasks.get(old.target_path)
|
||||
if task is None:
|
||||
continue
|
||||
if old.started_at and not task.recoverable:
|
||||
continue
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
input_data=json.loads(old.input_json), resources=task.resources,
|
||||
recovery_of_run_id=old.id,
|
||||
)
|
||||
if created:
|
||||
self._wake_worker(run.queue_group)
|
||||
for run in self.store.active_runs():
|
||||
if run.parent_run_id is None:
|
||||
self._wake_worker(run.queue_group)
|
||||
|
||||
def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict:
|
||||
active = active_runs if active_runs is not None else self.store.active_runs()
|
||||
own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None)
|
||||
resources = set(task.resources or (f"group:{task.group_id}",))
|
||||
blocker = next((run for run in active if run is not own and run.parent_run_id is None
|
||||
and resources.intersection(json.loads(run.resources_json))), None)
|
||||
return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""}
|
||||
|
||||
def progress(self, run: Run) -> str:
|
||||
try:
|
||||
data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text())
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
return str(data.get("text", ""))
|
||||
|
||||
def shutdown(self, timeout: float | None = None) -> None:
|
||||
self._closing.set()
|
||||
for event in self._wake_events.values():
|
||||
event.set()
|
||||
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
|
||||
with self._lock:
|
||||
processes = [*self._processes.values(), *self._background_processes]
|
||||
for process in processes:
|
||||
if process.poll() is None:
|
||||
try:
|
||||
os.kill(process.pid, signal.SIGUSR1)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
while any(process.poll() is None for process in processes) and time.monotonic() < deadline:
|
||||
time.sleep(.05)
|
||||
for process in processes:
|
||||
kill_remaining_group(process)
|
||||
for process in processes:
|
||||
process.wait(timeout=5)
|
||||
for worker in self._workers.values():
|
||||
worker.join(timeout=max(0, deadline - time.monotonic()))
|
||||
|
||||
def _work_root(self, run: Run) -> Path:
|
||||
while run.recovery_of_run_id is not None:
|
||||
previous = self.store.get(run.recovery_of_run_id)
|
||||
if previous is None:
|
||||
raise RuntimeError("The original recovery run no longer exists")
|
||||
run = previous
|
||||
return self.settings.state_root / "runs" / f"{run.id}.work"
|
||||
|
||||
def _save_work_input(self, run: Run, values: dict) -> None:
|
||||
atomic_json(self._work_root(run) / "input.json", values)
|
||||
|
||||
def _discard_work_input(self, run: Run) -> None:
|
||||
for name in ("input.json", "input.json.tmp"):
|
||||
(self._work_root(run) / name).unlink(missing_ok=True)
|
||||
|
||||
def request_cancel(self, run_id: int) -> Run | None:
|
||||
run = self.store.request_cancel(run_id)
|
||||
if run is None:
|
||||
@@ -90,6 +200,7 @@ class RunManager:
|
||||
if completed is not None and completed.status == "cancelled":
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(completed)
|
||||
return completed
|
||||
|
||||
def artifacts(self, run_id: int) -> list[str]:
|
||||
@@ -165,9 +276,10 @@ class RunManager:
|
||||
cutoff = time.time() - artifact_max_age_seconds
|
||||
if runs_root.is_dir():
|
||||
lifetimes = self.store.run_lifetimes()
|
||||
protected = self._recovery_context_ids()
|
||||
for path in runs_root.iterdir():
|
||||
run_id_text, separator, kind = path.name.partition(".")
|
||||
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
|
||||
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink():
|
||||
continue
|
||||
run_id = int(run_id_text)
|
||||
if run_id not in lifetimes:
|
||||
@@ -175,7 +287,7 @@ class RunManager:
|
||||
run = self.store.get(run_id)
|
||||
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
|
||||
status, finished_at = lifetimes[run_id]
|
||||
if status in {"queued", "running"}:
|
||||
if status in {"queued", "running"} or run_id in protected:
|
||||
continue
|
||||
if kind in {"input.json", "execution-input.json"} and path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
@@ -193,8 +305,50 @@ class RunManager:
|
||||
continue
|
||||
if completed_at < cutoff:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif kind == "work" and path.is_dir():
|
||||
(path / "input.json").unlink(missing_ok=True)
|
||||
if path.stat().st_mtime < time.time() - 14 * 86400:
|
||||
shutil.rmtree(path)
|
||||
elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400:
|
||||
path.unlink()
|
||||
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
|
||||
|
||||
def _recovery_context_ids(self) -> set[int]:
|
||||
protected = set()
|
||||
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
|
||||
for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]:
|
||||
task = tasks.get(run.target_path)
|
||||
if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()):
|
||||
continue
|
||||
while True:
|
||||
protected.add(run.id)
|
||||
if run.recovery_of_run_id is None:
|
||||
break
|
||||
run = self.store.get(run.recovery_of_run_id)
|
||||
if run is None:
|
||||
break
|
||||
return protected
|
||||
|
||||
def _promote_review_uploads(self, run: Run, task: Task) -> Run:
|
||||
if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs):
|
||||
return run
|
||||
values = json.loads(run.input_json)
|
||||
if values.get(task.execution.field.name) != task.execution.dry_run_value:
|
||||
return run
|
||||
origin = self._work_root(run).name.split(".", 1)[0]
|
||||
old_root = self._upload_root(int(origin))
|
||||
for field in task.inputs:
|
||||
if field.type != "file" or not values.get(field.name):
|
||||
continue
|
||||
source = Path(values[field.name])
|
||||
if source.parent != old_root or source.is_symlink() or not source.is_file():
|
||||
raise RuntimeError("The original uploaded review input is no longer available")
|
||||
destination = self._upload_root(run.id) / source.name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
shutil.copy2(source, destination)
|
||||
values[field.name] = str(destination)
|
||||
return self.store.set_input(run.id, values)
|
||||
|
||||
@staticmethod
|
||||
def _keep_diagnostic_reports(root: Path) -> None:
|
||||
try:
|
||||
@@ -254,24 +408,31 @@ class RunManager:
|
||||
|
||||
def run_background_task(self, task: BackgroundTask) -> None:
|
||||
"""Run a declared internal task without creating high-volume run history."""
|
||||
|
||||
if self._closing.is_set():
|
||||
return
|
||||
log_path = self.settings.state_root / "background" / f"{task.id}.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
environment = self._environment(task.group_id)
|
||||
environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work")
|
||||
try:
|
||||
with log_path.open("wb") as log_file:
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, str(task.path)],
|
||||
cwd=task.path.parent,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=environment,
|
||||
start_new_session=True,
|
||||
)
|
||||
exit_code = process.wait()
|
||||
if exit_code:
|
||||
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
|
||||
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
|
||||
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
|
||||
)
|
||||
self._background_processes.add(process)
|
||||
try:
|
||||
exit_code = process.wait()
|
||||
if exit_code not in (0, PAUSED_EXIT_CODE):
|
||||
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
|
||||
finally:
|
||||
kill_remaining_group(process)
|
||||
with self._lock:
|
||||
self._background_processes.discard(process)
|
||||
except OSError as error:
|
||||
log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8")
|
||||
|
||||
@@ -291,13 +452,17 @@ class RunManager:
|
||||
event.set()
|
||||
|
||||
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
|
||||
while True:
|
||||
while not self._closing.is_set():
|
||||
event.clear()
|
||||
run = self.store.claim_next(group_id)
|
||||
if run is None:
|
||||
event.wait()
|
||||
continue
|
||||
self._execute_root_run(run)
|
||||
try:
|
||||
self._execute_root_run(run)
|
||||
finally:
|
||||
for waiting in self._wake_events.values():
|
||||
waiting.set()
|
||||
|
||||
def _execute_root_run(self, run: Run) -> None:
|
||||
if run.target_kind == "task":
|
||||
@@ -355,6 +520,8 @@ class RunManager:
|
||||
*,
|
||||
parent_run_id: int | None = None,
|
||||
) -> Run:
|
||||
if self._closing.is_set():
|
||||
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
@@ -366,12 +533,20 @@ class RunManager:
|
||||
artifact_root = self._artifact_root(run.id)
|
||||
artifact_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
if not input_path.exists():
|
||||
work = self._work_root(run)
|
||||
saved_input = work / "input.json"
|
||||
if run.recovery_of_run_id is not None:
|
||||
input_path = saved_input
|
||||
elif not input_path.exists():
|
||||
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
|
||||
input_path.write_text(run.input_json + "\n", encoding="utf-8")
|
||||
input_path.chmod(0o600)
|
||||
if not saved_input.exists():
|
||||
self._save_work_input(run, json.loads(input_path.read_text()))
|
||||
except OSError as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
@@ -381,24 +556,33 @@ class RunManager:
|
||||
environment = self._environment(task.group_id)
|
||||
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
|
||||
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
|
||||
environment["SERVER_MAINTENANCE_WORK"] = str(work)
|
||||
environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json")
|
||||
if run.recovery_of_run_id is not None:
|
||||
old = self.store.get(run.recovery_of_run_id)
|
||||
if old and old.started_at:
|
||||
environment["SERVER_MAINTENANCE_RECOVER"] = "1"
|
||||
self.store.set_log_path(run.id, str(log_path))
|
||||
|
||||
try:
|
||||
with log_path.open("wb") as log_file:
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, str(task.path)],
|
||||
cwd=task.path.parent,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=environment,
|
||||
start_new_session=True,
|
||||
)
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
|
||||
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
|
||||
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
|
||||
)
|
||||
self._processes[run.id] = process
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
self._terminate_active_process(run.id)
|
||||
exit_code = process.wait()
|
||||
except OSError as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._discard_work_input(run)
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
@@ -408,7 +592,9 @@ class RunManager:
|
||||
)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._processes.pop(run.id, None)
|
||||
owned = self._processes.pop(run.id, None)
|
||||
if owned is not None:
|
||||
kill_remaining_group(owned)
|
||||
self._delete_execution_input(run.id)
|
||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||
|
||||
@@ -416,6 +602,7 @@ class RunManager:
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
self._delete_uploads(run.id)
|
||||
self._discard_work_input(run)
|
||||
self._keep_diagnostic_reports(artifact_root)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
@@ -423,21 +610,31 @@ class RunManager:
|
||||
exit_code=exit_code,
|
||||
log_path=str(log_path),
|
||||
)
|
||||
completed = self.store.finish(
|
||||
run.id,
|
||||
status="succeeded" if exit_code == 0 else "failed",
|
||||
exit_code=exit_code,
|
||||
log_path=str(log_path),
|
||||
)
|
||||
if not self.review_upload_paths(completed, task):
|
||||
if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0):
|
||||
return self.store.finish(
|
||||
run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path),
|
||||
message="Application shutdown; saved state will be checked by recovery after restart.",
|
||||
)
|
||||
status = "succeeded" if exit_code == 0 else "failed"
|
||||
message = None
|
||||
if status == "succeeded":
|
||||
try:
|
||||
run = self._promote_review_uploads(run, task)
|
||||
except (RuntimeError, OSError) as error:
|
||||
status, message = "failed", f"Could not preserve review input: {error}"
|
||||
candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat())
|
||||
if not self.review_upload_paths(candidate, task):
|
||||
self._delete_uploads(run.id)
|
||||
if completed.status != "succeeded":
|
||||
if status != "succeeded":
|
||||
self._keep_diagnostic_reports(artifact_root)
|
||||
else:
|
||||
try:
|
||||
artifact_root.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
completed = self.store.finish(run.id, status=status, exit_code=exit_code,
|
||||
message=message, log_path=str(log_path))
|
||||
self._discard_work_input(run)
|
||||
self.cleanup_transient_files()
|
||||
return completed
|
||||
|
||||
@@ -447,6 +644,7 @@ class RunManager:
|
||||
environment = {
|
||||
"PATH": "/usr/local/bin:/usr/bin:/bin",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONPATH": str(Path(__file__).resolve().parent.parent),
|
||||
"SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root),
|
||||
"SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root),
|
||||
"SERVER_MAINTENANCE_STATE": str(task_state),
|
||||
|
||||
@@ -30,6 +30,8 @@ class Run:
|
||||
log_path: str | None
|
||||
message: str | None
|
||||
input_json: str
|
||||
recovery_of_run_id: int | None = None
|
||||
resources_json: str = "[]"
|
||||
|
||||
|
||||
def now() -> str:
|
||||
@@ -72,6 +74,10 @@ class RunStore:
|
||||
connection.execute(
|
||||
"ALTER TABLE runs ADD COLUMN input_json TEXT NOT NULL DEFAULT '{}'"
|
||||
)
|
||||
if "recovery_of_run_id" not in columns:
|
||||
connection.execute("ALTER TABLE runs ADD COLUMN recovery_of_run_id INTEGER REFERENCES runs(id)")
|
||||
if "resources_json" not in columns:
|
||||
connection.execute("ALTER TABLE runs ADD COLUMN resources_json TEXT NOT NULL DEFAULT '[]'")
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS runs_queue_index
|
||||
@@ -106,6 +112,8 @@ class RunStore:
|
||||
queue_group: str,
|
||||
trigger: str = "manual",
|
||||
input_data: Mapping[str, str | int | list[str]] | None = None,
|
||||
resources: tuple[str, ...] = (),
|
||||
recovery_of_run_id: int | None = None,
|
||||
) -> tuple[Run, bool]:
|
||||
input_json = json.dumps(input_data or {}, sort_keys=True, separators=(",", ":"))
|
||||
with self._connect() as connection:
|
||||
@@ -126,10 +134,12 @@ class RunStore:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO runs (
|
||||
target_kind, target_path, queue_group, trigger, status, created_at, input_json
|
||||
) VALUES (?, ?, ?, ?, 'queued', ?, ?)
|
||||
target_kind, target_path, queue_group, trigger, status, created_at, input_json,
|
||||
resources_json, recovery_of_run_id
|
||||
) VALUES (?, ?, ?, ?, 'queued', ?, ?, ?, ?)
|
||||
""",
|
||||
(target_kind, target_path, queue_group, trigger, now(), input_json),
|
||||
(target_kind, target_path, queue_group, trigger, now(), input_json,
|
||||
json.dumps(resources), recovery_of_run_id),
|
||||
)
|
||||
return self.get(cursor.lastrowid, connection=connection), True
|
||||
|
||||
@@ -162,6 +172,10 @@ class RunStore:
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
requested = set(json.loads(row["resources_json"]))
|
||||
for active in connection.execute("SELECT resources_json FROM runs WHERE status='running' AND parent_run_id IS NULL"):
|
||||
if requested & set(json.loads(active["resources_json"])):
|
||||
return None
|
||||
run_id = row["id"]
|
||||
connection.execute(
|
||||
"UPDATE runs SET status = 'running', started_at = ? WHERE id = ?",
|
||||
@@ -263,6 +277,26 @@ class RunStore:
|
||||
for row in connection.execute("SELECT id, status, finished_at FROM runs")
|
||||
}
|
||||
|
||||
def active_runs(self) -> list[Run]:
|
||||
with self._connect() as connection:
|
||||
return [self._run(row) for row in connection.execute(
|
||||
"SELECT * FROM runs WHERE status IN ('queued','running') ORDER BY id"
|
||||
)]
|
||||
|
||||
def interrupted_without_recovery(self) -> list[Run]:
|
||||
with self._connect() as connection:
|
||||
return [self._run(row) for row in connection.execute(
|
||||
"""SELECT r.* FROM runs r WHERE r.status='interrupted' AND r.target_kind='task'
|
||||
AND r.cancel_requested_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM runs child WHERE child.recovery_of_run_id=r.id)
|
||||
ORDER BY r.id"""
|
||||
)]
|
||||
|
||||
def recovery_for(self, run_id: int) -> Run | None:
|
||||
with self._connect() as connection:
|
||||
row = connection.execute("SELECT * FROM runs WHERE recovery_of_run_id=? ORDER BY id DESC LIMIT 1", (run_id,)).fetchone()
|
||||
return self._run(row) if row else None
|
||||
|
||||
def set_log_path(self, run_id: int, path: str) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id))
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Cooperative task checkpoints and optional plain-text progress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PAUSED_EXIT_CODE = 75
|
||||
CHECKPOINT_SECONDS = 60
|
||||
_pause_requested = False
|
||||
_defer = 0
|
||||
_state: dict = {}
|
||||
_last_save = 0.0
|
||||
_last_progress = 0.0
|
||||
_previous_clean = False
|
||||
|
||||
|
||||
class TaskPaused(BaseException):
|
||||
"""Unwind at a safe boundary without treating a pause as an ordinary failure."""
|
||||
|
||||
|
||||
def work_directory() -> Path | None:
|
||||
value = os.environ.get("SERVER_MAINTENANCE_WORK")
|
||||
return Path(value) if value else None
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
with temporary.open("w", encoding="utf-8") as stream:
|
||||
temporary.chmod(0o600)
|
||||
json.dump(value, stream, ensure_ascii=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
temporary.replace(path)
|
||||
descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def initialize() -> None:
|
||||
global _state, _previous_clean, _pause_requested, _last_save
|
||||
_pause_requested = False
|
||||
root = work_directory()
|
||||
if root is not None:
|
||||
try:
|
||||
saved = json.loads((root / "checkpoint.json").read_text())
|
||||
except FileNotFoundError:
|
||||
saved = {}
|
||||
_state = saved.get("data", {})
|
||||
_previous_clean = bool(saved.get("clean_shutdown"))
|
||||
_write_checkpoint(False)
|
||||
_last_save = time.monotonic()
|
||||
signal.signal(signal.SIGUSR1, request_pause)
|
||||
|
||||
|
||||
def request_pause(_signal=None, _frame=None) -> None:
|
||||
global _pause_requested
|
||||
_pause_requested = True
|
||||
|
||||
|
||||
def pause_requested() -> bool:
|
||||
return _pause_requested and work_directory() is not None
|
||||
|
||||
|
||||
def checkpoint_data() -> dict:
|
||||
return _state if work_directory() is not None else {}
|
||||
|
||||
|
||||
def was_clean_shutdown() -> bool:
|
||||
return _previous_clean
|
||||
|
||||
|
||||
def _write_checkpoint(clean: bool) -> None:
|
||||
root = work_directory()
|
||||
if root is not None:
|
||||
atomic_json(root / "checkpoint.json", {
|
||||
"version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(),
|
||||
})
|
||||
|
||||
|
||||
def checkpoint(data: dict | None = None, *, force: bool = False) -> None:
|
||||
"""Keep the latest state in memory; persist ordinary repeatable work once a minute."""
|
||||
global _state, _last_save
|
||||
if data is not None:
|
||||
_state = data
|
||||
if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS:
|
||||
_write_checkpoint(False)
|
||||
_last_save = time.monotonic()
|
||||
check_stop()
|
||||
|
||||
|
||||
def check_stop() -> None:
|
||||
if _pause_requested and work_directory() is not None and not _defer:
|
||||
_write_checkpoint(True)
|
||||
raise TaskPaused()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def critical_section():
|
||||
"""Finish a small consistency-sensitive operation before observing a pause."""
|
||||
global _defer
|
||||
check_stop()
|
||||
_defer += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_defer -= 1
|
||||
check_stop()
|
||||
|
||||
|
||||
def sleep(seconds: float) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while (remaining := deadline - time.monotonic()) > 0:
|
||||
check_stop()
|
||||
time.sleep(min(remaining, 0.25))
|
||||
check_stop()
|
||||
|
||||
|
||||
def progress(text: str, *, force: bool = False) -> None:
|
||||
global _last_progress
|
||||
root = work_directory()
|
||||
if root is not None and (force or time.monotonic() - _last_progress >= 1):
|
||||
path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json")))
|
||||
atomic_json(path, {"text": text, "updated_at": time.time()})
|
||||
_last_progress = time.monotonic()
|
||||
|
||||
|
||||
class Progress:
|
||||
"""Item-count estimates, independently usable for several named steps."""
|
||||
|
||||
def __init__(self, totals: dict[str, int]):
|
||||
self.totals = totals
|
||||
self.done = dict.fromkeys(totals, 0)
|
||||
self.started: dict[str, float] = {}
|
||||
self.baseline: dict[str, int] = {}
|
||||
|
||||
def update(self, label: str, done: int, *, force: bool = False) -> None:
|
||||
self.started.setdefault(label, time.monotonic())
|
||||
self.baseline.setdefault(label, done)
|
||||
self.done[label] = done
|
||||
lines = []
|
||||
for name, total in self.totals.items():
|
||||
count = self.done[name]
|
||||
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
|
||||
line = f"{name}: {percent}%"
|
||||
processed = count - self.baseline.get(name, 0)
|
||||
if processed > 0 and count < total and name in self.started:
|
||||
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
|
||||
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
|
||||
line += f", ETA {estimate}"
|
||||
lines.append(line)
|
||||
progress("\n".join(lines), force=force)
|
||||
+105
-15
@@ -14,7 +14,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u
|
||||
from .config import Settings
|
||||
from .discovery import TaskCatalog
|
||||
from .inputs import PendingUpload, validate_inputs, validate_uploads
|
||||
from .runner import RunManager
|
||||
from .runner import RunManager, TaskConflictError
|
||||
from .scheduler import BackgroundScheduler
|
||||
from .store import RunStore
|
||||
|
||||
@@ -24,11 +24,8 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
catalog = TaskCatalog(settings.task_root)
|
||||
store = RunStore(settings.database_path)
|
||||
store.initialize()
|
||||
store.mark_interrupted()
|
||||
manager = RunManager(settings, catalog, store)
|
||||
manager.cleanup_transient_files()
|
||||
scheduler = BackgroundScheduler(catalog, manager)
|
||||
scheduler.start()
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
app = Flask(
|
||||
@@ -62,6 +59,66 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
return "—"
|
||||
return datetime.fromisoformat(value).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
||||
|
||||
def run_links(run):
|
||||
if run is None:
|
||||
return None
|
||||
display_status = run.status
|
||||
if run.recovery_of_run_id is not None and run.status in {"queued", "running"}:
|
||||
display_status = "Recovery queued" if run.status == "queued" else "recovering"
|
||||
return {
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
"display_status": display_status,
|
||||
"recovery_of_run_id": run.recovery_of_run_id,
|
||||
"artifacts_url": url_for("run_artifacts", run_id=run.id),
|
||||
"detail_url": url_for("run_detail", run_id=run.id),
|
||||
"output_url": url_for("run_output", run_id=run.id),
|
||||
"state_url": url_for("run_state", run_id=run.id),
|
||||
}
|
||||
|
||||
def task_state(task, active_runs=None):
|
||||
state = manager.task_status(task, active_runs=active_runs)
|
||||
return {
|
||||
"active_run": run_links(state["active_run"]),
|
||||
"blocked_by": run_links(state["blocked_by"]),
|
||||
"progress": state["progress"],
|
||||
}
|
||||
|
||||
def live_states(groups, latest):
|
||||
active = store.active_runs()
|
||||
tasks = {task.id: task_state(task, active) for group in groups for task in group.tasks}
|
||||
group_states = {}
|
||||
for group in groups:
|
||||
run = latest[("group", group.id)]
|
||||
active = run if run and run.status in {"queued", "running"} else None
|
||||
blocker = next((
|
||||
tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
|
||||
for task in group.tasks
|
||||
if tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
|
||||
), None)
|
||||
group_states[group.id] = {
|
||||
"active_run": run_links(active),
|
||||
"blocked_by": None if active else blocker,
|
||||
"progress": manager.progress(active) if active else "",
|
||||
}
|
||||
return {"tasks": tasks, "groups": group_states}
|
||||
|
||||
def unavailable(error):
|
||||
if isinstance(error, TaskConflictError):
|
||||
return task_conflict(error)
|
||||
message = str(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {"error": message}, 503
|
||||
return render_template("unavailable.html", message=message, blocking_run=None), 503
|
||||
|
||||
@app.errorhandler(TaskConflictError)
|
||||
def task_conflict(error):
|
||||
blocking_run = run_links(error.run)
|
||||
message = f"Task blocked by run #{error.run.id} ({blocking_run['display_status']})."
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {"error": message, "blocked_by": blocking_run}, 409
|
||||
return render_template("unavailable.html", message=message, blocking_run=blocking_run), 409
|
||||
|
||||
def render_index(
|
||||
*,
|
||||
form_values: dict[str, dict[str, list[str]]] | None = None,
|
||||
@@ -80,6 +137,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
"index.html",
|
||||
groups=groups,
|
||||
latest=latest,
|
||||
live_states=live_states(groups, latest),
|
||||
form_values=form_values or {},
|
||||
input_errors=input_errors or {},
|
||||
),
|
||||
@@ -90,6 +148,14 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
def index():
|
||||
return render_index()
|
||||
|
||||
@app.get("/tasks/state")
|
||||
def tasks_state():
|
||||
groups = catalog.groups()
|
||||
latest = {("group", group.id): store.last_run("group", group.id) for group in groups}
|
||||
response = app.json.response(live_states(groups, latest))
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
@app.post("/tasks/<path:task_id>/run")
|
||||
def run_task(task_id: str):
|
||||
task = catalog.task(task_id)
|
||||
@@ -114,15 +180,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
input_errors={task.id: errors},
|
||||
status=400,
|
||||
)
|
||||
run, _ = manager.run_task(task, input_data, uploads)
|
||||
try:
|
||||
run, _ = manager.run_task(task, input_data, uploads)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {
|
||||
"artifacts_url": url_for("run_artifacts", run_id=run.id),
|
||||
"detail_url": url_for("run_detail", run_id=run.id),
|
||||
"output_url": url_for("run_output", run_id=run.id),
|
||||
"run_id": run.id,
|
||||
"state_url": url_for("run_state", run_id=run.id),
|
||||
}, 202
|
||||
return run_links(run), 202
|
||||
return redirect(url_for("run_detail", run_id=run.id))
|
||||
|
||||
@app.post("/groups/<path:group_id>/run")
|
||||
@@ -130,7 +193,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
group = catalog.group(group_id)
|
||||
if group is None or not group.can_run_as_group:
|
||||
abort(404)
|
||||
run, _ = manager.run_group(group)
|
||||
try:
|
||||
run, _ = manager.run_group(group)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return run_links(run), 202
|
||||
return redirect(url_for("run_detail", run_id=run.id))
|
||||
|
||||
@app.get("/runs/<int:run_id>")
|
||||
@@ -147,6 +215,10 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
output=output,
|
||||
output_offset=offset,
|
||||
execution_task=execution_task,
|
||||
recovery_run=run_links(store.recovery_for(run.id)),
|
||||
progress=manager.progress(run),
|
||||
display_status=run_links(run)["display_status"],
|
||||
live_states={"tasks": {execution_task.id: task_state(execution_task)} if execution_task else {}, "groups": {}},
|
||||
)
|
||||
|
||||
def execution_replay_task(run):
|
||||
@@ -190,9 +262,14 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
except OSError:
|
||||
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
|
||||
uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream)
|
||||
executed, created = manager.run_task(task, validated, uploads)
|
||||
try:
|
||||
executed, created = manager.run_task(task, validated, uploads)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if created and executed.status == "queued":
|
||||
manager.discard_review_uploads(run.id)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return run_links(executed), 202
|
||||
return redirect(url_for("run_detail", run_id=executed.id))
|
||||
|
||||
@app.post("/runs/<int:run_id>/cancel")
|
||||
@@ -216,6 +293,9 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
return {
|
||||
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
|
||||
"status": run.status,
|
||||
"display_status": run_links(run)["display_status"],
|
||||
"recovery_of_run_id": run.recovery_of_run_id,
|
||||
"progress": manager.progress(run),
|
||||
}
|
||||
|
||||
def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
|
||||
@@ -299,8 +379,10 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
if run is None:
|
||||
abort(404)
|
||||
return {
|
||||
"status": run.status,
|
||||
**run_links(run),
|
||||
"progress": manager.progress(run),
|
||||
"finished_at": run.finished_at,
|
||||
"recovery_run": run_links(store.recovery_for(run.id)),
|
||||
"exit_code": run.exit_code,
|
||||
"cancel_requested_at": run.cancel_requested_at,
|
||||
}
|
||||
@@ -316,4 +398,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
runs=store.history(target_kind, target_path),
|
||||
)
|
||||
|
||||
try:
|
||||
manager.startup()
|
||||
manager.cleanup_transient_files()
|
||||
scheduler.start()
|
||||
except BaseException:
|
||||
scheduler.stop()
|
||||
manager.shutdown(timeout=5)
|
||||
raise
|
||||
return app
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Execute task code in isolation, optionally entering its embedded recovery hook."""
|
||||
|
||||
import os
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
from . import task_runtime as runtime
|
||||
|
||||
|
||||
def service_unavailable(error: Exception) -> bool:
|
||||
"""Retry connection failures, not authentication, invalid data or task bugs."""
|
||||
while error is not None:
|
||||
if isinstance(error, HTTPError):
|
||||
return error.code in {408, 502, 503, 504}
|
||||
if isinstance(error, (ConnectionError, TimeoutError)):
|
||||
return True
|
||||
if isinstance(error, URLError):
|
||||
return isinstance(error.reason, (OSError, TimeoutError))
|
||||
sqlstate = getattr(error, "sqlstate", None)
|
||||
if sqlstate and (sqlstate.startswith("08") or sqlstate in {"57P01", "57P03"}):
|
||||
return True
|
||||
if type(error).__module__.startswith("psycopg") and type(error).__name__ == "OperationalError":
|
||||
text = str(error).casefold()
|
||||
return any(reason in text for reason in ("connection refused", "timeout expired", "could not translate host name", "server closed the connection"))
|
||||
error = error.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
path = Path(sys.argv[1]).resolve()
|
||||
sys.path.insert(0, str(path.parent))
|
||||
sys.argv = [str(path)]
|
||||
runtime.initialize()
|
||||
try:
|
||||
if os.environ.get("SERVER_MAINTENANCE_RECOVER") == "1":
|
||||
namespace = runpy.run_path(str(path), run_name="__task_recovery__")
|
||||
recover = namespace.get("recover")
|
||||
if not callable(recover):
|
||||
raise RuntimeError("This task no longer provides its recovery function")
|
||||
print("Recovering interrupted task; verifying current state.", flush=True)
|
||||
attempts = 0
|
||||
while True:
|
||||
try:
|
||||
recover()
|
||||
break
|
||||
except Exception as error:
|
||||
if not service_unavailable(error):
|
||||
raise
|
||||
attempts += 1
|
||||
delay = min(30, 2 ** min(attempts, 5))
|
||||
message = f"Recovery waiting for an unavailable service; retry in {delay} seconds."
|
||||
runtime.progress(message, force=True)
|
||||
print(message, flush=True)
|
||||
runtime.sleep(delay)
|
||||
else:
|
||||
runpy.run_path(str(path), run_name="__main__")
|
||||
except runtime.TaskPaused:
|
||||
print("Paused at a safe checkpoint for recovery after restart.", flush=True)
|
||||
raise SystemExit(runtime.PAUSED_EXIT_CODE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user