Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2333
-149
No files matched your search
@@ -131,6 +131,12 @@ jobs:
|
||||
tar -C timepoll -cf - . | docker run --rm -i --workdir /app python:3.12-slim \
|
||||
sh -c 'tar -xf - && pip install --no-cache-dir -r requirements.txt pytest httpx && PYTHONDONTWRITEBYTECODE=1 pytest -q -p no:cacheprovider'
|
||||
|
||||
- name: Test Python Tools
|
||||
if: matrix.image.name == 'python-tools'
|
||||
run: |
|
||||
tar -C python-tools -cf - . | docker run --rm --init -i --workdir /app python:3 \
|
||||
sh -c 'tar -xf - && pip install --no-cache-dir -r requirements.txt pytest && PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app python -m pytest -q -p no:cacheprovider'
|
||||
|
||||
- name: Log in to Gitea container registry
|
||||
run: |
|
||||
printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USERNAME" --password-stdin
|
||||
|
||||
+99
-7
@@ -7,8 +7,9 @@ with `python -m app`.
|
||||
## Control Panel
|
||||
|
||||
The application discovers direct `*.py` files in each non-hidden directory
|
||||
under `/opt/tasks`. Each directory is a task group. Tasks in one group run
|
||||
sequentially; different groups may run concurrently. It records run history in
|
||||
under `/opt/tasks`. Each directory is a task group. A group is exclusive while
|
||||
one of its tasks runs; unrelated groups may run concurrently. Tasks can declare
|
||||
additional shared resources to prevent conflicts across groups. It records run history in
|
||||
SQLite and combined output in log files under `/var/lib/server-maintenance`,
|
||||
and supports cancellation of active task process groups.
|
||||
|
||||
@@ -81,9 +82,10 @@ Use `\n` for a line break or `\n\n` for a paragraph gap in descriptions.
|
||||
|
||||
Each task declaration may contain `title`, `description`, `inputs`,
|
||||
`wait_for_result`, `download_artifacts`, and `execution`. Set
|
||||
`wait_for_result` for short tasks where the form should remain blocked until it
|
||||
reports success or failure. The control panel disables its controls and warns
|
||||
before page unload while such a task runs. `download_artifacts` additionally
|
||||
`wait_for_result` for tasks whose result should appear beside the form instead
|
||||
of immediately opening the run-detail page. Running and conflicting controls
|
||||
are disabled, but navigation is allowed. Opening or reloading the page restores
|
||||
the current status and optional live progress from the server. `download_artifacts` additionally
|
||||
starts the first non-JSON artifact download after a successful browser run;
|
||||
reserve it for files such as archives that the browser should download
|
||||
automatically.
|
||||
@@ -101,8 +103,8 @@ Every input has `name`, `label`, and `type`; `required` defaults to `false`.
|
||||
|
||||
Set `sensitive` to `true` on text or file inputs that must not appear in run
|
||||
history. Text is masked, the database records `[redacted]`, and the original
|
||||
value is available only through a private execution-input file. Uploaded files
|
||||
are likewise retained privately for the run and then removed.
|
||||
value is available only through a private execution-input file. Interrupted
|
||||
recoverable tasks retain that input and their uploads until recovery finishes.
|
||||
|
||||
For review-first tasks, use `execution`. It produces one split button: click
|
||||
the main part to use its current mode, or use the arrow to select the other
|
||||
@@ -135,6 +137,96 @@ task without an entry in the manifest gets an empty JSON object. Uploaded files
|
||||
are stored in a private per-run directory; their absolute paths are provided in
|
||||
the input JSON. Files are removed when the task finishes, fails, or is cancelled,
|
||||
except for successful review uploads retained temporarily for execution as above.
|
||||
Interrupted tasks with recovery support retain their files privately.
|
||||
|
||||
## Shutdown And Recovery
|
||||
|
||||
The Compose example grants 120 seconds for container shutdown. The runner stops
|
||||
accepting work, pauses the background scheduler, and asks all task subprocesses
|
||||
to save and stop at safe boundaries. It waits up to 110 seconds by default, then
|
||||
terminates remaining process groups. The HTTP server gets a final short cleanup
|
||||
window. These are maximum waits; idle or cooperative jobs stop sooner. The
|
||||
Compose example also uses Docker's init process to reap orphaned subprocesses.
|
||||
`SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS` changes the application deadline; keep
|
||||
the container's `stop_grace_period` longer than that deadline.
|
||||
|
||||
Tasks opt in by defining a top-level, synchronous `recover()` function with no
|
||||
arguments in their own Python file. Discovery reads its syntax without importing
|
||||
the task. On startup, a new linked recovery run enters the ordinary task queue
|
||||
and calls that function in an isolated worker process. Original logs and status
|
||||
remain unchanged. Cancelled tasks never recover automatically. Tasks without a
|
||||
hook remain interrupted. Historical runs created before this recovery protocol
|
||||
are not automatically replayed.
|
||||
Recovery waits and retries transient connection failures while services start,
|
||||
with a maximum 30-second interval. Authentication failures and task validation
|
||||
errors are reported rather than automatically repeated.
|
||||
|
||||
Recovery code must inspect current files or remote state before repeating an
|
||||
operation. A lost response does not prove that the operation failed. Save an
|
||||
intent before an irreversible operation and reconcile its outcome after restart.
|
||||
For repeatable scans, a periodic checkpoint is sufficient. The runner cannot
|
||||
serialize Python stacks or reconstruct task-specific state automatically.
|
||||
|
||||
Use `from app import task_runtime as runtime` in task code:
|
||||
|
||||
- `runtime.checkpoint_data()` reads the saved dictionary on recovery.
|
||||
- `runtime.checkpoint(data)` updates the in-memory snapshot and persists ordinary
|
||||
repeatable work at most once a minute. Use `force=True` for important intents,
|
||||
completed results, and other state that must precede an external mutation.
|
||||
- `runtime.check_stop()` observes a pause at a safe boundary, saves the latest
|
||||
snapshot, and unwinds the task. Do not catch its `TaskPaused` exception.
|
||||
Exit code 75 is reserved for this cooperative pause.
|
||||
- `runtime.critical_section()` defers a pause through a small consistency-sensitive
|
||||
action. Do not wrap an entire job or an unbounded transfer in it.
|
||||
- `runtime.sleep(seconds)` observes pauses during waits.
|
||||
- `runtime.was_clean_shutdown()` distinguishes a completed shutdown checkpoint
|
||||
from a periodic checkpoint. External state must still be checked in both cases.
|
||||
- `runtime.work_directory()` is a private directory shared by the interrupted run
|
||||
and its recovery chain. Use it for durable intermediate files, not artifacts.
|
||||
|
||||
Keep the usual `if __name__ == "__main__":` guard. The recovery worker loads the
|
||||
module under a different name, then calls `recover()`; module import must not
|
||||
start work. A recovery hook may call the task's regular implementation with a
|
||||
recovery flag, provided that implementation performs the necessary reconciliation.
|
||||
|
||||
Work directories live at `runs/<original-run-id>.work`. Exact inputs and uploads
|
||||
remain protected while a run is active or awaits recovery. Raw work inputs are
|
||||
removed on completion or cancellation; finished checkpoint directories are
|
||||
retained for 14 days. Do not put passwords or other credentials in checkpoint or
|
||||
progress dictionaries. Read credentials from their normal mounted files.
|
||||
|
||||
### Shared Resources
|
||||
|
||||
Declare optional resource names as a literal tuple at module level:
|
||||
|
||||
```python
|
||||
RESOURCES = ("shared-files",)
|
||||
```
|
||||
|
||||
Two tasks sharing a resource cannot run together, including during automatic
|
||||
recovery. The backend rejects conflicting submissions with HTTP 409 and identifies
|
||||
the blocking run. Every task also holds its group's resource. Resource names are
|
||||
parsed as data; no task code executes during discovery.
|
||||
|
||||
### Live Progress
|
||||
|
||||
`runtime.progress(text)` publishes optional plain text, including line breaks,
|
||||
to the task and run pages. Updates are throttled to once a second; use
|
||||
`force=True` at stage boundaries. Progress is independent of log output, so it
|
||||
does not replace selected log text or require the browser to stay open.
|
||||
|
||||
For simple item-count percentages and per-step estimates:
|
||||
|
||||
```python
|
||||
tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20})
|
||||
tracker.update("Scanning files", 0, force=True)
|
||||
# After processing five files:
|
||||
tracker.update("Scanning files", 5)
|
||||
```
|
||||
|
||||
Each step has its own percentage and an approximate ETA based only on processed
|
||||
items and elapsed time. Pending steps have no ETA. Tasks may instead publish their
|
||||
own text when totals are unknown; no global estimate is required.
|
||||
|
||||
A group may also include a private `task-config.json` for stable local details
|
||||
such as mounted filesystem roots. This is not part of discovery or the web
|
||||
|
||||
@@ -3,15 +3,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
|
||||
from waitress import serve
|
||||
from waitress import create_server
|
||||
|
||||
from .web import create_app
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve(
|
||||
create_app(),
|
||||
def main() -> None:
|
||||
stopping = False
|
||||
requested_at = None
|
||||
|
||||
def request_shutdown(_signal, _frame):
|
||||
nonlocal stopping, requested_at
|
||||
stopping = True
|
||||
if requested_at is None:
|
||||
requested_at = time.monotonic()
|
||||
|
||||
signal.signal(signal.SIGTERM, request_shutdown)
|
||||
signal.signal(signal.SIGINT, request_shutdown)
|
||||
app, server = None, None
|
||||
try:
|
||||
app = create_app()
|
||||
server = create_server(
|
||||
app,
|
||||
host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")),
|
||||
)
|
||||
thread = threading.Thread(target=server.run, name="http-server", daemon=True)
|
||||
thread.start()
|
||||
while not stopping and thread.is_alive():
|
||||
time.sleep(.1)
|
||||
finally:
|
||||
print("Shutdown requested; pausing tasks at safe checkpoints.", flush=True)
|
||||
if app is not None:
|
||||
app.extensions["background_scheduler"].stop()
|
||||
manager = app.extensions["run_manager"]
|
||||
elapsed = time.monotonic() - requested_at if requested_at is not None else 0
|
||||
manager.shutdown(timeout=max(0, manager.settings.shutdown_grace_seconds - elapsed))
|
||||
if server is not None:
|
||||
server.task_dispatcher.shutdown(timeout=5)
|
||||
server.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -17,6 +17,7 @@ class Settings:
|
||||
timezone_name: str | None
|
||||
allowed_proxy_ips: frozenset[str]
|
||||
cancel_grace_seconds: float
|
||||
shutdown_grace_seconds: float = 110
|
||||
|
||||
@property
|
||||
def database_path(self) -> Path:
|
||||
@@ -55,4 +56,5 @@ class Settings:
|
||||
cancel_grace_seconds=float(
|
||||
os.environ.get("SERVER_MAINTENANCE_CANCEL_GRACE_SECONDS", "10")
|
||||
),
|
||||
shutdown_grace_seconds=float(os.environ.get("SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS", "110")),
|
||||
)
|
||||
@@ -39,6 +39,8 @@ class Task:
|
||||
wait_for_result: bool = False
|
||||
download_artifacts: bool = False
|
||||
execution: "ExecutionMode | None" = None
|
||||
recoverable: bool = False
|
||||
resources: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -190,6 +192,7 @@ class TaskCatalog:
|
||||
for filename in ordered_filenames:
|
||||
path = paths_by_filename[filename]
|
||||
declaration = declarations_by_filename.get(path.name)
|
||||
recoverable, resources = read_runtime_declaration(path)
|
||||
tasks.append(
|
||||
Task(
|
||||
id=f"{group_path.name}/{path.name}",
|
||||
@@ -210,6 +213,8 @@ class TaskCatalog:
|
||||
wait_for_result=declaration.wait_for_result if declaration else False,
|
||||
download_artifacts=declaration.download_artifacts if declaration else False,
|
||||
execution=declaration.execution if declaration else None,
|
||||
recoverable=recoverable,
|
||||
resources=(f"group:{group_path.name}", *resources),
|
||||
)
|
||||
)
|
||||
return tasks
|
||||
@@ -294,6 +299,23 @@ def read_description(path: Path) -> str | None:
|
||||
)
|
||||
|
||||
|
||||
def read_runtime_declaration(path: Path) -> tuple[bool, tuple[str, ...]]:
|
||||
"""Discover the opt-in hook and literal resource names without importing code."""
|
||||
try:
|
||||
module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
except (OSError, SyntaxError, UnicodeDecodeError):
|
||||
return False, ()
|
||||
recoverable = any(isinstance(node, ast.FunctionDef) and node.name == "recover" for node in module.body)
|
||||
resources = ()
|
||||
for node in module.body:
|
||||
if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "RESOURCES" for t in node.targets):
|
||||
value = ast.literal_eval(node.value)
|
||||
if not isinstance(value, (tuple, list)) or any(not isinstance(x, str) or not re.fullmatch(r"[a-z][a-z0-9_-]*", x) for x in value):
|
||||
raise ValueError(f"RESOURCES in {path} must be literal resource names")
|
||||
resources = tuple(dict.fromkeys(value))
|
||||
return recoverable, resources
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GroupManifest:
|
||||
title: str | None
|
||||
|
||||
+235
-37
@@ -10,6 +10,7 @@ import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -17,6 +18,7 @@ from .config import Settings
|
||||
from .discovery import BackgroundTask, Group, Task, TaskCatalog
|
||||
from .inputs import PendingUpload
|
||||
from .store import Run, RunStore
|
||||
from .task_runtime import PAUSED_EXIT_CODE, atomic_json
|
||||
|
||||
|
||||
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
|
||||
@@ -24,6 +26,20 @@ ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
|
||||
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
|
||||
|
||||
|
||||
class TaskConflictError(RuntimeError):
|
||||
def __init__(self, run: Run):
|
||||
self.run = run
|
||||
super().__init__(f"Run {run.id} is using resources required by this task.")
|
||||
|
||||
|
||||
def kill_remaining_group(process: subprocess.Popen) -> None:
|
||||
# The leader can already be gone while a transfer subprocess is still alive.
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
class RunManager:
|
||||
def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None:
|
||||
self.settings = settings
|
||||
@@ -32,7 +48,9 @@ class RunManager:
|
||||
self._wake_events: dict[str, threading.Event] = {}
|
||||
self._workers: dict[str, threading.Thread] = {}
|
||||
self._processes: dict[int, subprocess.Popen[bytes]] = {}
|
||||
self._background_processes: set[subprocess.Popen[bytes]] = set()
|
||||
self._lock = threading.Lock()
|
||||
self._closing = threading.Event()
|
||||
|
||||
def run_task(
|
||||
self,
|
||||
@@ -40,13 +58,19 @@ class RunManager:
|
||||
input_data: dict[str, str | int | list[str]] | None = None,
|
||||
uploads: Mapping[str, PendingUpload] | None = None,
|
||||
) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down; try again after it restarts.")
|
||||
sensitive_names = {field.name for field in task.inputs if field.sensitive}
|
||||
stored_input = self._redact_input(input_data or {}, sensitive_names)
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
blocker = self.task_status(task)["blocked_by"]
|
||||
if blocker:
|
||||
raise TaskConflictError(blocker)
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="task",
|
||||
target_path=task.id,
|
||||
queue_group=task.group_id,
|
||||
input_data=stored_input,
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
|
||||
)
|
||||
if created:
|
||||
try:
|
||||
@@ -56,9 +80,11 @@ class RunManager:
|
||||
run = self.store.set_input(run.id, stored_input)
|
||||
if sensitive_names:
|
||||
self._write_execution_input(run.id, execution_input)
|
||||
self._save_work_input(run, execution_input)
|
||||
except (OSError, ValueError) as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
status="failed",
|
||||
@@ -68,15 +94,99 @@ class RunManager:
|
||||
return run, created
|
||||
|
||||
def run_group(self, group: Group) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
|
||||
resources = resources or (f"group:{group.id}",)
|
||||
with self._lock:
|
||||
for active in self.store.active_runs():
|
||||
if active.target_kind == "group" and active.target_path == group.id:
|
||||
return active, False
|
||||
if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)):
|
||||
raise TaskConflictError(active)
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="group",
|
||||
target_path=group.id,
|
||||
queue_group=group.id,
|
||||
target_kind="group", target_path=group.id, queue_group=group.id, resources=resources,
|
||||
)
|
||||
if created:
|
||||
self._wake_worker(group.id)
|
||||
return run, created
|
||||
|
||||
def startup(self) -> None:
|
||||
"""Recover only runs with the durable input/context written by this runner."""
|
||||
self.store.mark_interrupted()
|
||||
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
|
||||
for old in self.store.interrupted_without_recovery():
|
||||
work = self._work_root(old)
|
||||
if not (work / "input.json").is_file():
|
||||
continue
|
||||
task = tasks.get(old.target_path)
|
||||
if task is None:
|
||||
continue
|
||||
if old.started_at and not task.recoverable:
|
||||
continue
|
||||
run, created = self.store.enqueue(
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
input_data=json.loads(old.input_json), resources=task.resources,
|
||||
recovery_of_run_id=old.id,
|
||||
)
|
||||
if created:
|
||||
self._wake_worker(run.queue_group)
|
||||
for run in self.store.active_runs():
|
||||
if run.parent_run_id is None:
|
||||
self._wake_worker(run.queue_group)
|
||||
|
||||
def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict:
|
||||
active = active_runs if active_runs is not None else self.store.active_runs()
|
||||
own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None)
|
||||
resources = set(task.resources or (f"group:{task.group_id}",))
|
||||
blocker = next((run for run in active if run is not own and run.parent_run_id is None
|
||||
and resources.intersection(json.loads(run.resources_json))), None)
|
||||
return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""}
|
||||
|
||||
def progress(self, run: Run) -> str:
|
||||
try:
|
||||
data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text())
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
return str(data.get("text", ""))
|
||||
|
||||
def shutdown(self, timeout: float | None = None) -> None:
|
||||
self._closing.set()
|
||||
for event in self._wake_events.values():
|
||||
event.set()
|
||||
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
|
||||
with self._lock:
|
||||
processes = [*self._processes.values(), *self._background_processes]
|
||||
for process in processes:
|
||||
if process.poll() is None:
|
||||
try:
|
||||
os.kill(process.pid, signal.SIGUSR1)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
while any(process.poll() is None for process in processes) and time.monotonic() < deadline:
|
||||
time.sleep(.05)
|
||||
for process in processes:
|
||||
kill_remaining_group(process)
|
||||
for process in processes:
|
||||
process.wait(timeout=5)
|
||||
for worker in self._workers.values():
|
||||
worker.join(timeout=max(0, deadline - time.monotonic()))
|
||||
|
||||
def _work_root(self, run: Run) -> Path:
|
||||
while run.recovery_of_run_id is not None:
|
||||
previous = self.store.get(run.recovery_of_run_id)
|
||||
if previous is None:
|
||||
raise RuntimeError("The original recovery run no longer exists")
|
||||
run = previous
|
||||
return self.settings.state_root / "runs" / f"{run.id}.work"
|
||||
|
||||
def _save_work_input(self, run: Run, values: dict) -> None:
|
||||
atomic_json(self._work_root(run) / "input.json", values)
|
||||
|
||||
def _discard_work_input(self, run: Run) -> None:
|
||||
for name in ("input.json", "input.json.tmp"):
|
||||
(self._work_root(run) / name).unlink(missing_ok=True)
|
||||
|
||||
def request_cancel(self, run_id: int) -> Run | None:
|
||||
run = self.store.request_cancel(run_id)
|
||||
if run is None:
|
||||
@@ -90,6 +200,7 @@ class RunManager:
|
||||
if completed is not None and completed.status == "cancelled":
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(completed)
|
||||
return completed
|
||||
|
||||
def artifacts(self, run_id: int) -> list[str]:
|
||||
@@ -165,9 +276,10 @@ class RunManager:
|
||||
cutoff = time.time() - artifact_max_age_seconds
|
||||
if runs_root.is_dir():
|
||||
lifetimes = self.store.run_lifetimes()
|
||||
protected = self._recovery_context_ids()
|
||||
for path in runs_root.iterdir():
|
||||
run_id_text, separator, kind = path.name.partition(".")
|
||||
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
|
||||
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink():
|
||||
continue
|
||||
run_id = int(run_id_text)
|
||||
if run_id not in lifetimes:
|
||||
@@ -175,7 +287,7 @@ class RunManager:
|
||||
run = self.store.get(run_id)
|
||||
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
|
||||
status, finished_at = lifetimes[run_id]
|
||||
if status in {"queued", "running"}:
|
||||
if status in {"queued", "running"} or run_id in protected:
|
||||
continue
|
||||
if kind in {"input.json", "execution-input.json"} and path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
@@ -193,8 +305,50 @@ class RunManager:
|
||||
continue
|
||||
if completed_at < cutoff:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif kind == "work" and path.is_dir():
|
||||
(path / "input.json").unlink(missing_ok=True)
|
||||
if path.stat().st_mtime < time.time() - 14 * 86400:
|
||||
shutil.rmtree(path)
|
||||
elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400:
|
||||
path.unlink()
|
||||
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
|
||||
|
||||
def _recovery_context_ids(self) -> set[int]:
|
||||
protected = set()
|
||||
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
|
||||
for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]:
|
||||
task = tasks.get(run.target_path)
|
||||
if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()):
|
||||
continue
|
||||
while True:
|
||||
protected.add(run.id)
|
||||
if run.recovery_of_run_id is None:
|
||||
break
|
||||
run = self.store.get(run.recovery_of_run_id)
|
||||
if run is None:
|
||||
break
|
||||
return protected
|
||||
|
||||
def _promote_review_uploads(self, run: Run, task: Task) -> Run:
|
||||
if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs):
|
||||
return run
|
||||
values = json.loads(run.input_json)
|
||||
if values.get(task.execution.field.name) != task.execution.dry_run_value:
|
||||
return run
|
||||
origin = self._work_root(run).name.split(".", 1)[0]
|
||||
old_root = self._upload_root(int(origin))
|
||||
for field in task.inputs:
|
||||
if field.type != "file" or not values.get(field.name):
|
||||
continue
|
||||
source = Path(values[field.name])
|
||||
if source.parent != old_root or source.is_symlink() or not source.is_file():
|
||||
raise RuntimeError("The original uploaded review input is no longer available")
|
||||
destination = self._upload_root(run.id) / source.name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
shutil.copy2(source, destination)
|
||||
values[field.name] = str(destination)
|
||||
return self.store.set_input(run.id, values)
|
||||
|
||||
@staticmethod
|
||||
def _keep_diagnostic_reports(root: Path) -> None:
|
||||
try:
|
||||
@@ -254,24 +408,31 @@ class RunManager:
|
||||
|
||||
def run_background_task(self, task: BackgroundTask) -> None:
|
||||
"""Run a declared internal task without creating high-volume run history."""
|
||||
|
||||
if self._closing.is_set():
|
||||
return
|
||||
log_path = self.settings.state_root / "background" / f"{task.id}.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
environment = self._environment(task.group_id)
|
||||
environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work")
|
||||
try:
|
||||
with log_path.open("wb") as log_file:
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, str(task.path)],
|
||||
cwd=task.path.parent,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=environment,
|
||||
start_new_session=True,
|
||||
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
|
||||
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
|
||||
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
|
||||
)
|
||||
self._background_processes.add(process)
|
||||
try:
|
||||
exit_code = process.wait()
|
||||
if exit_code:
|
||||
if exit_code not in (0, PAUSED_EXIT_CODE):
|
||||
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
|
||||
finally:
|
||||
kill_remaining_group(process)
|
||||
with self._lock:
|
||||
self._background_processes.discard(process)
|
||||
except OSError as error:
|
||||
log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8")
|
||||
|
||||
@@ -291,13 +452,17 @@ class RunManager:
|
||||
event.set()
|
||||
|
||||
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
|
||||
while True:
|
||||
while not self._closing.is_set():
|
||||
event.clear()
|
||||
run = self.store.claim_next(group_id)
|
||||
if run is None:
|
||||
event.wait()
|
||||
continue
|
||||
try:
|
||||
self._execute_root_run(run)
|
||||
finally:
|
||||
for waiting in self._wake_events.values():
|
||||
waiting.set()
|
||||
|
||||
def _execute_root_run(self, run: Run) -> None:
|
||||
if run.target_kind == "task":
|
||||
@@ -355,6 +520,8 @@ class RunManager:
|
||||
*,
|
||||
parent_run_id: int | None = None,
|
||||
) -> Run:
|
||||
if self._closing.is_set():
|
||||
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
@@ -366,12 +533,20 @@ class RunManager:
|
||||
artifact_root = self._artifact_root(run.id)
|
||||
artifact_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
if not input_path.exists():
|
||||
work = self._work_root(run)
|
||||
saved_input = work / "input.json"
|
||||
if run.recovery_of_run_id is not None:
|
||||
input_path = saved_input
|
||||
elif not input_path.exists():
|
||||
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
|
||||
input_path.write_text(run.input_json + "\n", encoding="utf-8")
|
||||
input_path.chmod(0o600)
|
||||
if not saved_input.exists():
|
||||
self._save_work_input(run, json.loads(input_path.read_text()))
|
||||
except OSError as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
@@ -381,24 +556,33 @@ class RunManager:
|
||||
environment = self._environment(task.group_id)
|
||||
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
|
||||
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
|
||||
environment["SERVER_MAINTENANCE_WORK"] = str(work)
|
||||
environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json")
|
||||
if run.recovery_of_run_id is not None:
|
||||
old = self.store.get(run.recovery_of_run_id)
|
||||
if old and old.started_at:
|
||||
environment["SERVER_MAINTENANCE_RECOVER"] = "1"
|
||||
self.store.set_log_path(run.id, str(log_path))
|
||||
|
||||
try:
|
||||
with log_path.open("wb") as log_file:
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, str(task.path)],
|
||||
cwd=task.path.parent,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log_file,
|
||||
stderr=subprocess.STDOUT,
|
||||
env=environment,
|
||||
start_new_session=True,
|
||||
)
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
|
||||
process = subprocess.Popen(
|
||||
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
|
||||
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
|
||||
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
|
||||
)
|
||||
self._processes[run.id] = process
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
self._terminate_active_process(run.id)
|
||||
exit_code = process.wait()
|
||||
except OSError as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._discard_work_input(run)
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
@@ -408,7 +592,9 @@ class RunManager:
|
||||
)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._processes.pop(run.id, None)
|
||||
owned = self._processes.pop(run.id, None)
|
||||
if owned is not None:
|
||||
kill_remaining_group(owned)
|
||||
self._delete_execution_input(run.id)
|
||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||
|
||||
@@ -416,6 +602,7 @@ class RunManager:
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
self._delete_uploads(run.id)
|
||||
self._discard_work_input(run)
|
||||
self._keep_diagnostic_reports(artifact_root)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
@@ -423,21 +610,31 @@ class RunManager:
|
||||
exit_code=exit_code,
|
||||
log_path=str(log_path),
|
||||
)
|
||||
completed = self.store.finish(
|
||||
run.id,
|
||||
status="succeeded" if exit_code == 0 else "failed",
|
||||
exit_code=exit_code,
|
||||
log_path=str(log_path),
|
||||
if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0):
|
||||
return self.store.finish(
|
||||
run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path),
|
||||
message="Application shutdown; saved state will be checked by recovery after restart.",
|
||||
)
|
||||
if not self.review_upload_paths(completed, task):
|
||||
status = "succeeded" if exit_code == 0 else "failed"
|
||||
message = None
|
||||
if status == "succeeded":
|
||||
try:
|
||||
run = self._promote_review_uploads(run, task)
|
||||
except (RuntimeError, OSError) as error:
|
||||
status, message = "failed", f"Could not preserve review input: {error}"
|
||||
candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat())
|
||||
if not self.review_upload_paths(candidate, task):
|
||||
self._delete_uploads(run.id)
|
||||
if completed.status != "succeeded":
|
||||
if status != "succeeded":
|
||||
self._keep_diagnostic_reports(artifact_root)
|
||||
else:
|
||||
try:
|
||||
artifact_root.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
completed = self.store.finish(run.id, status=status, exit_code=exit_code,
|
||||
message=message, log_path=str(log_path))
|
||||
self._discard_work_input(run)
|
||||
self.cleanup_transient_files()
|
||||
return completed
|
||||
|
||||
@@ -447,6 +644,7 @@ class RunManager:
|
||||
environment = {
|
||||
"PATH": "/usr/local/bin:/usr/bin:/bin",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONPATH": str(Path(__file__).resolve().parent.parent),
|
||||
"SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root),
|
||||
"SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root),
|
||||
"SERVER_MAINTENANCE_STATE": str(task_state),
|
||||
|
||||
@@ -30,6 +30,8 @@ class Run:
|
||||
log_path: str | None
|
||||
message: str | None
|
||||
input_json: str
|
||||
recovery_of_run_id: int | None = None
|
||||
resources_json: str = "[]"
|
||||
|
||||
|
||||
def now() -> str:
|
||||
@@ -72,6 +74,10 @@ class RunStore:
|
||||
connection.execute(
|
||||
"ALTER TABLE runs ADD COLUMN input_json TEXT NOT NULL DEFAULT '{}'"
|
||||
)
|
||||
if "recovery_of_run_id" not in columns:
|
||||
connection.execute("ALTER TABLE runs ADD COLUMN recovery_of_run_id INTEGER REFERENCES runs(id)")
|
||||
if "resources_json" not in columns:
|
||||
connection.execute("ALTER TABLE runs ADD COLUMN resources_json TEXT NOT NULL DEFAULT '[]'")
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS runs_queue_index
|
||||
@@ -106,6 +112,8 @@ class RunStore:
|
||||
queue_group: str,
|
||||
trigger: str = "manual",
|
||||
input_data: Mapping[str, str | int | list[str]] | None = None,
|
||||
resources: tuple[str, ...] = (),
|
||||
recovery_of_run_id: int | None = None,
|
||||
) -> tuple[Run, bool]:
|
||||
input_json = json.dumps(input_data or {}, sort_keys=True, separators=(",", ":"))
|
||||
with self._connect() as connection:
|
||||
@@ -126,10 +134,12 @@ class RunStore:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO runs (
|
||||
target_kind, target_path, queue_group, trigger, status, created_at, input_json
|
||||
) VALUES (?, ?, ?, ?, 'queued', ?, ?)
|
||||
target_kind, target_path, queue_group, trigger, status, created_at, input_json,
|
||||
resources_json, recovery_of_run_id
|
||||
) VALUES (?, ?, ?, ?, 'queued', ?, ?, ?, ?)
|
||||
""",
|
||||
(target_kind, target_path, queue_group, trigger, now(), input_json),
|
||||
(target_kind, target_path, queue_group, trigger, now(), input_json,
|
||||
json.dumps(resources), recovery_of_run_id),
|
||||
)
|
||||
return self.get(cursor.lastrowid, connection=connection), True
|
||||
|
||||
@@ -162,6 +172,10 @@ class RunStore:
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
requested = set(json.loads(row["resources_json"]))
|
||||
for active in connection.execute("SELECT resources_json FROM runs WHERE status='running' AND parent_run_id IS NULL"):
|
||||
if requested & set(json.loads(active["resources_json"])):
|
||||
return None
|
||||
run_id = row["id"]
|
||||
connection.execute(
|
||||
"UPDATE runs SET status = 'running', started_at = ? WHERE id = ?",
|
||||
@@ -263,6 +277,26 @@ class RunStore:
|
||||
for row in connection.execute("SELECT id, status, finished_at FROM runs")
|
||||
}
|
||||
|
||||
def active_runs(self) -> list[Run]:
|
||||
with self._connect() as connection:
|
||||
return [self._run(row) for row in connection.execute(
|
||||
"SELECT * FROM runs WHERE status IN ('queued','running') ORDER BY id"
|
||||
)]
|
||||
|
||||
def interrupted_without_recovery(self) -> list[Run]:
|
||||
with self._connect() as connection:
|
||||
return [self._run(row) for row in connection.execute(
|
||||
"""SELECT r.* FROM runs r WHERE r.status='interrupted' AND r.target_kind='task'
|
||||
AND r.cancel_requested_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM runs child WHERE child.recovery_of_run_id=r.id)
|
||||
ORDER BY r.id"""
|
||||
)]
|
||||
|
||||
def recovery_for(self, run_id: int) -> Run | None:
|
||||
with self._connect() as connection:
|
||||
row = connection.execute("SELECT * FROM runs WHERE recovery_of_run_id=? ORDER BY id DESC LIMIT 1", (run_id,)).fetchone()
|
||||
return self._run(row) if row else None
|
||||
|
||||
def set_log_path(self, run_id: int, path: str) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id))
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Cooperative task checkpoints and optional plain-text progress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PAUSED_EXIT_CODE = 75
|
||||
CHECKPOINT_SECONDS = 60
|
||||
_pause_requested = False
|
||||
_defer = 0
|
||||
_state: dict = {}
|
||||
_last_save = 0.0
|
||||
_last_progress = 0.0
|
||||
_previous_clean = False
|
||||
|
||||
|
||||
class TaskPaused(BaseException):
|
||||
"""Unwind at a safe boundary without treating a pause as an ordinary failure."""
|
||||
|
||||
|
||||
def work_directory() -> Path | None:
|
||||
value = os.environ.get("SERVER_MAINTENANCE_WORK")
|
||||
return Path(value) if value else None
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
with temporary.open("w", encoding="utf-8") as stream:
|
||||
temporary.chmod(0o600)
|
||||
json.dump(value, stream, ensure_ascii=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
temporary.replace(path)
|
||||
descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def initialize() -> None:
|
||||
global _state, _previous_clean, _pause_requested, _last_save
|
||||
_pause_requested = False
|
||||
root = work_directory()
|
||||
if root is not None:
|
||||
try:
|
||||
saved = json.loads((root / "checkpoint.json").read_text())
|
||||
except FileNotFoundError:
|
||||
saved = {}
|
||||
_state = saved.get("data", {})
|
||||
_previous_clean = bool(saved.get("clean_shutdown"))
|
||||
_write_checkpoint(False)
|
||||
_last_save = time.monotonic()
|
||||
signal.signal(signal.SIGUSR1, request_pause)
|
||||
|
||||
|
||||
def request_pause(_signal=None, _frame=None) -> None:
|
||||
global _pause_requested
|
||||
_pause_requested = True
|
||||
|
||||
|
||||
def pause_requested() -> bool:
|
||||
return _pause_requested and work_directory() is not None
|
||||
|
||||
|
||||
def checkpoint_data() -> dict:
|
||||
return _state if work_directory() is not None else {}
|
||||
|
||||
|
||||
def was_clean_shutdown() -> bool:
|
||||
return _previous_clean
|
||||
|
||||
|
||||
def _write_checkpoint(clean: bool) -> None:
|
||||
root = work_directory()
|
||||
if root is not None:
|
||||
atomic_json(root / "checkpoint.json", {
|
||||
"version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(),
|
||||
})
|
||||
|
||||
|
||||
def checkpoint(data: dict | None = None, *, force: bool = False) -> None:
|
||||
"""Keep the latest state in memory; persist ordinary repeatable work once a minute."""
|
||||
global _state, _last_save
|
||||
if data is not None:
|
||||
_state = data
|
||||
if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS:
|
||||
_write_checkpoint(False)
|
||||
_last_save = time.monotonic()
|
||||
check_stop()
|
||||
|
||||
|
||||
def check_stop() -> None:
|
||||
if _pause_requested and work_directory() is not None and not _defer:
|
||||
_write_checkpoint(True)
|
||||
raise TaskPaused()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def critical_section():
|
||||
"""Finish a small consistency-sensitive operation before observing a pause."""
|
||||
global _defer
|
||||
check_stop()
|
||||
_defer += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_defer -= 1
|
||||
check_stop()
|
||||
|
||||
|
||||
def sleep(seconds: float) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while (remaining := deadline - time.monotonic()) > 0:
|
||||
check_stop()
|
||||
time.sleep(min(remaining, 0.25))
|
||||
check_stop()
|
||||
|
||||
|
||||
def progress(text: str, *, force: bool = False) -> None:
|
||||
global _last_progress
|
||||
root = work_directory()
|
||||
if root is not None and (force or time.monotonic() - _last_progress >= 1):
|
||||
path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json")))
|
||||
atomic_json(path, {"text": text, "updated_at": time.time()})
|
||||
_last_progress = time.monotonic()
|
||||
|
||||
|
||||
class Progress:
|
||||
"""Item-count estimates, independently usable for several named steps."""
|
||||
|
||||
def __init__(self, totals: dict[str, int]):
|
||||
self.totals = totals
|
||||
self.done = dict.fromkeys(totals, 0)
|
||||
self.started: dict[str, float] = {}
|
||||
self.baseline: dict[str, int] = {}
|
||||
|
||||
def update(self, label: str, done: int, *, force: bool = False) -> None:
|
||||
self.started.setdefault(label, time.monotonic())
|
||||
self.baseline.setdefault(label, done)
|
||||
self.done[label] = done
|
||||
lines = []
|
||||
for name, total in self.totals.items():
|
||||
count = self.done[name]
|
||||
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
|
||||
line = f"{name}: {percent}%"
|
||||
processed = count - self.baseline.get(name, 0)
|
||||
if processed > 0 and count < total and name in self.started:
|
||||
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
|
||||
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
|
||||
line += f", ETA {estimate}"
|
||||
lines.append(line)
|
||||
progress("\n".join(lines), force=force)
|
||||
+102
-12
@@ -14,7 +14,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u
|
||||
from .config import Settings
|
||||
from .discovery import TaskCatalog
|
||||
from .inputs import PendingUpload, validate_inputs, validate_uploads
|
||||
from .runner import RunManager
|
||||
from .runner import RunManager, TaskConflictError
|
||||
from .scheduler import BackgroundScheduler
|
||||
from .store import RunStore
|
||||
|
||||
@@ -24,11 +24,8 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
catalog = TaskCatalog(settings.task_root)
|
||||
store = RunStore(settings.database_path)
|
||||
store.initialize()
|
||||
store.mark_interrupted()
|
||||
manager = RunManager(settings, catalog, store)
|
||||
manager.cleanup_transient_files()
|
||||
scheduler = BackgroundScheduler(catalog, manager)
|
||||
scheduler.start()
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
app = Flask(
|
||||
@@ -62,6 +59,66 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
return "—"
|
||||
return datetime.fromisoformat(value).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
||||
|
||||
def run_links(run):
|
||||
if run is None:
|
||||
return None
|
||||
display_status = run.status
|
||||
if run.recovery_of_run_id is not None and run.status in {"queued", "running"}:
|
||||
display_status = "Recovery queued" if run.status == "queued" else "recovering"
|
||||
return {
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
"display_status": display_status,
|
||||
"recovery_of_run_id": run.recovery_of_run_id,
|
||||
"artifacts_url": url_for("run_artifacts", run_id=run.id),
|
||||
"detail_url": url_for("run_detail", run_id=run.id),
|
||||
"output_url": url_for("run_output", run_id=run.id),
|
||||
"state_url": url_for("run_state", run_id=run.id),
|
||||
}
|
||||
|
||||
def task_state(task, active_runs=None):
|
||||
state = manager.task_status(task, active_runs=active_runs)
|
||||
return {
|
||||
"active_run": run_links(state["active_run"]),
|
||||
"blocked_by": run_links(state["blocked_by"]),
|
||||
"progress": state["progress"],
|
||||
}
|
||||
|
||||
def live_states(groups, latest):
|
||||
active = store.active_runs()
|
||||
tasks = {task.id: task_state(task, active) for group in groups for task in group.tasks}
|
||||
group_states = {}
|
||||
for group in groups:
|
||||
run = latest[("group", group.id)]
|
||||
active = run if run and run.status in {"queued", "running"} else None
|
||||
blocker = next((
|
||||
tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
|
||||
for task in group.tasks
|
||||
if tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
|
||||
), None)
|
||||
group_states[group.id] = {
|
||||
"active_run": run_links(active),
|
||||
"blocked_by": None if active else blocker,
|
||||
"progress": manager.progress(active) if active else "",
|
||||
}
|
||||
return {"tasks": tasks, "groups": group_states}
|
||||
|
||||
def unavailable(error):
|
||||
if isinstance(error, TaskConflictError):
|
||||
return task_conflict(error)
|
||||
message = str(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {"error": message}, 503
|
||||
return render_template("unavailable.html", message=message, blocking_run=None), 503
|
||||
|
||||
@app.errorhandler(TaskConflictError)
|
||||
def task_conflict(error):
|
||||
blocking_run = run_links(error.run)
|
||||
message = f"Task blocked by run #{error.run.id} ({blocking_run['display_status']})."
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {"error": message, "blocked_by": blocking_run}, 409
|
||||
return render_template("unavailable.html", message=message, blocking_run=blocking_run), 409
|
||||
|
||||
def render_index(
|
||||
*,
|
||||
form_values: dict[str, dict[str, list[str]]] | None = None,
|
||||
@@ -80,6 +137,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
"index.html",
|
||||
groups=groups,
|
||||
latest=latest,
|
||||
live_states=live_states(groups, latest),
|
||||
form_values=form_values or {},
|
||||
input_errors=input_errors or {},
|
||||
),
|
||||
@@ -90,6 +148,14 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
def index():
|
||||
return render_index()
|
||||
|
||||
@app.get("/tasks/state")
|
||||
def tasks_state():
|
||||
groups = catalog.groups()
|
||||
latest = {("group", group.id): store.last_run("group", group.id) for group in groups}
|
||||
response = app.json.response(live_states(groups, latest))
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
@app.post("/tasks/<path:task_id>/run")
|
||||
def run_task(task_id: str):
|
||||
task = catalog.task(task_id)
|
||||
@@ -114,15 +180,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
input_errors={task.id: errors},
|
||||
status=400,
|
||||
)
|
||||
try:
|
||||
run, _ = manager.run_task(task, input_data, uploads)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {
|
||||
"artifacts_url": url_for("run_artifacts", run_id=run.id),
|
||||
"detail_url": url_for("run_detail", run_id=run.id),
|
||||
"output_url": url_for("run_output", run_id=run.id),
|
||||
"run_id": run.id,
|
||||
"state_url": url_for("run_state", run_id=run.id),
|
||||
}, 202
|
||||
return run_links(run), 202
|
||||
return redirect(url_for("run_detail", run_id=run.id))
|
||||
|
||||
@app.post("/groups/<path:group_id>/run")
|
||||
@@ -130,7 +193,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
group = catalog.group(group_id)
|
||||
if group is None or not group.can_run_as_group:
|
||||
abort(404)
|
||||
try:
|
||||
run, _ = manager.run_group(group)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return run_links(run), 202
|
||||
return redirect(url_for("run_detail", run_id=run.id))
|
||||
|
||||
@app.get("/runs/<int:run_id>")
|
||||
@@ -147,6 +215,10 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
output=output,
|
||||
output_offset=offset,
|
||||
execution_task=execution_task,
|
||||
recovery_run=run_links(store.recovery_for(run.id)),
|
||||
progress=manager.progress(run),
|
||||
display_status=run_links(run)["display_status"],
|
||||
live_states={"tasks": {execution_task.id: task_state(execution_task)} if execution_task else {}, "groups": {}},
|
||||
)
|
||||
|
||||
def execution_replay_task(run):
|
||||
@@ -190,9 +262,14 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
except OSError:
|
||||
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
|
||||
uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream)
|
||||
try:
|
||||
executed, created = manager.run_task(task, validated, uploads)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if created and executed.status == "queued":
|
||||
manager.discard_review_uploads(run.id)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return run_links(executed), 202
|
||||
return redirect(url_for("run_detail", run_id=executed.id))
|
||||
|
||||
@app.post("/runs/<int:run_id>/cancel")
|
||||
@@ -216,6 +293,9 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
return {
|
||||
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
|
||||
"status": run.status,
|
||||
"display_status": run_links(run)["display_status"],
|
||||
"recovery_of_run_id": run.recovery_of_run_id,
|
||||
"progress": manager.progress(run),
|
||||
}
|
||||
|
||||
def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
|
||||
@@ -299,8 +379,10 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
if run is None:
|
||||
abort(404)
|
||||
return {
|
||||
"status": run.status,
|
||||
**run_links(run),
|
||||
"progress": manager.progress(run),
|
||||
"finished_at": run.finished_at,
|
||||
"recovery_run": run_links(store.recovery_for(run.id)),
|
||||
"exit_code": run.exit_code,
|
||||
"cancel_requested_at": run.cancel_requested_at,
|
||||
}
|
||||
@@ -316,4 +398,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
runs=store.history(target_kind, target_path),
|
||||
)
|
||||
|
||||
try:
|
||||
manager.startup()
|
||||
manager.cleanup_transient_files()
|
||||
scheduler.start()
|
||||
except BaseException:
|
||||
scheduler.stop()
|
||||
manager.shutdown(timeout=5)
|
||||
raise
|
||||
return app
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Execute task code in isolation, optionally entering its embedded recovery hook."""
|
||||
|
||||
import os
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
from . import task_runtime as runtime
|
||||
|
||||
|
||||
def service_unavailable(error: Exception) -> bool:
|
||||
"""Retry connection failures, not authentication, invalid data or task bugs."""
|
||||
while error is not None:
|
||||
if isinstance(error, HTTPError):
|
||||
return error.code in {408, 502, 503, 504}
|
||||
if isinstance(error, (ConnectionError, TimeoutError)):
|
||||
return True
|
||||
if isinstance(error, URLError):
|
||||
return isinstance(error.reason, (OSError, TimeoutError))
|
||||
sqlstate = getattr(error, "sqlstate", None)
|
||||
if sqlstate and (sqlstate.startswith("08") or sqlstate in {"57P01", "57P03"}):
|
||||
return True
|
||||
if type(error).__module__.startswith("psycopg") and type(error).__name__ == "OperationalError":
|
||||
text = str(error).casefold()
|
||||
return any(reason in text for reason in ("connection refused", "timeout expired", "could not translate host name", "server closed the connection"))
|
||||
error = error.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
path = Path(sys.argv[1]).resolve()
|
||||
sys.path.insert(0, str(path.parent))
|
||||
sys.argv = [str(path)]
|
||||
runtime.initialize()
|
||||
try:
|
||||
if os.environ.get("SERVER_MAINTENANCE_RECOVER") == "1":
|
||||
namespace = runpy.run_path(str(path), run_name="__task_recovery__")
|
||||
recover = namespace.get("recover")
|
||||
if not callable(recover):
|
||||
raise RuntimeError("This task no longer provides its recovery function")
|
||||
print("Recovering interrupted task; verifying current state.", flush=True)
|
||||
attempts = 0
|
||||
while True:
|
||||
try:
|
||||
recover()
|
||||
break
|
||||
except Exception as error:
|
||||
if not service_unavailable(error):
|
||||
raise
|
||||
attempts += 1
|
||||
delay = min(30, 2 ** min(attempts, 5))
|
||||
message = f"Recovery waiting for an unavailable service; retry in {delay} seconds."
|
||||
runtime.progress(message, force=True)
|
||||
print(message, flush=True)
|
||||
runtime.sleep(delay)
|
||||
else:
|
||||
runpy.run_path(str(path), run_name="__main__")
|
||||
except runtime.TaskPaused:
|
||||
print("Paused at a safe checkpoint for recovery after restart.", flush=True)
|
||||
raise SystemExit(runtime.PAUSED_EXIT_CODE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,3 +14,5 @@ services:
|
||||
- ${PYTHON_TOOLS_VOLUME_ROOT:?Set the persistent volume root}/credentials:/opt/credentials:ro
|
||||
# Add task-specific bind mounts here when required.
|
||||
restart: always
|
||||
stop_grace_period: 120s
|
||||
init: true
|
||||
+194
-72
@@ -3,7 +3,7 @@ const output = document.querySelector("#run-output");
|
||||
if (output) {
|
||||
let offset = Number(output.dataset.outputOffset || 0);
|
||||
let detailsFinalized = !["queued", "running"].includes(
|
||||
document.querySelector("#run-status")?.textContent.trim(),
|
||||
document.querySelector("#run-status")?.dataset.runStatus,
|
||||
);
|
||||
const refresh = async () => {
|
||||
let delay = 2000;
|
||||
@@ -24,7 +24,12 @@ if (output) {
|
||||
offset = state.offset;
|
||||
const status = document.querySelector("#run-status");
|
||||
if (status) {
|
||||
status.textContent = state.status;
|
||||
status.textContent = state.display_status;
|
||||
status.dataset.runStatus = state.status;
|
||||
const progress = document.querySelector("#run-progress");
|
||||
if (progress && progress.textContent !== (state.progress || "—")) {
|
||||
progress.textContent = state.progress || "—";
|
||||
}
|
||||
if (state.has_more) {
|
||||
delay = 0;
|
||||
} else if (state.status !== "queued" && state.status !== "running") {
|
||||
@@ -42,6 +47,7 @@ if (output) {
|
||||
document.querySelectorAll(actions).forEach((form) => form.remove());
|
||||
status.closest("dl").replaceWith(nextDetails);
|
||||
nextDetails.after(...page.querySelectorAll(actions));
|
||||
initializeTaskForms(page);
|
||||
detailsFinalized = true;
|
||||
}
|
||||
const artifacts = document.querySelector("#run-artifacts");
|
||||
@@ -115,98 +121,135 @@ for (const choice of document.querySelectorAll("button[data-execution-choice]"))
|
||||
});
|
||||
}
|
||||
|
||||
for (const form of document.querySelectorAll("form[data-wait-for-result]")) {
|
||||
const result = form.querySelector(".task-result");
|
||||
const controls = [...form.querySelectorAll("input, select, button")];
|
||||
let waiting = false;
|
||||
|
||||
const warnBeforeLeaving = (event) => {
|
||||
event.preventDefault();
|
||||
event.returnValue = "";
|
||||
};
|
||||
|
||||
const setWaiting = (value) => {
|
||||
waiting = value;
|
||||
form.classList.toggle("is-waiting", value);
|
||||
controls.forEach((control) => {
|
||||
control.disabled = value;
|
||||
});
|
||||
form.querySelectorAll("details.split-menu").forEach((menu) => {
|
||||
menu.open = false;
|
||||
});
|
||||
if (value) {
|
||||
window.addEventListener("beforeunload", warnBeforeLeaving);
|
||||
} else {
|
||||
window.removeEventListener("beforeunload", warnBeforeLeaving);
|
||||
}
|
||||
};
|
||||
|
||||
const showResult = (message, kind, detailUrl) => {
|
||||
result.replaceChildren();
|
||||
result.className = `task-result ${kind}`;
|
||||
result.append(message);
|
||||
if (detailUrl) {
|
||||
const link = document.createElement("a");
|
||||
link.href = detailUrl;
|
||||
link.textContent = "View details";
|
||||
result.append(" ", link);
|
||||
}
|
||||
};
|
||||
const activeStatus = (status) => status === "queued" || status === "running";
|
||||
const taskControllers = new Map();
|
||||
let tasksStateUrl;
|
||||
let taskStateVersion = 0;
|
||||
let refreshingTasks = false;
|
||||
let taskRefreshTimer;
|
||||
|
||||
const downloadArtifacts = async (run) => {
|
||||
const response = await fetch(run.artifacts_url, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
return;
|
||||
}
|
||||
const artifacts = (await response.json()).artifacts;
|
||||
const artifact = artifacts.find((item) => !item.view_url);
|
||||
if (!artifact) {
|
||||
return;
|
||||
throw new Error("Could not read task downloads. Open the run details to download them.");
|
||||
}
|
||||
const artifact = (await response.json()).artifacts.find((item) => !item.view_url);
|
||||
if (artifact) {
|
||||
const link = document.createElement("a");
|
||||
link.href = artifact.url;
|
||||
link.download = "";
|
||||
document.body.append(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
}
|
||||
};
|
||||
|
||||
const waitForRun = async (run) => {
|
||||
const started = Date.now();
|
||||
while (true) {
|
||||
function taskController(form, initialState) {
|
||||
const result = form.querySelector(".task-result");
|
||||
const message = result.querySelector(".task-message");
|
||||
const progress = result.querySelector(".task-progress");
|
||||
let state = initialState;
|
||||
let observedRun = state.active_run;
|
||||
let submitting = false;
|
||||
let lastResult;
|
||||
let previousMessage;
|
||||
|
||||
const render = (text, kind = "", run = null, progressText = "", suffix = "", label = "View details") => {
|
||||
const signature = JSON.stringify([text, kind, run?.detail_url, suffix, label]);
|
||||
if (signature !== previousMessage) {
|
||||
message.replaceChildren(text);
|
||||
if (run) {
|
||||
const link = document.createElement("a");
|
||||
link.href = run.detail_url;
|
||||
link.textContent = label;
|
||||
message.append(" ", link, suffix);
|
||||
}
|
||||
previousMessage = signature;
|
||||
}
|
||||
if (progress.textContent !== progressText) {
|
||||
progress.textContent = progressText;
|
||||
}
|
||||
result.className = `task-result ${kind}`;
|
||||
result.hidden = !text;
|
||||
};
|
||||
|
||||
const showState = () => {
|
||||
const active = state.active_run;
|
||||
const blocker = state.blocked_by;
|
||||
const disabled = submitting || Boolean(active || blocker);
|
||||
form.querySelector(".task-controls").disabled = disabled;
|
||||
form.classList.toggle("is-waiting", submitting || Boolean(active));
|
||||
form.setAttribute("aria-busy", String(submitting || Boolean(active)));
|
||||
form.querySelectorAll("details.split-menu").forEach((menu) => {
|
||||
menu.inert = disabled;
|
||||
if (disabled) {
|
||||
menu.open = false;
|
||||
}
|
||||
});
|
||||
if (submitting) {
|
||||
render("Starting...", "running");
|
||||
} else if (active) {
|
||||
const status = active.display_status;
|
||||
render(`${status[0].toUpperCase()}${status.slice(1)}.`, "running", active,
|
||||
state.progress, "", `Run #${active.run_id}`);
|
||||
} else if (blocker) {
|
||||
render("Blocked by", "blocked", blocker, "", ` (${blocker.display_status}).`, `run #${blocker.run_id}`);
|
||||
} else if (lastResult) {
|
||||
render(lastResult.text, lastResult.kind, lastResult.run);
|
||||
} else {
|
||||
render("");
|
||||
}
|
||||
};
|
||||
|
||||
const apply = async (nextState, version) => {
|
||||
if (submitting || version !== taskStateVersion) {
|
||||
return;
|
||||
}
|
||||
state = nextState;
|
||||
if (state.active_run) {
|
||||
observedRun = state.active_run;
|
||||
} else if (observedRun) {
|
||||
const run = observedRun;
|
||||
const response = await fetch(run.state_url, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
throw new Error("Could not read task status.");
|
||||
}
|
||||
const state = await response.json();
|
||||
if (state.status !== "queued" && state.status !== "running") {
|
||||
if (state.status === "succeeded") {
|
||||
showResult("Completed successfully.", "succeeded", run.detail_url);
|
||||
if (form.dataset.downloadArtifacts !== undefined) {
|
||||
await downloadArtifacts(run);
|
||||
}
|
||||
} else {
|
||||
showResult(`Finished with status: ${state.status}.`, "failed", run.detail_url);
|
||||
}
|
||||
const completed = await response.json();
|
||||
if (submitting || version !== taskStateVersion) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => window.setTimeout(resolve, Date.now() - started < 10000 ? 500 : 2000));
|
||||
if (activeStatus(completed.status)) {
|
||||
state = { ...state, active_run: completed, progress: completed.progress };
|
||||
} else {
|
||||
observedRun = null;
|
||||
lastResult = {
|
||||
text: completed.status === "succeeded" ? "Completed successfully." : `Finished with status: ${completed.status}.`,
|
||||
kind: completed.status === "succeeded" ? "succeeded" : "failed",
|
||||
run,
|
||||
};
|
||||
if (completed.status === "succeeded" && form.dataset.downloadArtifacts !== undefined) {
|
||||
try {
|
||||
await downloadArtifacts(run);
|
||||
} catch (error) {
|
||||
lastResult = { text: error.message, kind: "failed", run };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
showState();
|
||||
};
|
||||
|
||||
form.addEventListener("submit", async (event) => {
|
||||
if (waiting) {
|
||||
event.preventDefault();
|
||||
if (submitting || state.active_run || state.blocked_by || !form.reportValidity()) {
|
||||
return;
|
||||
}
|
||||
if (!form.reportValidity()) {
|
||||
return;
|
||||
}
|
||||
event.preventDefault();
|
||||
// Disabled fields are omitted from FormData, so collect values first.
|
||||
const formData = new FormData(form);
|
||||
setWaiting(true);
|
||||
showResult("Running...", "running");
|
||||
submitting = true;
|
||||
lastResult = null;
|
||||
taskStateVersion += 1;
|
||||
showState();
|
||||
try {
|
||||
const response = await fetch(form.action, {
|
||||
method: "POST",
|
||||
@@ -214,15 +257,94 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
const messages = data.errors ? Object.values(data.errors).join(" ") : "Could not start task.";
|
||||
throw new Error(messages);
|
||||
taskStateVersion += 1;
|
||||
if (response.status === 409 && data.blocked_by) {
|
||||
state = { active_run: null, blocked_by: data.blocked_by, progress: "" };
|
||||
} else if (!response.ok) {
|
||||
throw new Error(data.errors ? Object.values(data.errors).join(" ") : data.error || "Could not start task.");
|
||||
} else {
|
||||
observedRun = data;
|
||||
state = { active_run: data, blocked_by: null, progress: "" };
|
||||
if (form.dataset.waitForResult === undefined) {
|
||||
window.location.assign(data.detail_url);
|
||||
}
|
||||
}
|
||||
await waitForRun(data);
|
||||
} catch (error) {
|
||||
showResult(error instanceof Error ? error.message : "Could not run task.", "failed");
|
||||
lastResult = { text: error instanceof Error ? error.message : "Could not run task.", kind: "failed" };
|
||||
} finally {
|
||||
setWaiting(false);
|
||||
submitting = false;
|
||||
showState();
|
||||
refreshTaskStates();
|
||||
}
|
||||
});
|
||||
showState();
|
||||
return { apply };
|
||||
}
|
||||
|
||||
async function refreshTaskStates() {
|
||||
if (refreshingTasks || !tasksStateUrl) {
|
||||
return;
|
||||
}
|
||||
refreshingTasks = true;
|
||||
window.clearTimeout(taskRefreshTimer);
|
||||
const version = taskStateVersion;
|
||||
const errorMessage = document.querySelector("#task-status-error");
|
||||
try {
|
||||
const response = await fetch(tasksStateUrl, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
throw new Error("Could not read live task status.");
|
||||
}
|
||||
const states = await response.json();
|
||||
for (const [form, controller] of taskControllers) {
|
||||
if (!form.isConnected) {
|
||||
taskControllers.delete(form);
|
||||
continue;
|
||||
}
|
||||
const state = form.dataset.taskId ? states.tasks[form.dataset.taskId] : states.groups[form.dataset.groupId];
|
||||
if (state) {
|
||||
await controller.apply(state, version);
|
||||
}
|
||||
}
|
||||
if (errorMessage) {
|
||||
errorMessage.hidden = true;
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("Could not refresh task status:", error);
|
||||
if (errorMessage) {
|
||||
errorMessage.textContent = "Live status unavailable. Retrying...";
|
||||
errorMessage.hidden = false;
|
||||
}
|
||||
} finally {
|
||||
refreshingTasks = false;
|
||||
if (taskControllers.size) {
|
||||
taskRefreshTimer = window.setTimeout(refreshTaskStates, 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function initializeTaskForms(source = document) {
|
||||
const data = source.querySelector("#task-states");
|
||||
if (!data) {
|
||||
return;
|
||||
}
|
||||
tasksStateUrl = data.dataset.stateUrl;
|
||||
const states = JSON.parse(data.textContent);
|
||||
for (const form of document.querySelectorAll("form[data-task-id], form[data-group-id]")) {
|
||||
if (!taskControllers.has(form)) {
|
||||
const state = form.dataset.taskId ? states.tasks[form.dataset.taskId] : states.groups[form.dataset.groupId];
|
||||
taskControllers.set(form, taskController(form, state));
|
||||
}
|
||||
}
|
||||
if (taskControllers.size) {
|
||||
if (!document.querySelector("#task-status-error")) {
|
||||
const errorMessage = document.createElement("p");
|
||||
errorMessage.id = "task-status-error";
|
||||
errorMessage.setAttribute("role", "status");
|
||||
errorMessage.hidden = true;
|
||||
document.querySelector("main").prepend(errorMessage);
|
||||
}
|
||||
refreshTaskStates();
|
||||
}
|
||||
}
|
||||
|
||||
initializeTaskForms();
|
||||
@@ -34,6 +34,7 @@ a {
|
||||
.group-heading {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 1rem;
|
||||
justify-content: space-between;
|
||||
}
|
||||
@@ -69,6 +70,21 @@ a {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.task-controls {
|
||||
border: 0;
|
||||
margin: 0;
|
||||
min-width: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.task-form > .task-controls {
|
||||
align-items: end;
|
||||
display: grid;
|
||||
gap: .75rem;
|
||||
grid-column: 1 / -1;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
}
|
||||
|
||||
.task-options {
|
||||
display: grid;
|
||||
gap: .75rem;
|
||||
@@ -204,23 +220,24 @@ button {
|
||||
}
|
||||
|
||||
button:disabled {
|
||||
cursor: wait;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
border: .15em solid currentColor;
|
||||
border-right-color: transparent;
|
||||
border-radius: 50%;
|
||||
display: none;
|
||||
display: inline-block;
|
||||
height: .75em;
|
||||
margin-left: .35em;
|
||||
vertical-align: -.05em;
|
||||
visibility: hidden;
|
||||
width: .75em;
|
||||
}
|
||||
|
||||
.is-waiting .spinner {
|
||||
animation: spin .7s linear infinite;
|
||||
display: inline-block;
|
||||
visibility: visible;
|
||||
}
|
||||
|
||||
.task-result {
|
||||
@@ -228,6 +245,26 @@ button:disabled {
|
||||
margin: .5rem 0 0;
|
||||
}
|
||||
|
||||
.task-progress,
|
||||
.run-progress {
|
||||
overflow-wrap: anywhere;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.task-progress {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.task-result,
|
||||
.group-heading > form {
|
||||
min-width: 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.task-result[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.task-result.succeeded {
|
||||
color: #167c3a;
|
||||
}
|
||||
@@ -247,7 +284,8 @@ button:disabled {
|
||||
}
|
||||
|
||||
@media (max-width: 42rem) {
|
||||
.task-form {
|
||||
.task-form,
|
||||
.task-form > .task-controls {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
<li>
|
||||
<a href="{{ url_for('run_detail', run_id=run.id) }}">Run #{{ run.id }}</a>
|
||||
· {{ run.status }}
|
||||
{% if run.recovery_of_run_id is not none %}· recovery of <a href="{{ url_for('run_detail', run_id=run.recovery_of_run_id) }}">#{{ run.recovery_of_run_id }}</a>{% endif %}
|
||||
· {{ run.created_at|timestamp }}
|
||||
{% if run.exit_code is not none %}· exit {{ run.exit_code }}{% endif %}
|
||||
</li>
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "task_status.html" import task_status, live_state_data %}
|
||||
{% block content %}
|
||||
<h1>Tasks</h1>
|
||||
{{ live_state_data(live_states) }}
|
||||
{% if not groups %}
|
||||
<p>No task groups found.</p>
|
||||
{% endif %}
|
||||
@@ -10,8 +12,12 @@
|
||||
<div class="group-heading">
|
||||
<h2>{{ group.display_name }}</h2>
|
||||
{% if group.can_run_as_group %}
|
||||
<form method="post" action="{{ url_for('run_group', group_id=group.id) }}">
|
||||
{% set group_state = live_states.groups[group.id] %}
|
||||
<form method="post" action="{{ url_for('run_group', group_id=group.id) }}" data-group-id="{{ group.id }}">
|
||||
<fieldset class="task-controls"{% if group_state.active_run or group_state.blocked_by %} disabled{% endif %}>
|
||||
<button type="submit">Run group</button>
|
||||
</fieldset>
|
||||
{{ task_status(group_state) }}
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
@@ -24,6 +30,8 @@
|
||||
<ul class="tasks">
|
||||
{% for task in group.tasks %}
|
||||
{% set task_run = latest[("task", task.id)] %}
|
||||
{% set state = live_states.tasks[task.id] %}
|
||||
{% set busy = state.active_run or state.blocked_by %}
|
||||
<li class="task">
|
||||
<div class="task-details">
|
||||
<h3>{{ task.display_name }}</h3>
|
||||
@@ -37,9 +45,10 @@
|
||||
</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
<form class="task-form" method="post" action="{{ url_for('run_task', task_id=task.id) }}" enctype="multipart/form-data"{% if task.wait_for_result %} data-wait-for-result{% endif %}{% if task.download_artifacts %} data-download-artifacts{% endif %}>
|
||||
<form class="task-form{% if state.active_run %} is-waiting{% endif %}" data-task-id="{{ task.id }}" method="post" action="{{ url_for('run_task', task_id=task.id) }}" enctype="multipart/form-data"{% if task.wait_for_result %} data-wait-for-result{% endif %}{% if task.download_artifacts %} data-download-artifacts{% endif %}>
|
||||
{% set values = form_values.get(task.id, {}) %}
|
||||
{% set errors = input_errors.get(task.id, {}) %}
|
||||
<fieldset class="task-controls"{% if busy %} disabled{% endif %}>
|
||||
<div class="task-options">
|
||||
{% for field in task.inputs %}
|
||||
{% if not task.execution or field.name != task.execution.field.name %}
|
||||
@@ -101,9 +110,9 @@
|
||||
<input type="hidden" name="{{ execution.name }}" value="{{ selected }}" data-execution-value>
|
||||
<button class="split-primary" type="submit">
|
||||
<span class="button-label">{{ selected_option.label if selected_option else execution.options[0].label }}</span>
|
||||
{% if task.wait_for_result %}<span class="spinner" aria-hidden="true"></span>{% endif %}
|
||||
<span class="spinner" aria-hidden="true"></span>
|
||||
</button>
|
||||
<details class="split-menu">
|
||||
<details class="split-menu"{% if busy %} inert{% endif %}>
|
||||
<summary aria-label="Choose run mode" title="Choose run mode">▾</summary>
|
||||
<div class="split-menu-options">
|
||||
{% for option in execution.options %}
|
||||
@@ -115,11 +124,12 @@
|
||||
{% else %}
|
||||
<button type="submit">
|
||||
<span class="button-label">Run</span>
|
||||
{% if task.wait_for_result %}<span class="spinner" aria-hidden="true"></span>{% endif %}
|
||||
<span class="spinner" aria-hidden="true"></span>
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% if task.wait_for_result %}<p class="task-result" aria-live="polite"></p>{% endif %}
|
||||
</fieldset>
|
||||
{{ task_status(state) }}
|
||||
</form>
|
||||
</li>
|
||||
{% else %}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "task_status.html" import task_status, live_state_data %}
|
||||
{% block title %}Run {{ run.id }} · Server Maintenance{% endblock %}
|
||||
{% block content %}
|
||||
<p>
|
||||
@@ -8,7 +9,10 @@
|
||||
<h1>{{ run.target_kind|title }} run #{{ run.id }}</h1>
|
||||
<dl>
|
||||
<dt>Target</dt><dd>{{ run.target_path }}</dd>
|
||||
<dt>Status</dt><dd id="run-status" data-state-url="{{ url_for('run_state', run_id=run.id) }}">{{ run.status }}</dd>
|
||||
<dt>Status</dt><dd id="run-status" data-run-status="{{ run.status }}" data-state-url="{{ url_for('run_state', run_id=run.id) }}">{{ display_status }}</dd>
|
||||
<dt>Progress</dt><dd id="run-progress" class="run-progress">{{ progress or "—" }}</dd>
|
||||
{% if run.recovery_of_run_id is not none %}<dt>Recovery of</dt><dd><a href="{{ url_for('run_detail', run_id=run.recovery_of_run_id) }}">Run #{{ run.recovery_of_run_id }}</a></dd>{% endif %}
|
||||
{% if recovery_run %}<dt>Recovery run</dt><dd><a href="{{ recovery_run.detail_url }}">Run #{{ recovery_run.run_id }}</a> · {{ recovery_run.display_status }}</dd>{% endif %}
|
||||
<dt>Started</dt><dd>{{ run.started_at|timestamp }}</dd>
|
||||
<dt>Finished</dt><dd>{{ run.finished_at|timestamp }}</dd>
|
||||
<dt>Exit code</dt><dd>{{ run.exit_code if run.exit_code is not none else "—" }}</dd>
|
||||
@@ -20,8 +24,13 @@
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if execution_task %}
|
||||
<form method="post" action="{{ url_for('execute_dry_run', run_id=run.id) }}">
|
||||
{{ live_state_data(live_states) }}
|
||||
{% set state = live_states.tasks[execution_task.id] %}
|
||||
<form method="post" action="{{ url_for('execute_dry_run', run_id=run.id) }}" data-task-id="{{ execution_task.id }}">
|
||||
<fieldset class="task-controls"{% if state.active_run or state.blocked_by %} disabled{% endif %}>
|
||||
<button type="submit">{{ execution_task.execution.field.options[1].label }}</button>
|
||||
</fieldset>
|
||||
{{ task_status(state) }}
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if children %}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{% macro task_status(state) %}
|
||||
{% set active = state.active_run %}
|
||||
{% set blocker = state.blocked_by %}
|
||||
<p class="task-result {{ 'running' if active else 'blocked' if blocker else '' }}" aria-live="polite"{% if not active and not blocker %} hidden{% endif %}>
|
||||
<span class="task-message">{% if active %}{{ active.display_status|capitalize }}. <a href="{{ active.detail_url }}">Run #{{ active.run_id }}</a>{% elif blocker %}Blocked by <a href="{{ blocker.detail_url }}">run #{{ blocker.run_id }}</a> ({{ blocker.display_status }}).{% endif %}</span>
|
||||
<span class="task-progress">{{ state.progress if active else '' }}</span>
|
||||
</p>
|
||||
{% endmacro %}
|
||||
|
||||
{% macro live_state_data(states) %}
|
||||
<p id="task-status-error" role="status" hidden></p>
|
||||
<script id="task-states" type="application/json" data-state-url="{{ url_for('tasks_state') }}">{{ states|tojson }}</script>
|
||||
{% endmacro %}
|
||||
@@ -0,0 +1,10 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Task unavailable · Server Maintenance{% endblock %}
|
||||
{% block content %}
|
||||
<p><a href="{{ url_for('index') }}">Tasks</a></p>
|
||||
<h1>Task unavailable</h1>
|
||||
<p>{{ message }}</p>
|
||||
{% if blocking_run %}
|
||||
<p><a href="{{ blocking_run.detail_url }}">View run #{{ blocking_run.run_id }}</a></p>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -9,6 +9,27 @@ from app.discovery import Group, TaskCatalog
|
||||
from .conftest import write_task
|
||||
|
||||
|
||||
def test_discovers_embedded_recovery_and_literal_resources_without_import(tmp_path):
|
||||
path=write_task(tmp_path,'tasks','change.py', '''
|
||||
"""A recoverable task."""
|
||||
RESOURCES = ('shared-files', 'shared-files')
|
||||
def recover():
|
||||
raise RuntimeError('Must not run during discovery')
|
||||
raise RuntimeError('Must not import during discovery')
|
||||
''')
|
||||
task=TaskCatalog(tmp_path).task('tasks/change.py')
|
||||
assert task.recoverable is True
|
||||
assert task.resources==('group:tasks','shared-files')
|
||||
path.write_text('class Example:\n def recover(self): pass\n')
|
||||
assert TaskCatalog(tmp_path).task('tasks/change.py').recoverable is False
|
||||
|
||||
|
||||
def test_resources_do_not_execute_expressions(tmp_path):
|
||||
write_task(tmp_path,'tasks','change.py',"RESOURCES = tuple(['shared-files'])\n")
|
||||
with pytest.raises(ValueError):
|
||||
TaskCatalog(tmp_path).groups()
|
||||
|
||||
|
||||
def test_discovers_only_direct_non_hidden_python_tasks(tmp_path: Path) -> None:
|
||||
root = tmp_path / "tasks"
|
||||
write_task(root, "media_tasks", "check_one.py", '"""Checks one thing."""\n')
|
||||
|
||||
@@ -21,7 +21,11 @@ def output_run(settings):
|
||||
def test_log_polling_returns_only_appended_bytes_and_status(output_run):
|
||||
client, run_id, path, store = output_run
|
||||
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
|
||||
assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"}
|
||||
assert first == {
|
||||
"output": "first\n", "offset": 6, "reset": False, "has_more": False,
|
||||
"status": "running", "display_status": "running", "progress": "",
|
||||
"recovery_of_run_id": None,
|
||||
}
|
||||
with path.open("ab") as stream:
|
||||
stream.write("next: \u20ac\n".encode())
|
||||
store.finish(run_id, status="succeeded")
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Web contracts for restoring live task state and rejecting resource conflicts."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.runner import RunManager
|
||||
from app.scheduler import BackgroundScheduler
|
||||
from app.web import create_app
|
||||
|
||||
from .conftest import write_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_app(settings, monkeypatch):
|
||||
for group, filename, source in (
|
||||
("example", "long.py", 'RESOURCES = ("shared-data",)\n'),
|
||||
("example", "sibling.py", ""),
|
||||
("related", "change.py", 'RESOURCES = ("shared-data",)\n'),
|
||||
("independent", "free.py", ""),
|
||||
):
|
||||
write_task(settings.task_root, group, filename, source + 'print("done")\n')
|
||||
monkeypatch.setattr(RunManager, "_wake_worker", lambda *_: None)
|
||||
app = create_app(settings)
|
||||
yield app
|
||||
app.extensions["background_scheduler"].stop()
|
||||
|
||||
|
||||
def start_live_run(app, *, status="running", recovery=False):
|
||||
manager = app.extensions["run_manager"]
|
||||
task = app.extensions["catalog"].task("example/long.py")
|
||||
run, _ = manager.run_task(task)
|
||||
if recovery:
|
||||
old = manager.store.finish(run.id, status="interrupted")
|
||||
run, _ = manager.store.enqueue(
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
resources=task.resources, recovery_of_run_id=old.id,
|
||||
)
|
||||
if status == "running":
|
||||
run = manager.store.claim_next(task.group_id)
|
||||
progress = manager.settings.state_root / "runs" / f"{run.id}.progress.json"
|
||||
progress.parent.mkdir(parents=True, exist_ok=True)
|
||||
progress.write_text(json.dumps({"text": "2 of 5\nETA: 3 minutes"}))
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["queued", "running"])
|
||||
def test_task_states_restore_active_conflicts_and_progress(live_app, status):
|
||||
run = start_live_run(live_app, status=status)
|
||||
client = live_app.test_client()
|
||||
response = client.get("/tasks/state")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
states = response.get_json()
|
||||
own = states["tasks"]["example/long.py"]
|
||||
assert own["active_run"]["run_id"] == run.id
|
||||
assert own["active_run"]["display_status"] == status
|
||||
assert own["progress"] == "2 of 5\nETA: 3 minutes"
|
||||
assert own["blocked_by"] is None
|
||||
for name in ("example/sibling.py", "related/change.py"):
|
||||
state = states["tasks"][name]
|
||||
assert state["active_run"] is None
|
||||
assert state["blocked_by"]["detail_url"] == f"/runs/{run.id}"
|
||||
assert states["tasks"]["independent/free.py"] == {
|
||||
"active_run": None, "blocked_by": None, "progress": "",
|
||||
}
|
||||
assert states["groups"]["related"]["blocked_by"]["run_id"] == run.id
|
||||
assert not {"input_json", "resources_json", "log_path"}.intersection(own["active_run"])
|
||||
page = client.get("/")
|
||||
assert b'fieldset class="task-controls" disabled' in page.data
|
||||
assert b"2 of 5\nETA: 3 minutes" in page.data
|
||||
assert f'Run #{run.id}'.encode() in page.data
|
||||
assert client.post("/tasks/state").status_code == 405
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["queued", "running"])
|
||||
def test_recovery_links_and_live_run_progress(live_app, status):
|
||||
run = start_live_run(live_app, status=status, recovery=True)
|
||||
client = live_app.test_client()
|
||||
display_status = "Recovery queued" if status == "queued" else "recovering"
|
||||
state = client.get(f"/runs/{run.id}/state").get_json()
|
||||
assert state["status"] == status
|
||||
assert state["display_status"] == display_status
|
||||
assert state["recovery_of_run_id"] == run.recovery_of_run_id
|
||||
assert state["progress"] == "2 of 5\nETA: 3 minutes"
|
||||
output = client.get(f"/runs/{run.id}/output?offset=0").get_json()
|
||||
assert output["progress"] == state["progress"]
|
||||
assert output["display_status"] == display_status
|
||||
detail = client.get(f"/runs/{run.id}")
|
||||
assert f'href="/runs/{run.recovery_of_run_id}"'.encode() in detail.data
|
||||
assert b'id="run-progress" class="run-progress">2 of 5\nETA: 3 minutes' in detail.data
|
||||
old_detail = client.get(f"/runs/{run.recovery_of_run_id}")
|
||||
assert b"Recovery run" in old_detail.data
|
||||
assert f'href="/runs/{run.id}"'.encode() in old_detail.data
|
||||
old_state = client.get(f"/runs/{run.recovery_of_run_id}/state").get_json()
|
||||
assert old_state["recovery_run"]["run_id"] == run.id
|
||||
assert b"recovery of" in client.get("/history/task/example/long.py").data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
|
||||
@pytest.mark.parametrize("target", ["/tasks/related/change.py/run", "/groups/related/run"])
|
||||
def test_conflicting_submissions_return_409_with_blocking_run(live_app, accept, target):
|
||||
run = start_live_run(live_app)
|
||||
response = live_app.test_client().post(target, headers={"Accept": accept})
|
||||
assert response.status_code == 409
|
||||
if accept == "application/json":
|
||||
payload = response.get_json()
|
||||
assert payload["blocked_by"]["run_id"] == run.id
|
||||
assert payload["blocked_by"]["detail_url"] == f"/runs/{run.id}"
|
||||
assert f"#{run.id}" in payload["error"]
|
||||
else:
|
||||
assert f'href="/runs/{run.id}"'.encode() in response.data
|
||||
assert live_app.extensions["run_store"].last_run("task", "related/change.py") is None
|
||||
unrelated = live_app.test_client().post("/tasks/independent/free.py/run", headers={"Accept": accept})
|
||||
assert unrelated.status_code == (202 if accept == "application/json" else 302)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
|
||||
@pytest.mark.parametrize("target", ["/tasks/example/long.py/run", "/groups/example/run"])
|
||||
def test_shutdown_is_reported_without_enqueuing(live_app, accept, target):
|
||||
live_app.extensions["run_manager"]._closing.set()
|
||||
response = live_app.test_client().post(target, headers={"Accept": accept})
|
||||
assert response.status_code == 503
|
||||
assert b"shutting down" in response.data
|
||||
|
||||
|
||||
def test_startup_precedes_cleanup_and_scheduler(settings, monkeypatch):
|
||||
calls = []
|
||||
monkeypatch.setattr(RunManager, "startup", lambda *_: calls.append("startup"))
|
||||
monkeypatch.setattr(RunManager, "cleanup_transient_files", lambda *_: calls.append("cleanup"))
|
||||
monkeypatch.setattr(BackgroundScheduler, "start", lambda *_: calls.append("scheduler"))
|
||||
create_app(settings)
|
||||
assert calls == ["startup", "cleanup", "scheduler"]
|
||||
|
||||
|
||||
def test_progress_is_plain_text_and_escaped(live_app):
|
||||
run = start_live_run(live_app)
|
||||
path = live_app.config["settings"].state_root / "runs" / f"{run.id}.progress.json"
|
||||
path.write_text(json.dumps({"text": "<script>not HTML</script>\nNext item"}))
|
||||
for url in ("/", f"/runs/{run.id}"):
|
||||
page = live_app.test_client().get(url)
|
||||
assert b"<script>not HTML</script>\nNext item" in page.data
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Optional Playwright checks against disposable, locally served task fixtures."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from .conftest import write_task
|
||||
from .test_live_status import live_app as live_app, start_live_run
|
||||
|
||||
playwright = pytest.importorskip("playwright.sync_api")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def browser():
|
||||
with playwright.sync_playwright() as driver:
|
||||
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE", driver.chromium.executable_path)
|
||||
if not Path(executable).is_file():
|
||||
pytest.skip("Install Playwright's Chromium or set PLAYWRIGHT_CHROMIUM_EXECUTABLE")
|
||||
browser = driver.chromium.launch(executable_path=executable)
|
||||
yield browser
|
||||
browser.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_server(live_app):
|
||||
server = make_server("127.0.0.1", 0, live_app, threaded=True)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield live_app, f"http://127.0.0.1:{server.server_port}"
|
||||
server.shutdown()
|
||||
thread.join(timeout=5)
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_page(browser, live_server):
|
||||
app, url = live_server
|
||||
context = browser.new_context(accept_downloads=True)
|
||||
context.add_init_script("""(() => {
|
||||
window.leaveListeners = [];
|
||||
const original = window.addEventListener;
|
||||
window.addEventListener = function(type, ...args) {
|
||||
if (type === 'beforeunload') window.leaveListeners.push(type);
|
||||
return original.call(this, type, ...args);
|
||||
};
|
||||
})();""")
|
||||
page = context.new_page()
|
||||
errors = []
|
||||
dialogs = []
|
||||
page.on("pageerror", lambda error: errors.append(str(error)))
|
||||
|
||||
def dismiss(dialog):
|
||||
dialogs.append(dialog.type)
|
||||
dialog.dismiss()
|
||||
|
||||
page.on("dialog", dismiss)
|
||||
yield app, url, page
|
||||
context.close()
|
||||
assert not errors
|
||||
assert not dialogs
|
||||
|
||||
|
||||
def task_form(page, task_id):
|
||||
return page.locator(f'form[data-task-id="{task_id}"]')
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [1280, 390, 320])
|
||||
@pytest.mark.parametrize("status", ["queued", "running", "recovering", "recovery queued"])
|
||||
def test_reload_navigation_and_multiline_progress(live_page, tmp_path, width, status):
|
||||
app, url, page = live_page
|
||||
run = start_live_run(
|
||||
app, status="queued" if status in {"queued", "recovery queued"} else "running",
|
||||
recovery=status in {"recovering", "recovery queued"},
|
||||
)
|
||||
page.set_viewport_size({"width": width, "height": 844})
|
||||
page.goto(url)
|
||||
own = task_form(page, "example/long.py")
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize())
|
||||
playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
assert own.locator('button[type="submit"]').is_disabled()
|
||||
for name in ("example/sibling.py", "related/change.py"):
|
||||
form = task_form(page, name)
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Blocked by")
|
||||
assert form.locator('button[type="submit"]').is_disabled()
|
||||
assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled()
|
||||
page.reload()
|
||||
playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
own.locator(f'.task-result a[href="/runs/{run.id}"]').click()
|
||||
playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
page.reload()
|
||||
playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
playwright.expect(page.locator("#run-status")).to_have_text("Recovery queued" if status == "recovery queued" else status)
|
||||
assert page.locator("#run-progress").evaluate("el => getComputedStyle(el).whiteSpace") == "pre-wrap"
|
||||
page.screenshot(path=str(tmp_path / f"run-{status}-{width}.png"), full_page=True)
|
||||
assert page.evaluate("window.leaveListeners") == []
|
||||
page.locator('main a[href="/"]').first.click()
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize())
|
||||
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
|
||||
page.screenshot(path=str(tmp_path / f"live-{status}-{width}.png"), full_page=True)
|
||||
|
||||
|
||||
def test_first_render_blocks_conflicts_without_javascript(browser, live_server):
|
||||
app, url = live_server
|
||||
start_live_run(app)
|
||||
context = browser.new_context(java_script_enabled=False)
|
||||
try:
|
||||
page = context.new_page()
|
||||
page.goto(url)
|
||||
assert task_form(page, "example/long.py").locator('button[type="submit"]').is_disabled()
|
||||
assert task_form(page, "related/change.py").locator('button[type="submit"]').is_disabled()
|
||||
assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled()
|
||||
assert task_form(page, "example/long.py").locator(".task-progress").inner_text() == "2 of 5\nETA: 3 minutes"
|
||||
finally:
|
||||
context.close()
|
||||
|
||||
|
||||
def test_live_changes_and_racing_conflict_show_blocking_run(live_page):
|
||||
app, url, page = live_page
|
||||
page.goto(url)
|
||||
snapshot = {"value": app.test_client().get("/tasks/state").get_json()}
|
||||
page.route("**/tasks/state", lambda route: route.fulfill(json=snapshot["value"]))
|
||||
run = start_live_run(app, status="queued")
|
||||
related = task_form(page, "related/change.py")
|
||||
with page.expect_response(lambda response: response.url.endswith("/tasks/related/change.py/run") and response.status == 409):
|
||||
related.locator('button[type="submit"]').click()
|
||||
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
|
||||
playwright.expect(related.locator(".task-result")).to_contain_text(f"Blocked by run #{run.id}")
|
||||
assert related.locator('button[type="submit"]').is_disabled()
|
||||
own = task_form(page, "example/long.py")
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text("Queued")
|
||||
app.extensions["run_store"].claim_next("example")
|
||||
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text("Running")
|
||||
app.extensions["run_store"].finish(run.id, status="succeeded")
|
||||
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text("Completed successfully.")
|
||||
playwright.expect(related.locator('button[type="submit"]')).to_be_enabled()
|
||||
|
||||
|
||||
def test_long_task_submission_redirects_and_restores_on_return(live_page):
|
||||
app, url, page = live_page
|
||||
page.goto(url)
|
||||
task_form(page, "example/long.py").locator('button[type="submit"]').click()
|
||||
page.wait_for_url("**/runs/*")
|
||||
playwright.expect(page.locator("#run-status")).to_have_text("queued")
|
||||
page.locator('main a[href="/"]').first.click()
|
||||
playwright.expect(task_form(page, "example/long.py").locator(".task-result")).to_contain_text("Queued")
|
||||
assert app.extensions["run_store"].last_run("task", "example/long.py").status == "queued"
|
||||
|
||||
|
||||
def test_csv_download_after_reload_does_not_repeat(live_page, settings):
|
||||
app, url, page = live_page
|
||||
task = write_task(settings.task_root, "exports", "review.py", 'print("done")\n')
|
||||
(task.parent / "task-inputs.json").write_text(json.dumps({
|
||||
"version": 1, "tasks": {task.name: {"wait_for_result": True, "download_artifacts": True}},
|
||||
}))
|
||||
page.goto(url)
|
||||
form = task_form(page, "exports/review.py")
|
||||
form.locator('button[type="submit"]').click()
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Queued")
|
||||
page.reload()
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Queued")
|
||||
assert page.evaluate("window.leaveListeners") == []
|
||||
run = app.extensions["run_store"].last_run("task", "exports/review.py")
|
||||
artifacts = settings.state_root / "runs" / f"{run.id}.artifacts"
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
(artifacts / "report.json").write_text('{"ok":true}')
|
||||
(artifacts / "review.csv").write_text("item,decision\n1,keep\n")
|
||||
downloads = []
|
||||
page.on("download", lambda download: downloads.append(download.suggested_filename))
|
||||
with page.expect_download() as downloaded:
|
||||
app.extensions["run_store"].finish(run.id, status="succeeded")
|
||||
download = downloaded.value
|
||||
assert download.suggested_filename == "review.csv"
|
||||
assert Path(download.path()).read_text() == "item,decision\n1,keep\n"
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Completed successfully.")
|
||||
assert form.locator('button[type="submit"]').is_enabled()
|
||||
page.wait_for_timeout(2200)
|
||||
assert downloads == ["review.csv"]
|
||||
assert (artifacts / "report.json").is_file()
|
||||
assert not (artifacts / "review.csv").exists()
|
||||
|
||||
|
||||
def test_incremental_output_preserves_selection_and_stops_polling(live_page, settings):
|
||||
app, url, page = live_page
|
||||
run = start_live_run(app)
|
||||
log = settings.state_root / "runs" / f"{run.id}.log"
|
||||
log.write_text("Select this text\n")
|
||||
store = app.extensions["run_store"]
|
||||
store.set_log_path(run.id, str(log))
|
||||
polls = []
|
||||
artifacts = []
|
||||
|
||||
def record(request):
|
||||
if "/output?" in request.url:
|
||||
polls.append(request.url)
|
||||
if request.url.endswith("/artifacts"):
|
||||
artifacts.append(request.url)
|
||||
|
||||
page.on("request", record)
|
||||
page.goto(f"{url}/runs/{run.id}")
|
||||
page.evaluate("""() => {
|
||||
const node = document.querySelector('#run-output').firstChild;
|
||||
const range = document.createRange();
|
||||
range.setStart(node, 0); range.setEnd(node, 16);
|
||||
getSelection().removeAllRanges(); getSelection().addRange(range);
|
||||
}""")
|
||||
with log.open("a") as stream:
|
||||
stream.write("More output\n")
|
||||
progress = settings.state_root / "runs" / f"{run.id}.progress.json"
|
||||
temporary = progress.with_suffix(".tmp")
|
||||
temporary.write_text(json.dumps({"text": "3 of 5\nETA: 2 minutes"}))
|
||||
temporary.replace(progress)
|
||||
playwright.expect(page.locator("#run-output")).to_contain_text("More output")
|
||||
playwright.expect(page.locator("#run-progress")).to_have_text("3 of 5\nETA: 2 minutes")
|
||||
assert page.evaluate("getSelection().toString()") == "Select this text"
|
||||
assert not artifacts
|
||||
with log.open("a") as stream:
|
||||
stream.write("Finished\n")
|
||||
store.finish(run.id, status="succeeded")
|
||||
playwright.expect(page.locator("#run-output")).to_contain_text("Finished")
|
||||
playwright.expect(page.locator('form[action$="/cancel"]')).to_have_count(0)
|
||||
assert page.evaluate("getSelection().toString()") == "Select this text"
|
||||
finished_polls = len(polls)
|
||||
page.wait_for_timeout(2300)
|
||||
assert len(polls) == finished_polls
|
||||
assert len(artifacts) == 1
|
||||
assert all("offset=" in request for request in polls)
|
||||
|
||||
|
||||
def test_execute_control_appears_and_tracks_conflicts_after_completion(live_page, settings):
|
||||
app, url, page = live_page
|
||||
manifest = settings.task_root / "example" / "task-inputs.json"
|
||||
manifest.write_text(json.dumps({
|
||||
"version": 1, "tasks": {"long.py": {"execution": {
|
||||
"field": "mode", "dry_run_value": "review", "dry_run_label": "Review",
|
||||
"execute_value": "apply", "execute_label": "Execute",
|
||||
}}},
|
||||
}))
|
||||
manager = app.extensions["run_manager"]
|
||||
catalog = app.extensions["catalog"]
|
||||
run, _ = manager.run_task(catalog.task("example/long.py"), {"mode": "review"})
|
||||
manager.store.claim_next("example")
|
||||
page.goto(f"{url}/runs/{run.id}")
|
||||
manager.store.finish(run.id, status="succeeded")
|
||||
execute = task_form(page, "example/long.py")
|
||||
playwright.expect(execute.locator('button[type="submit"]')).to_be_visible()
|
||||
blocker, _ = manager.run_task(catalog.task("example/sibling.py"))
|
||||
playwright.expect(execute.locator(".task-result")).to_contain_text(f"Blocked by run #{blocker.id}")
|
||||
assert execute.locator('button[type="submit"]').is_disabled()
|
||||
manager.store.finish(blocker.id, status="succeeded")
|
||||
playwright.expect(execute.locator('button[type="submit"]')).to_be_enabled()
|
||||
execute.locator('button[type="submit"]').click()
|
||||
page.wait_for_url(lambda current: current.startswith(f"{url}/runs/") and current != f"{url}/runs/{run.id}")
|
||||
accepted = manager.store.last_run("task", "example/long.py")
|
||||
assert accepted.id != run.id
|
||||
assert json.loads(accepted.input_json) == {"mode": "apply"}
|
||||
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.discovery import TaskCatalog
|
||||
from app.runner import RunManager, TaskConflictError
|
||||
from app.store import RunStore
|
||||
|
||||
from .conftest import write_task
|
||||
from .test_runner import wait_for, wait_for_output
|
||||
|
||||
|
||||
COUNTER = '''"""Reconcile a small external ledger before continuing."""
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
def main():
|
||||
data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text())
|
||||
ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt'
|
||||
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
||||
if not runtime.checkpoint_data():
|
||||
runtime.checkpoint({'next': 0}, force=True)
|
||||
for number in range(len(existing), data.get('count', 50)):
|
||||
runtime.check_stop()
|
||||
with ledger.open('a') as stream:
|
||||
stream.write(str(number) + '\\n')
|
||||
runtime.checkpoint({'next': number + 1})
|
||||
runtime.progress(f'Items: {number + 1}', force=True)
|
||||
runtime.sleep(.03)
|
||||
|
||||
def recover():
|
||||
print('Checking external ledger', flush=True)
|
||||
main()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager):
|
||||
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
|
||||
task = manager.catalog.task('count/task.py')
|
||||
run, _ = manager.run_task(task, {'count': 50})
|
||||
wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3')
|
||||
manager.shutdown(timeout=1)
|
||||
assert manager.store.get(run.id).status == 'interrupted'
|
||||
work = manager._work_root(run)
|
||||
saved = json.loads((work / 'checkpoint.json').read_text())
|
||||
assert saved['clean_shutdown'] is True
|
||||
assert (work / 'input.json').exists()
|
||||
manager.cleanup_transient_files(artifact_max_age_seconds=0)
|
||||
assert (work / 'input.json').exists()
|
||||
resumed = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
resumed.startup()
|
||||
recovery = manager.store.recovery_for(run.id)
|
||||
assert recovery is not None
|
||||
assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded'
|
||||
assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
|
||||
assert manager.store.get(run.id).status == 'interrupted'
|
||||
assert not (work / 'input.json').exists()
|
||||
assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text()
|
||||
resumed.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_forced_container_style_exit_recovers_from_older_checkpoint(settings):
|
||||
settings.task_root.mkdir(parents=True)
|
||||
write_task(settings.task_root, 'count', 'task.py', COUNTER)
|
||||
control = settings.state_root / 'control.json'
|
||||
program = '''
|
||||
import json,time
|
||||
from pathlib import Path
|
||||
from app.config import Settings
|
||||
from app.discovery import TaskCatalog
|
||||
from app.runner import RunManager
|
||||
from app.store import RunStore
|
||||
s=Settings.from_environment()
|
||||
c=TaskCatalog(s.task_root)
|
||||
db=RunStore(s.database_path)
|
||||
db.initialize()
|
||||
m=RunManager(s,c,db)
|
||||
r,_=m.run_task(c.task('count/task.py'),{'count':50})
|
||||
while r.id not in m._processes:
|
||||
time.sleep(.01)
|
||||
Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid}))
|
||||
while True:
|
||||
time.sleep(1)
|
||||
'''
|
||||
environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]),
|
||||
'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root),
|
||||
'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root),
|
||||
'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)}
|
||||
process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True)
|
||||
child_pid = None
|
||||
resumed = None
|
||||
try:
|
||||
wait_for_output(control, 'pid')
|
||||
data = json.loads(control.read_text())
|
||||
child_pid = data['pid']
|
||||
wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
||||
process.kill()
|
||||
process.wait(timeout=3)
|
||||
os.killpg(child_pid, signal.SIGKILL)
|
||||
child_pid = None
|
||||
saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text())
|
||||
assert saved['clean_shutdown'] is False
|
||||
assert saved['data']['next'] == 0
|
||||
store = RunStore(settings.database_path)
|
||||
resumed = RunManager(settings, TaskCatalog(settings.task_root), store)
|
||||
resumed.startup()
|
||||
recovery = store.recovery_for(data['run'])
|
||||
assert recovery is not None
|
||||
assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded'
|
||||
assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=3)
|
||||
if child_pid is not None:
|
||||
try:
|
||||
os.killpg(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
if resumed:
|
||||
resumed.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_cancelled_tasks_never_auto_recover(manager):
|
||||
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
|
||||
task = manager.catalog.task('count/task.py')
|
||||
run, _ = manager.run_task(task, {'count': 100})
|
||||
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
||||
manager.request_cancel(run.id)
|
||||
wait_for(manager, run.id, {'cancelled'})
|
||||
manager.shutdown(timeout=1)
|
||||
new = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
new.startup()
|
||||
assert manager.store.recovery_for(run.id) is None
|
||||
assert not (manager._work_root(run) / 'input.json').exists()
|
||||
new.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager):
|
||||
path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():'))
|
||||
run, _ = manager.run_task(manager.catalog.task('count/task.py'))
|
||||
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
||||
manager.shutdown(timeout=1)
|
||||
assert manager.store.get(run.id).status == 'interrupted'
|
||||
path.write_text(COUNTER)
|
||||
legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy')
|
||||
write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER)
|
||||
# The current task has opted out again; neither it nor a pre-runtime run may resume.
|
||||
path.write_text(COUNTER.replace('def recover():', 'def unused():'))
|
||||
new = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
new.startup()
|
||||
assert manager.store.recovery_for(run.id) is None
|
||||
assert manager.store.recovery_for(legacy.id) is None
|
||||
assert manager.store.active_runs() == []
|
||||
new.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_shared_resources_block_cross_group_submissions_and_recoveries(manager):
|
||||
for group in ('first','second'):
|
||||
write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER)
|
||||
first = manager.catalog.task('first/task.py')
|
||||
second = manager.catalog.task('second/task.py')
|
||||
run, _ = manager.run_task(first, {'count':50})
|
||||
with pytest.raises(TaskConflictError) as error:
|
||||
manager.run_task(second)
|
||||
assert error.value.run.id == run.id
|
||||
manager.shutdown(timeout=1)
|
||||
new = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
new.startup()
|
||||
recovered = manager.store.recovery_for(run.id)
|
||||
assert recovered is not None
|
||||
with pytest.raises(TaskConflictError) as error:
|
||||
new.run_task(second)
|
||||
assert error.value.run.id == recovered.id
|
||||
new.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_shutdown_rejects_new_work(manager):
|
||||
write_task(manager.settings.task_root,'tasks','task.py','print("hello")')
|
||||
manager.shutdown(timeout=.1)
|
||||
with pytest.raises(RuntimeError,match='shutting down'):
|
||||
manager.run_task(manager.catalog.task('tasks/task.py'))
|
||||
@@ -58,7 +58,11 @@ def test_tasks_in_one_group_are_serialized_and_groups_overlap(manager: RunManage
|
||||
other_task = catalog.task("second_group/other.py")
|
||||
assert first_task is not None and second_task is not None and other_task is not None
|
||||
first, _ = manager.run_task(first_task)
|
||||
second, _ = manager.run_task(second_task)
|
||||
second, _ = manager.store.enqueue(
|
||||
target_kind="task", target_path=second_task.id, queue_group=second_task.group_id,
|
||||
resources=second_task.resources,
|
||||
)
|
||||
manager._wake_worker(second_task.group_id)
|
||||
other, _ = manager.run_task(other_task)
|
||||
|
||||
wait_for(manager, first.id, {"running"})
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
"""Recovery through the real Waitress entry point, without manager test doubles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import closing
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.test import EnvironBuilder
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration")
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n"
|
||||
SECRET = "disposable-process-test-secret"
|
||||
|
||||
|
||||
LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
RESOURCES = ("shared-ledger",)
|
||||
state = Path(os.environ["SERVER_MAINTENANCE_STATE"])
|
||||
|
||||
def values():
|
||||
data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||
assert data["secret"] == "disposable-process-test-secret"
|
||||
assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n"
|
||||
return data
|
||||
|
||||
def main():
|
||||
data = values()
|
||||
ledger = state / "ledger.txt"
|
||||
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
||||
if not runtime.checkpoint_data():
|
||||
runtime.checkpoint({"next": 0}, force=True)
|
||||
items = Path(data["items"]).read_text().splitlines()
|
||||
for number in range(len(existing), len(items)):
|
||||
runtime.check_stop()
|
||||
with ledger.open("a") as stream:
|
||||
stream.write(items[number] + "\\n")
|
||||
runtime.checkpoint({"next": number + 1})
|
||||
if number == 2:
|
||||
runtime.progress("Original waiting", force=True)
|
||||
while not (state / "finish").exists():
|
||||
runtime.sleep(.02)
|
||||
|
||||
def recover():
|
||||
values()
|
||||
(state / "observation.json").write_text(json.dumps({
|
||||
"checkpoint": runtime.checkpoint_data(),
|
||||
"clean_shutdown": runtime.was_clean_shutdown(),
|
||||
"ledger": (state / "ledger.txt").read_text().splitlines(),
|
||||
}))
|
||||
runtime.progress("Recovery waiting", force=True)
|
||||
while not (state / "finish").exists():
|
||||
runtime.sleep(.02)
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
runtime.checkpoint({"next": 0}, force=True)
|
||||
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
|
||||
runtime.checkpoint({"next": 1})
|
||||
Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text(
|
||||
json.dumps([os.getpid(), child.pid]))
|
||||
runtime.progress("Blocking", force=True)
|
||||
time.sleep(60)
|
||||
'''
|
||||
|
||||
|
||||
def wait_until(condition, description, timeout=8):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = condition()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(.02)
|
||||
pytest.fail(f"Timed out waiting for {description}")
|
||||
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def process_running(pid):
|
||||
try:
|
||||
status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1]
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return status[0] not in {"Z", "X"}
|
||||
|
||||
|
||||
class ProductionApp:
|
||||
def __init__(self, root):
|
||||
self.task_root = root / "tasks"
|
||||
self.state_root = root / "state"
|
||||
self.credentials_root = root / "credentials"
|
||||
for path in (self.task_root, self.state_root, self.credentials_root):
|
||||
path.mkdir()
|
||||
self.database = self.state_root / "server-maintenance.sqlite3"
|
||||
self.log_path = root / "server.log"
|
||||
self.process = None
|
||||
self.port = None
|
||||
self.grace = 1
|
||||
|
||||
def write_task(self, group, filename, source):
|
||||
path = self.task_root / group / filename
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(source, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def start(self):
|
||||
with socket.socket() as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
port = listener.getsockname()[1]
|
||||
self.launch(port)
|
||||
|
||||
def ready():
|
||||
assert self.process.poll() is None, self.log_path.read_text()
|
||||
try:
|
||||
status, _ = self.request("GET", "/tasks/state")
|
||||
except (ConnectionError, TimeoutError):
|
||||
return False
|
||||
return status == 200
|
||||
|
||||
wait_until(ready, f"Waitress startup; log: {self.log_path}")
|
||||
|
||||
def launch(self, port):
|
||||
assert self.process is None or self.process.poll() is not None
|
||||
self.port = port
|
||||
environment = {key: value for key, value in os.environ.items()
|
||||
if not key.startswith("SERVER_MAINTENANCE_")}
|
||||
environment.update({
|
||||
"PYTHONPATH": str(PROJECT_ROOT),
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"SERVER_MAINTENANCE_HOST": "127.0.0.1",
|
||||
"SERVER_MAINTENANCE_PORT": str(self.port),
|
||||
"SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root),
|
||||
"SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root),
|
||||
"SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root),
|
||||
"SERVER_MAINTENANCE_PYTHON": sys.executable,
|
||||
"SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace),
|
||||
})
|
||||
with self.log_path.open("ab") as log:
|
||||
self.process = subprocess.Popen(
|
||||
[sys.executable, "-m", "app"], cwd=PROJECT_ROOT,
|
||||
env=environment, stdin=subprocess.DEVNULL, stdout=log,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
)
|
||||
|
||||
def request(self, method, path, data=None):
|
||||
headers = {"Accept": "application/json"}
|
||||
body = None
|
||||
if data is not None:
|
||||
with closing(EnvironBuilder(method=method, data=data)) as builder:
|
||||
environ = builder.get_environ()
|
||||
body = environ["wsgi.input"].read()
|
||||
headers["Content-Type"] = environ["CONTENT_TYPE"]
|
||||
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2)
|
||||
try:
|
||||
connection.request(method, path, body=body, headers=headers)
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
assert response.getheader("Content-Type", "").startswith("application/json"), payload
|
||||
return response.status, json.loads(payload)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def rows(self):
|
||||
with closing(sqlite3.connect(self.database)) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")]
|
||||
|
||||
def run(self, task, data=None):
|
||||
status, response = self.request("POST", f"/tasks/{task}/run", data)
|
||||
assert status == 202, response
|
||||
return response["run_id"]
|
||||
|
||||
def wait_status(self, run_id, status):
|
||||
def matching():
|
||||
run = next(row for row in self.rows() if row["id"] == run_id)
|
||||
if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}:
|
||||
assert run["status"] == status, (run, self.log_path.read_text())
|
||||
return run if run["status"] == status else None
|
||||
return wait_until(matching, f"run {run_id} to become {status}")
|
||||
|
||||
def wait_progress(self, run_id, text):
|
||||
path = self.state_root / "runs" / f"{run_id}.progress.json"
|
||||
wait_until(lambda: (read_json(path) or {}).get("text") == text, text)
|
||||
|
||||
def pids(self):
|
||||
# Match only this disposable instance, including separately-sessioned workers.
|
||||
marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode()
|
||||
found = []
|
||||
for path in Path("/proc").iterdir():
|
||||
if not path.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
environment = (path / "environ").read_bytes().split(b"\0")
|
||||
except (FileNotFoundError, ProcessLookupError, PermissionError):
|
||||
continue
|
||||
if marker in environment:
|
||||
found.append(int(path.name))
|
||||
return found
|
||||
|
||||
def kill_remaining(self):
|
||||
def kill_snapshot():
|
||||
pids = self.pids()
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return not pids
|
||||
|
||||
kill_snapshot()
|
||||
if self.process is not None:
|
||||
self.process.wait(timeout=3)
|
||||
wait_until(kill_snapshot, "disposable instance teardown", timeout=3)
|
||||
|
||||
def stop(self):
|
||||
started = time.monotonic()
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
try:
|
||||
code = self.process.wait(timeout=self.grace + 3)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}")
|
||||
elapsed = time.monotonic() - started
|
||||
assert code == 0, self.log_path.read_text()
|
||||
assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text())
|
||||
return elapsed
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def production_app(tmp_path):
|
||||
app = ProductionApp(tmp_path)
|
||||
try:
|
||||
yield app
|
||||
finally:
|
||||
app.kill_remaining()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"])
|
||||
def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal):
|
||||
app = production_app
|
||||
ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK)
|
||||
(ledger_task.parent / "task-inputs.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"tasks": {"task.py": {"inputs": [
|
||||
{"name": "items", "label": "Items", "type": "file", "required": True,
|
||||
"accept": [".txt"], "maximum_bytes": 1024},
|
||||
{"name": "secret", "label": "Secret", "type": "text", "required": True,
|
||||
"sensitive": True},
|
||||
]}},
|
||||
}))
|
||||
app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n')
|
||||
app.write_task("maintenance", "task.py", 'print("done")\n')
|
||||
app.start()
|
||||
original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")})
|
||||
app.wait_progress(original_id, "Original waiting")
|
||||
runs = app.state_root / "runs"
|
||||
work = runs / f"{original_id}.work"
|
||||
saved_input = json.loads((work / "input.json").read_text())
|
||||
upload = Path(saved_input["items"])
|
||||
assert saved_input["secret"] == SECRET
|
||||
assert SECRET not in app.rows()[0]["input_json"]
|
||||
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
||||
old_pids = app.pids()
|
||||
if shutdown_signal == signal.SIGTERM:
|
||||
app.stop()
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
assert app.rows()[0]["exit_code"] == 75
|
||||
else:
|
||||
# SIGKILL the entire disposable instance, as container teardown does.
|
||||
app.kill_remaining()
|
||||
assert app.process.returncode == -signal.SIGKILL
|
||||
assert app.rows()[0]["status"] == "running"
|
||||
wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit")
|
||||
saved = read_json(work / "checkpoint.json")
|
||||
clean = shutdown_signal == signal.SIGTERM
|
||||
assert saved["clean_shutdown"] is clean
|
||||
assert saved["data"] == {"next": 3 if clean else 0}
|
||||
assert upload.read_bytes() == UPLOAD
|
||||
assert json.loads((work / "input.json").read_text()) == saved_input
|
||||
|
||||
stale = time.time() - 45 * 86400
|
||||
for path in (work, work / "input.json", upload.parent, upload):
|
||||
os.utime(path, (stale, stale))
|
||||
app.start()
|
||||
recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id)
|
||||
app.wait_progress(recovery["id"], "Recovery waiting")
|
||||
observation = read_json(app.state_root / "tasks/ledger/observation.json")
|
||||
assert observation == {
|
||||
"checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean,
|
||||
"ledger": ["0", "1", "2"],
|
||||
}
|
||||
status, states = app.request("GET", "/tasks/state")
|
||||
assert status == 200
|
||||
assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering"
|
||||
assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"]
|
||||
status, conflict = app.request("POST", "/tasks/contender/task.py/run")
|
||||
assert status == 409, conflict
|
||||
assert conflict["blocked_by"]["run_id"] == recovery["id"]
|
||||
assert not any(row["target_path"] == "contender/task.py" for row in app.rows())
|
||||
|
||||
# A real completion invokes cleanup while recovery still needs its ancestor's files.
|
||||
cleanup_probe = runs / "999999.input.json"
|
||||
cleanup_probe.write_text("{}\n")
|
||||
maintenance_id = app.run("maintenance/task.py")
|
||||
app.wait_status(maintenance_id, "succeeded")
|
||||
wait_until(lambda: not cleanup_probe.exists(), "completion cleanup")
|
||||
assert upload.read_bytes() == UPLOAD
|
||||
assert json.loads((work / "input.json").read_text()) == saved_input
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
|
||||
(app.state_root / "tasks/ledger/finish").touch()
|
||||
app.wait_status(recovery["id"], "succeeded")
|
||||
assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)]
|
||||
wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup")
|
||||
app.stop()
|
||||
assert not app.pids(), "A task escaped registration/shutdown"
|
||||
|
||||
|
||||
def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app):
|
||||
app = production_app
|
||||
app.write_task("blocking", "task.py", BLOCKING_TASK)
|
||||
app.write_task("watcher", "internal/task.py", BLOCKING_TASK)
|
||||
(app.task_root / "watcher/background-tasks.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}],
|
||||
}))
|
||||
app.start()
|
||||
run_id = app.run("blocking/task.py")
|
||||
app.wait_progress(run_id, "Blocking")
|
||||
background_work = app.state_root / "background/watcher/watcher.work"
|
||||
wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking",
|
||||
"background worker to start")
|
||||
work = app.state_root / "runs" / f"{run_id}.work"
|
||||
task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json")
|
||||
assert all(process_running(pid) for pid in task_pids)
|
||||
elapsed = app.stop()
|
||||
assert elapsed >= app.grace * .8, "The hard deadline was not exercised"
|
||||
wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit")
|
||||
assert not app.pids(), "A foreground/background task escaped shutdown registration"
|
||||
assert read_json(work / "checkpoint.json")["clean_shutdown"] is False
|
||||
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
||||
assert (work / "input.json").exists()
|
||||
|
||||
|
||||
def test_production_graceful_pause_does_not_leave_live_descendants(production_app):
|
||||
app = production_app
|
||||
source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n")
|
||||
app.write_task("cooperative", "task.py", source)
|
||||
app.start()
|
||||
run_id = app.run("cooperative/task.py")
|
||||
app.wait_progress(run_id, "Blocking")
|
||||
work = app.state_root / "runs" / f"{run_id}.work"
|
||||
worker_pid, child_pid = read_json(work / "pids.json")
|
||||
app.stop()
|
||||
assert read_json(work / "checkpoint.json")["clean_shutdown"] is True
|
||||
assert not process_running(worker_pid)
|
||||
wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5)
|
||||
|
||||
|
||||
def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app):
|
||||
app = production_app
|
||||
source = '''
|
||||
from app import task_runtime as runtime
|
||||
runtime.progress("Started", force=True)
|
||||
while True:
|
||||
runtime.sleep(.02)
|
||||
'''
|
||||
groups = [f"queue_{index}" for index in range(12)]
|
||||
for group in groups:
|
||||
app.write_task(group, "task.py", source)
|
||||
app.start()
|
||||
first_id = app.run(f"{groups[0]}/task.py")
|
||||
app.wait_progress(first_id, "Started")
|
||||
|
||||
def submit(group):
|
||||
try:
|
||||
return app.request("POST", f"/tasks/{group}/task.py/run")[0]
|
||||
except (ConnectionError, TimeoutError, http.client.RemoteDisconnected):
|
||||
# The listener may close before this in-flight request is accepted.
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as requests:
|
||||
pending = [requests.submit(submit, group) for group in groups[1:]]
|
||||
app.stop()
|
||||
statuses = [request.result(timeout=3) for request in pending]
|
||||
assert all(status in {202, 503, None} for status in statuses), statuses
|
||||
assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration"
|
||||
assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text()
|
||||
|
||||
|
||||
def test_production_bind_failure_stops_workers_started_by_recovery(production_app):
|
||||
app = production_app
|
||||
source = '''
|
||||
from app import task_runtime as runtime
|
||||
|
||||
def main():
|
||||
runtime.progress("Started", force=True)
|
||||
while True:
|
||||
runtime.sleep(.02)
|
||||
|
||||
def recover():
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
app.write_task("recoverable", "task.py", source)
|
||||
app.start()
|
||||
run_id = app.run("recoverable/task.py")
|
||||
app.wait_progress(run_id, "Started")
|
||||
app.stop()
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
|
||||
# Delay startup cleanup without patching app code, until recovery is running.
|
||||
cleanup_gate = app.state_root / "tasks/.orphaned-groups.json"
|
||||
os.mkfifo(cleanup_gate)
|
||||
with socket.socket() as occupied:
|
||||
occupied.bind(("127.0.0.1", 0))
|
||||
occupied.listen()
|
||||
app.launch(occupied.getsockname()[1])
|
||||
recovery = wait_until(
|
||||
lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None),
|
||||
"startup recovery registration",
|
||||
)
|
||||
app.wait_progress(recovery["id"], "Started")
|
||||
|
||||
def release_cleanup():
|
||||
try:
|
||||
descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK)
|
||||
except OSError as error:
|
||||
if error.errno == errno.ENXIO:
|
||||
return False
|
||||
raise
|
||||
with os.fdopen(descriptor, "w") as gate:
|
||||
json.dump({}, gate)
|
||||
return True
|
||||
|
||||
wait_until(release_cleanup, "startup cleanup gate", timeout=2)
|
||||
assert app.process.wait(timeout=4) != 0
|
||||
assert "Address already in use" in app.log_path.read_text()
|
||||
assert any(row["recovery_of_run_id"] == run_id for row in app.rows())
|
||||
wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)
|
||||
@@ -0,0 +1,81 @@
|
||||
import json
|
||||
import signal
|
||||
|
||||
import pytest
|
||||
|
||||
from app import task_runtime as runtime
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def work(tmp_path,monkeypatch):
|
||||
monkeypatch.setenv('SERVER_MAINTENANCE_WORK',str(tmp_path))
|
||||
old=signal.getsignal(signal.SIGUSR1)
|
||||
runtime.initialize()
|
||||
yield tmp_path
|
||||
signal.signal(signal.SIGUSR1,old)
|
||||
runtime._pause_requested=False
|
||||
runtime._defer=0
|
||||
|
||||
|
||||
def test_safe_point_saves_latest_memory_state_and_marks_clean(work):
|
||||
runtime.checkpoint({'next':1},force=True)
|
||||
runtime.checkpoint({'next':2})
|
||||
assert json.loads((work/'checkpoint.json').read_text())['data']=={'next':1}
|
||||
runtime.request_pause()
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
runtime.check_stop()
|
||||
saved=json.loads((work/'checkpoint.json').read_text())
|
||||
assert saved['data']=={'next':2}
|
||||
assert saved['clean_shutdown'] is True
|
||||
runtime.initialize()
|
||||
assert runtime.was_clean_shutdown() is True
|
||||
assert json.loads((work/'checkpoint.json').read_text())['clean_shutdown'] is False
|
||||
|
||||
|
||||
def test_small_critical_operation_finishes_before_pause(work):
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
with runtime.critical_section():
|
||||
runtime.request_pause()
|
||||
runtime.checkpoint({'committed':True},force=True)
|
||||
assert runtime.checkpoint_data()=={'committed':True}
|
||||
saved=json.loads((work/'checkpoint.json').read_text())
|
||||
assert saved['data']=={'committed':True}
|
||||
assert saved['clean_shutdown'] is True
|
||||
|
||||
|
||||
def test_sleep_is_interruptible(work):
|
||||
runtime.request_pause()
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
runtime.sleep(30)
|
||||
|
||||
|
||||
def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
|
||||
clock=[100.]
|
||||
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
|
||||
progress=runtime.Progress({'Removing files':10,'Linking files':20})
|
||||
progress.update('Removing files',0,force=True)
|
||||
clock[0]+=30
|
||||
progress.update('Removing files',5,force=True)
|
||||
text=json.loads((work/'progress.json').read_text())['text']
|
||||
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
|
||||
assert (work/'progress.json').stat().st_mode&0o777==0o600
|
||||
|
||||
|
||||
def test_runtime_does_not_write_or_pause_direct_scripts_without_work(monkeypatch):
|
||||
monkeypatch.delenv('SERVER_MAINTENANCE_WORK',raising=False)
|
||||
runtime.request_pause()
|
||||
runtime.checkpoint({'ignored':True})
|
||||
runtime.progress('hello',force=True)
|
||||
assert runtime.checkpoint_data()=={}
|
||||
runtime._pause_requested=False
|
||||
|
||||
|
||||
def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
|
||||
clock=[100.]
|
||||
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
|
||||
progress=runtime.Progress({'Items':50})
|
||||
progress.update('Items',39,force=True)
|
||||
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
|
||||
clock[0]+=60
|
||||
progress.update('Items',40,force=True)
|
||||
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
|
||||
@@ -141,3 +141,35 @@ def test_removing_file_input_from_task_cleans_up_unused_review_upload(settings):
|
||||
manifest.write_text(json.dumps(definition))
|
||||
app.extensions["run_manager"].cleanup_transient_files()
|
||||
assert not original.parent.exists()
|
||||
|
||||
|
||||
def test_conflicting_execute_is_linked_and_keeps_review_upload(settings, monkeypatch):
|
||||
app, client, review_id = start_review(settings)
|
||||
manager = app.extensions["run_manager"]
|
||||
original = Path(json.loads(manager.store.get(review_id).input_json)["csv"])
|
||||
write_task(settings.task_root, "imports", "other.py", 'print("done")\n')
|
||||
monkeypatch.setattr(manager, "_wake_worker", lambda *_: None)
|
||||
blocker, _ = manager.run_task(app.extensions["catalog"].task("imports/other.py"))
|
||||
page = client.get(f"/runs/{review_id}")
|
||||
assert b'fieldset class="task-controls" disabled' in page.data
|
||||
assert f'href="/runs/{blocker.id}"'.encode() in page.data
|
||||
response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
|
||||
assert response.status_code == 409
|
||||
assert response.get_json()["blocked_by"]["run_id"] == blocker.id
|
||||
assert original.is_file()
|
||||
manager.store.finish(blocker.id, status="cancelled")
|
||||
accepted = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
|
||||
assert accepted.status_code == 202
|
||||
assert accepted.get_json()["detail_url"].startswith("/runs/")
|
||||
assert accepted.get_json()["run_id"] != review_id
|
||||
|
||||
|
||||
def test_shutdown_during_execute_keeps_review_upload(settings):
|
||||
app, client, review_id = start_review(settings)
|
||||
manager = app.extensions["run_manager"]
|
||||
original = Path(json.loads(manager.store.get(review_id).input_json)["csv"])
|
||||
manager._closing.set()
|
||||
response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
|
||||
assert response.status_code == 503
|
||||
assert "shutting down" in response.get_json()["error"]
|
||||
assert original.is_file()
|
||||
@@ -0,0 +1,56 @@
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
import pytest
|
||||
|
||||
from app.worker import service_unavailable
|
||||
from app import worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize('error', [
|
||||
ConnectionRefusedError(), TimeoutError(), URLError(OSError('connection refused')),
|
||||
HTTPError('http://example.invalid',503,'unavailable',{},None),
|
||||
])
|
||||
def test_service_failures_can_retry_recovery(error):
|
||||
wrapped=RuntimeError('Service request failed')
|
||||
wrapped.__cause__=error
|
||||
assert service_unavailable(wrapped)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('error', [
|
||||
RuntimeError('Unrecognized file'), FileNotFoundError(), PermissionError(),
|
||||
HTTPError('http://example.invalid',401,'unauthorized',{},None),
|
||||
HTTPError('http://example.invalid',400,'bad input',{},None),
|
||||
])
|
||||
def test_task_and_authentication_errors_are_not_automatically_repeated(error):
|
||||
assert not service_unavailable(error)
|
||||
|
||||
|
||||
def test_database_starting_up_is_transient_but_authentication_is_not():
|
||||
error=RuntimeError('Database unavailable')
|
||||
error.sqlstate='57P03'
|
||||
assert service_unavailable(error)
|
||||
error.sqlstate='28P01'
|
||||
assert not service_unavailable(error)
|
||||
|
||||
|
||||
def test_recovery_rechecks_live_state_after_lost_service_response(tmp_path, monkeypatch):
|
||||
ledger = []
|
||||
calls = []
|
||||
waits = []
|
||||
|
||||
def recover():
|
||||
calls.append(list(ledger))
|
||||
if not ledger:
|
||||
ledger.append('applied')
|
||||
raise URLError(ConnectionResetError('response lost'))
|
||||
|
||||
monkeypatch.setattr(worker.sys, 'argv', ['worker', str(tmp_path/'task.py')])
|
||||
monkeypatch.setenv('SERVER_MAINTENANCE_RECOVER','1')
|
||||
monkeypatch.setattr(worker.runtime,'initialize',lambda:None)
|
||||
monkeypatch.setattr(worker.runtime,'progress',lambda *a,**kw:None)
|
||||
monkeypatch.setattr(worker.runtime,'sleep',waits.append)
|
||||
monkeypatch.setattr(worker.runpy,'run_path',lambda *a,**kw:{'recover':recover})
|
||||
worker.main()
|
||||
assert ledger==['applied']
|
||||
assert calls==[[],['applied']]
|
||||
assert waits==[2]
|
||||
Reference in new issue
Block a user