diff --git a/.gitea/workflows/build-images.yaml b/.gitea/workflows/build-images.yaml index a72d7d0..041711c 100644 --- a/.gitea/workflows/build-images.yaml +++ b/.gitea/workflows/build-images.yaml @@ -131,6 +131,12 @@ jobs: tar -C timepoll -cf - . | docker run --rm -i --workdir /app python:3.12-slim \ sh -c 'tar -xf - && pip install --no-cache-dir -r requirements.txt pytest httpx && PYTHONDONTWRITEBYTECODE=1 pytest -q -p no:cacheprovider' + - name: Test Python Tools + if: matrix.image.name == 'python-tools' + run: | + tar -C python-tools -cf - . | docker run --rm --init -i --workdir /app python:3 \ + sh -c 'tar -xf - && pip install --no-cache-dir -r requirements.txt pytest && PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app python -m pytest -q -p no:cacheprovider' + - name: Log in to Gitea container registry run: | printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USERNAME" --password-stdin diff --git a/python-tools/README.md b/python-tools/README.md index f5fb5b3..0a24225 100644 --- a/python-tools/README.md +++ b/python-tools/README.md @@ -7,8 +7,9 @@ with `python -m app`. ## Control Panel The application discovers direct `*.py` files in each non-hidden directory -under `/opt/tasks`. Each directory is a task group. Tasks in one group run -sequentially; different groups may run concurrently. It records run history in +under `/opt/tasks`. Each directory is a task group. A group is exclusive while +one of its tasks runs; unrelated groups may run concurrently. Tasks can declare +additional shared resources to prevent conflicts across groups. It records run history in SQLite and combined output in log files under `/var/lib/server-maintenance`, and supports cancellation of active task process groups. @@ -81,9 +82,10 @@ Use `\n` for a line break or `\n\n` for a paragraph gap in descriptions. Each task declaration may contain `title`, `description`, `inputs`, `wait_for_result`, `download_artifacts`, and `execution`. Set -`wait_for_result` for short tasks where the form should remain blocked until it -reports success or failure. The control panel disables its controls and warns -before page unload while such a task runs. `download_artifacts` additionally +`wait_for_result` for tasks whose result should appear beside the form instead +of immediately opening the run-detail page. Running and conflicting controls +are disabled, but navigation is allowed. Opening or reloading the page restores +the current status and optional live progress from the server. `download_artifacts` additionally starts the first non-JSON artifact download after a successful browser run; reserve it for files such as archives that the browser should download automatically. @@ -101,8 +103,8 @@ Every input has `name`, `label`, and `type`; `required` defaults to `false`. Set `sensitive` to `true` on text or file inputs that must not appear in run history. Text is masked, the database records `[redacted]`, and the original -value is available only through a private execution-input file. Uploaded files -are likewise retained privately for the run and then removed. +value is available only through a private execution-input file. Interrupted +recoverable tasks retain that input and their uploads until recovery finishes. For review-first tasks, use `execution`. It produces one split button: click the main part to use its current mode, or use the arrow to select the other @@ -135,6 +137,96 @@ task without an entry in the manifest gets an empty JSON object. Uploaded files are stored in a private per-run directory; their absolute paths are provided in the input JSON. Files are removed when the task finishes, fails, or is cancelled, except for successful review uploads retained temporarily for execution as above. +Interrupted tasks with recovery support retain their files privately. + +## Shutdown And Recovery + +The Compose example grants 120 seconds for container shutdown. The runner stops +accepting work, pauses the background scheduler, and asks all task subprocesses +to save and stop at safe boundaries. It waits up to 110 seconds by default, then +terminates remaining process groups. The HTTP server gets a final short cleanup +window. These are maximum waits; idle or cooperative jobs stop sooner. The +Compose example also uses Docker's init process to reap orphaned subprocesses. +`SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS` changes the application deadline; keep +the container's `stop_grace_period` longer than that deadline. + +Tasks opt in by defining a top-level, synchronous `recover()` function with no +arguments in their own Python file. Discovery reads its syntax without importing +the task. On startup, a new linked recovery run enters the ordinary task queue +and calls that function in an isolated worker process. Original logs and status +remain unchanged. Cancelled tasks never recover automatically. Tasks without a +hook remain interrupted. Historical runs created before this recovery protocol +are not automatically replayed. +Recovery waits and retries transient connection failures while services start, +with a maximum 30-second interval. Authentication failures and task validation +errors are reported rather than automatically repeated. + +Recovery code must inspect current files or remote state before repeating an +operation. A lost response does not prove that the operation failed. Save an +intent before an irreversible operation and reconcile its outcome after restart. +For repeatable scans, a periodic checkpoint is sufficient. The runner cannot +serialize Python stacks or reconstruct task-specific state automatically. + +Use `from app import task_runtime as runtime` in task code: + +- `runtime.checkpoint_data()` reads the saved dictionary on recovery. +- `runtime.checkpoint(data)` updates the in-memory snapshot and persists ordinary + repeatable work at most once a minute. Use `force=True` for important intents, + completed results, and other state that must precede an external mutation. +- `runtime.check_stop()` observes a pause at a safe boundary, saves the latest + snapshot, and unwinds the task. Do not catch its `TaskPaused` exception. + Exit code 75 is reserved for this cooperative pause. +- `runtime.critical_section()` defers a pause through a small consistency-sensitive + action. Do not wrap an entire job or an unbounded transfer in it. +- `runtime.sleep(seconds)` observes pauses during waits. +- `runtime.was_clean_shutdown()` distinguishes a completed shutdown checkpoint + from a periodic checkpoint. External state must still be checked in both cases. +- `runtime.work_directory()` is a private directory shared by the interrupted run + and its recovery chain. Use it for durable intermediate files, not artifacts. + +Keep the usual `if __name__ == "__main__":` guard. The recovery worker loads the +module under a different name, then calls `recover()`; module import must not +start work. A recovery hook may call the task's regular implementation with a +recovery flag, provided that implementation performs the necessary reconciliation. + +Work directories live at `runs/.work`. Exact inputs and uploads +remain protected while a run is active or awaits recovery. Raw work inputs are +removed on completion or cancellation; finished checkpoint directories are +retained for 14 days. Do not put passwords or other credentials in checkpoint or +progress dictionaries. Read credentials from their normal mounted files. + +### Shared Resources + +Declare optional resource names as a literal tuple at module level: + +```python +RESOURCES = ("shared-files",) +``` + +Two tasks sharing a resource cannot run together, including during automatic +recovery. The backend rejects conflicting submissions with HTTP 409 and identifies +the blocking run. Every task also holds its group's resource. Resource names are +parsed as data; no task code executes during discovery. + +### Live Progress + +`runtime.progress(text)` publishes optional plain text, including line breaks, +to the task and run pages. Updates are throttled to once a second; use +`force=True` at stage boundaries. Progress is independent of log output, so it +does not replace selected log text or require the browser to stay open. + +For simple item-count percentages and per-step estimates: + +```python +tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20}) +tracker.update("Scanning files", 0, force=True) +# After processing five files: +tracker.update("Scanning files", 5) +``` + +Each step has its own percentage and an approximate ETA based only on processed +items and elapsed time. Pending steps have no ETA. Tasks may instead publish their +own text when totals are unknown; no global estimate is required. A group may also include a private `task-config.json` for stable local details such as mounted filesystem roots. This is not part of discovery or the web diff --git a/python-tools/app/__main__.py b/python-tools/app/__main__.py index 59156a1..1883aa5 100644 --- a/python-tools/app/__main__.py +++ b/python-tools/app/__main__.py @@ -3,15 +3,50 @@ from __future__ import annotations import os +import signal +import threading +import time -from waitress import serve +from waitress import create_server from .web import create_app +def main() -> None: + stopping = False + requested_at = None + + def request_shutdown(_signal, _frame): + nonlocal stopping, requested_at + stopping = True + if requested_at is None: + requested_at = time.monotonic() + + signal.signal(signal.SIGTERM, request_shutdown) + signal.signal(signal.SIGINT, request_shutdown) + app, server = None, None + try: + app = create_app() + server = create_server( + app, + host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"), + port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")), + ) + thread = threading.Thread(target=server.run, name="http-server", daemon=True) + thread.start() + while not stopping and thread.is_alive(): + time.sleep(.1) + finally: + print("Shutdown requested; pausing tasks at safe checkpoints.", flush=True) + if app is not None: + app.extensions["background_scheduler"].stop() + manager = app.extensions["run_manager"] + elapsed = time.monotonic() - requested_at if requested_at is not None else 0 + manager.shutdown(timeout=max(0, manager.settings.shutdown_grace_seconds - elapsed)) + if server is not None: + server.task_dispatcher.shutdown(timeout=5) + server.close() + + if __name__ == "__main__": - serve( - create_app(), - host=os.environ.get("SERVER_MAINTENANCE_HOST", "0.0.0.0"), - port=int(os.environ.get("SERVER_MAINTENANCE_PORT", "8080")), - ) + main() diff --git a/python-tools/app/config.py b/python-tools/app/config.py index e9df242..ddcadd3 100644 --- a/python-tools/app/config.py +++ b/python-tools/app/config.py @@ -17,6 +17,7 @@ class Settings: timezone_name: str | None allowed_proxy_ips: frozenset[str] cancel_grace_seconds: float + shutdown_grace_seconds: float = 110 @property def database_path(self) -> Path: @@ -55,4 +56,5 @@ class Settings: cancel_grace_seconds=float( os.environ.get("SERVER_MAINTENANCE_CANCEL_GRACE_SECONDS", "10") ), + shutdown_grace_seconds=float(os.environ.get("SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS", "110")), ) diff --git a/python-tools/app/discovery.py b/python-tools/app/discovery.py index d02b093..2075a5c 100644 --- a/python-tools/app/discovery.py +++ b/python-tools/app/discovery.py @@ -39,6 +39,8 @@ class Task: wait_for_result: bool = False download_artifacts: bool = False execution: "ExecutionMode | None" = None + recoverable: bool = False + resources: tuple[str, ...] = () @dataclass(frozen=True) @@ -190,6 +192,7 @@ class TaskCatalog: for filename in ordered_filenames: path = paths_by_filename[filename] declaration = declarations_by_filename.get(path.name) + recoverable, resources = read_runtime_declaration(path) tasks.append( Task( id=f"{group_path.name}/{path.name}", @@ -210,6 +213,8 @@ class TaskCatalog: wait_for_result=declaration.wait_for_result if declaration else False, download_artifacts=declaration.download_artifacts if declaration else False, execution=declaration.execution if declaration else None, + recoverable=recoverable, + resources=(f"group:{group_path.name}", *resources), ) ) return tasks @@ -294,6 +299,23 @@ def read_description(path: Path) -> str | None: ) +def read_runtime_declaration(path: Path) -> tuple[bool, tuple[str, ...]]: + """Discover the opt-in hook and literal resource names without importing code.""" + try: + module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except (OSError, SyntaxError, UnicodeDecodeError): + return False, () + recoverable = any(isinstance(node, ast.FunctionDef) and node.name == "recover" for node in module.body) + resources = () + for node in module.body: + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "RESOURCES" for t in node.targets): + value = ast.literal_eval(node.value) + if not isinstance(value, (tuple, list)) or any(not isinstance(x, str) or not re.fullmatch(r"[a-z][a-z0-9_-]*", x) for x in value): + raise ValueError(f"RESOURCES in {path} must be literal resource names") + resources = tuple(dict.fromkeys(value)) + return recoverable, resources + + @dataclass(frozen=True) class GroupManifest: title: str | None diff --git a/python-tools/app/runner.py b/python-tools/app/runner.py index e7f37ca..b53c450 100644 --- a/python-tools/app/runner.py +++ b/python-tools/app/runner.py @@ -10,6 +10,7 @@ import subprocess import threading import time from collections.abc import Mapping +from dataclasses import replace from datetime import datetime from pathlib import Path @@ -17,6 +18,7 @@ from .config import Settings from .discovery import BackgroundTask, Group, Task, TaskCatalog from .inputs import PendingUpload from .store import Run, RunStore +from .task_runtime import PAUSED_EXIT_CODE, atomic_json ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json" @@ -24,6 +26,20 @@ ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400 REVIEW_UPLOAD_RETENTION_SECONDS = 86400 +class TaskConflictError(RuntimeError): + def __init__(self, run: Run): + self.run = run + super().__init__(f"Run {run.id} is using resources required by this task.") + + +def kill_remaining_group(process: subprocess.Popen) -> None: + # The leader can already be gone while a transfer subprocess is still alive. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + class RunManager: def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None: self.settings = settings @@ -32,7 +48,9 @@ class RunManager: self._wake_events: dict[str, threading.Event] = {} self._workers: dict[str, threading.Thread] = {} self._processes: dict[int, subprocess.Popen[bytes]] = {} + self._background_processes: set[subprocess.Popen[bytes]] = set() self._lock = threading.Lock() + self._closing = threading.Event() def run_task( self, @@ -40,14 +58,20 @@ class RunManager: input_data: dict[str, str | int | list[str]] | None = None, uploads: Mapping[str, PendingUpload] | None = None, ) -> tuple[Run, bool]: + if self._closing.is_set(): + raise RuntimeError("The application is shutting down; try again after it restarts.") sensitive_names = {field.name for field in task.inputs if field.sensitive} stored_input = self._redact_input(input_data or {}, sensitive_names) - run, created = self.store.enqueue( - target_kind="task", - target_path=task.id, - queue_group=task.group_id, - input_data=stored_input, - ) + with self._lock: + if self._closing.is_set(): + raise RuntimeError("The application is shutting down.") + blocker = self.task_status(task)["blocked_by"] + if blocker: + raise TaskConflictError(blocker) + run, created = self.store.enqueue( + target_kind="task", target_path=task.id, queue_group=task.group_id, + input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",), + ) if created: try: uploaded_inputs = self._save_uploads(run.id, uploads or {}) @@ -56,9 +80,11 @@ class RunManager: run = self.store.set_input(run.id, stored_input) if sensitive_names: self._write_execution_input(run.id, execution_input) + self._save_work_input(run, execution_input) except (OSError, ValueError) as error: self._delete_uploads(run.id) self._delete_execution_input(run.id) + self._discard_work_input(run) return self.store.finish( run.id, status="failed", @@ -68,15 +94,99 @@ class RunManager: return run, created def run_group(self, group: Group) -> tuple[Run, bool]: - run, created = self.store.enqueue( - target_kind="group", - target_path=group.id, - queue_group=group.id, - ) + if self._closing.is_set(): + raise RuntimeError("The application is shutting down.") + resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources)) + resources = resources or (f"group:{group.id}",) + with self._lock: + for active in self.store.active_runs(): + if active.target_kind == "group" and active.target_path == group.id: + return active, False + if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)): + raise TaskConflictError(active) + run, created = self.store.enqueue( + target_kind="group", target_path=group.id, queue_group=group.id, resources=resources, + ) if created: self._wake_worker(group.id) return run, created + def startup(self) -> None: + """Recover only runs with the durable input/context written by this runner.""" + self.store.mark_interrupted() + tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks} + for old in self.store.interrupted_without_recovery(): + work = self._work_root(old) + if not (work / "input.json").is_file(): + continue + task = tasks.get(old.target_path) + if task is None: + continue + if old.started_at and not task.recoverable: + continue + run, created = self.store.enqueue( + target_kind="task", target_path=task.id, queue_group=task.group_id, + input_data=json.loads(old.input_json), resources=task.resources, + recovery_of_run_id=old.id, + ) + if created: + self._wake_worker(run.queue_group) + for run in self.store.active_runs(): + if run.parent_run_id is None: + self._wake_worker(run.queue_group) + + def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict: + active = active_runs if active_runs is not None else self.store.active_runs() + own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None) + resources = set(task.resources or (f"group:{task.group_id}",)) + blocker = next((run for run in active if run is not own and run.parent_run_id is None + and resources.intersection(json.loads(run.resources_json))), None) + return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""} + + def progress(self, run: Run) -> str: + try: + data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text()) + except FileNotFoundError: + return "" + return str(data.get("text", "")) + + def shutdown(self, timeout: float | None = None) -> None: + self._closing.set() + for event in self._wake_events.values(): + event.set() + deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout) + with self._lock: + processes = [*self._processes.values(), *self._background_processes] + for process in processes: + if process.poll() is None: + try: + os.kill(process.pid, signal.SIGUSR1) + except ProcessLookupError: + pass + while any(process.poll() is None for process in processes) and time.monotonic() < deadline: + time.sleep(.05) + for process in processes: + kill_remaining_group(process) + for process in processes: + process.wait(timeout=5) + for worker in self._workers.values(): + worker.join(timeout=max(0, deadline - time.monotonic())) + + def _work_root(self, run: Run) -> Path: + while run.recovery_of_run_id is not None: + previous = self.store.get(run.recovery_of_run_id) + if previous is None: + raise RuntimeError("The original recovery run no longer exists") + run = previous + return self.settings.state_root / "runs" / f"{run.id}.work" + + def _save_work_input(self, run: Run, values: dict) -> None: + atomic_json(self._work_root(run) / "input.json", values) + + def _discard_work_input(self, run: Run) -> None: + for name in ("input.json", "input.json.tmp"): + (self._work_root(run) / name).unlink(missing_ok=True) + def request_cancel(self, run_id: int) -> Run | None: run = self.store.request_cancel(run_id) if run is None: @@ -90,6 +200,7 @@ class RunManager: if completed is not None and completed.status == "cancelled": self._delete_uploads(run.id) self._delete_execution_input(run.id) + self._discard_work_input(completed) return completed def artifacts(self, run_id: int) -> list[str]: @@ -165,9 +276,10 @@ class RunManager: cutoff = time.time() - artifact_max_age_seconds if runs_root.is_dir(): lifetimes = self.store.run_lifetimes() + protected = self._recovery_context_ids() for path in runs_root.iterdir(): run_id_text, separator, kind = path.name.partition(".") - if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink(): + if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink(): continue run_id = int(run_id_text) if run_id not in lifetimes: @@ -175,7 +287,7 @@ class RunManager: run = self.store.get(run_id) lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None) status, finished_at = lifetimes[run_id] - if status in {"queued", "running"}: + if status in {"queued", "running"} or run_id in protected: continue if kind in {"input.json", "execution-input.json"} and path.is_file(): path.unlink(missing_ok=True) @@ -193,8 +305,50 @@ class RunManager: continue if completed_at < cutoff: shutil.rmtree(path, ignore_errors=True) + elif kind == "work" and path.is_dir(): + (path / "input.json").unlink(missing_ok=True) + if path.stat().st_mtime < time.time() - 14 * 86400: + shutil.rmtree(path) + elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400: + path.unlink() self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds) + def _recovery_context_ids(self) -> set[int]: + protected = set() + tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks} + for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]: + task = tasks.get(run.target_path) + if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()): + continue + while True: + protected.add(run.id) + if run.recovery_of_run_id is None: + break + run = self.store.get(run.recovery_of_run_id) + if run is None: + break + return protected + + def _promote_review_uploads(self, run: Run, task: Task) -> Run: + if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs): + return run + values = json.loads(run.input_json) + if values.get(task.execution.field.name) != task.execution.dry_run_value: + return run + origin = self._work_root(run).name.split(".", 1)[0] + old_root = self._upload_root(int(origin)) + for field in task.inputs: + if field.type != "file" or not values.get(field.name): + continue + source = Path(values[field.name]) + if source.parent != old_root or source.is_symlink() or not source.is_file(): + raise RuntimeError("The original uploaded review input is no longer available") + destination = self._upload_root(run.id) / source.name + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + shutil.copy2(source, destination) + values[field.name] = str(destination) + return self.store.set_input(run.id, values) + @staticmethod def _keep_diagnostic_reports(root: Path) -> None: try: @@ -254,24 +408,31 @@ class RunManager: def run_background_task(self, task: BackgroundTask) -> None: """Run a declared internal task without creating high-volume run history.""" - + if self._closing.is_set(): + return log_path = self.settings.state_root / "background" / f"{task.id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) environment = self._environment(task.group_id) + environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work") try: with log_path.open("wb") as log_file: - process = subprocess.Popen( - [self.settings.python_executable, str(task.path)], - cwd=task.path.parent, - stdin=subprocess.DEVNULL, - stdout=log_file, - stderr=subprocess.STDOUT, - env=environment, - start_new_session=True, - ) - exit_code = process.wait() - if exit_code: - log_file.write(f"Background task exited with status {exit_code}.\n".encode()) + with self._lock: + if self._closing.is_set(): + return + process = subprocess.Popen( + [self.settings.python_executable, "-m", "app.worker", str(task.path)], + cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file, + stderr=subprocess.STDOUT, env=environment, start_new_session=True, + ) + self._background_processes.add(process) + try: + exit_code = process.wait() + if exit_code not in (0, PAUSED_EXIT_CODE): + log_file.write(f"Background task exited with status {exit_code}.\n".encode()) + finally: + kill_remaining_group(process) + with self._lock: + self._background_processes.discard(process) except OSError as error: log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8") @@ -291,13 +452,17 @@ class RunManager: event.set() def _worker_loop(self, group_id: str, event: threading.Event) -> None: - while True: + while not self._closing.is_set(): event.clear() run = self.store.claim_next(group_id) if run is None: event.wait() continue - self._execute_root_run(run) + try: + self._execute_root_run(run) + finally: + for waiting in self._wake_events.values(): + waiting.set() def _execute_root_run(self, run: Run) -> None: if run.target_kind == "task": @@ -355,6 +520,8 @@ class RunManager: *, parent_run_id: int | None = None, ) -> Run: + if self._closing.is_set(): + return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.") if self.store.is_cancel_requested(run.id) or ( parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) ): @@ -366,12 +533,20 @@ class RunManager: artifact_root = self._artifact_root(run.id) artifact_root.mkdir(parents=True, exist_ok=True) try: - if not input_path.exists(): + work = self._work_root(run) + saved_input = work / "input.json" + if run.recovery_of_run_id is not None: + input_path = saved_input + elif not input_path.exists(): input_path = self.settings.state_root / "runs" / f"{run.id}.input.json" input_path.write_text(run.input_json + "\n", encoding="utf-8") input_path.chmod(0o600) + if not saved_input.exists(): + self._save_work_input(run, json.loads(input_path.read_text())) except OSError as error: self._delete_uploads(run.id) + self._delete_execution_input(run.id) + self._discard_work_input(run) shutil.rmtree(artifact_root, ignore_errors=True) return self.store.finish( run.id, @@ -381,24 +556,33 @@ class RunManager: environment = self._environment(task.group_id) environment["SERVER_MAINTENANCE_INPUT"] = str(input_path) environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root) + environment["SERVER_MAINTENANCE_WORK"] = str(work) + environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json") + if run.recovery_of_run_id is not None: + old = self.store.get(run.recovery_of_run_id) + if old and old.started_at: + environment["SERVER_MAINTENANCE_RECOVER"] = "1" self.store.set_log_path(run.id, str(log_path)) try: with log_path.open("wb") as log_file: - process = subprocess.Popen( - [self.settings.python_executable, str(task.path)], - cwd=task.path.parent, - stdin=subprocess.DEVNULL, - stdout=log_file, - stderr=subprocess.STDOUT, - env=environment, - start_new_session=True, - ) with self._lock: + if self._closing.is_set(): + return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.") + process = subprocess.Popen( + [self.settings.python_executable, "-m", "app.worker", str(task.path)], + cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file, + stderr=subprocess.STDOUT, env=environment, start_new_session=True, + ) self._processes[run.id] = process + if self.store.is_cancel_requested(run.id) or ( + parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) + ): + self._terminate_active_process(run.id) exit_code = process.wait() except OSError as error: self._delete_uploads(run.id) + self._discard_work_input(run) shutil.rmtree(artifact_root, ignore_errors=True) return self.store.finish( run.id, @@ -408,7 +592,9 @@ class RunManager: ) finally: with self._lock: - self._processes.pop(run.id, None) + owned = self._processes.pop(run.id, None) + if owned is not None: + kill_remaining_group(owned) self._delete_execution_input(run.id) (self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True) @@ -416,6 +602,7 @@ class RunManager: parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) ): self._delete_uploads(run.id) + self._discard_work_input(run) self._keep_diagnostic_reports(artifact_root) return self.store.finish( run.id, @@ -423,21 +610,31 @@ class RunManager: exit_code=exit_code, log_path=str(log_path), ) - completed = self.store.finish( - run.id, - status="succeeded" if exit_code == 0 else "failed", - exit_code=exit_code, - log_path=str(log_path), - ) - if not self.review_upload_paths(completed, task): + if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0): + return self.store.finish( + run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path), + message="Application shutdown; saved state will be checked by recovery after restart.", + ) + status = "succeeded" if exit_code == 0 else "failed" + message = None + if status == "succeeded": + try: + run = self._promote_review_uploads(run, task) + except (RuntimeError, OSError) as error: + status, message = "failed", f"Could not preserve review input: {error}" + candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat()) + if not self.review_upload_paths(candidate, task): self._delete_uploads(run.id) - if completed.status != "succeeded": + if status != "succeeded": self._keep_diagnostic_reports(artifact_root) else: try: artifact_root.rmdir() except OSError: pass + completed = self.store.finish(run.id, status=status, exit_code=exit_code, + message=message, log_path=str(log_path)) + self._discard_work_input(run) self.cleanup_transient_files() return completed @@ -447,6 +644,7 @@ class RunManager: environment = { "PATH": "/usr/local/bin:/usr/bin:/bin", "PYTHONUNBUFFERED": "1", + "PYTHONPATH": str(Path(__file__).resolve().parent.parent), "SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root), "SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root), "SERVER_MAINTENANCE_STATE": str(task_state), diff --git a/python-tools/app/store.py b/python-tools/app/store.py index bb587b7..e557ee3 100644 --- a/python-tools/app/store.py +++ b/python-tools/app/store.py @@ -30,6 +30,8 @@ class Run: log_path: str | None message: str | None input_json: str + recovery_of_run_id: int | None = None + resources_json: str = "[]" def now() -> str: @@ -72,6 +74,10 @@ class RunStore: connection.execute( "ALTER TABLE runs ADD COLUMN input_json TEXT NOT NULL DEFAULT '{}'" ) + if "recovery_of_run_id" not in columns: + connection.execute("ALTER TABLE runs ADD COLUMN recovery_of_run_id INTEGER REFERENCES runs(id)") + if "resources_json" not in columns: + connection.execute("ALTER TABLE runs ADD COLUMN resources_json TEXT NOT NULL DEFAULT '[]'") connection.execute( """ CREATE INDEX IF NOT EXISTS runs_queue_index @@ -106,6 +112,8 @@ class RunStore: queue_group: str, trigger: str = "manual", input_data: Mapping[str, str | int | list[str]] | None = None, + resources: tuple[str, ...] = (), + recovery_of_run_id: int | None = None, ) -> tuple[Run, bool]: input_json = json.dumps(input_data or {}, sort_keys=True, separators=(",", ":")) with self._connect() as connection: @@ -126,10 +134,12 @@ class RunStore: cursor = connection.execute( """ INSERT INTO runs ( - target_kind, target_path, queue_group, trigger, status, created_at, input_json - ) VALUES (?, ?, ?, ?, 'queued', ?, ?) + target_kind, target_path, queue_group, trigger, status, created_at, input_json, + resources_json, recovery_of_run_id + ) VALUES (?, ?, ?, ?, 'queued', ?, ?, ?, ?) """, - (target_kind, target_path, queue_group, trigger, now(), input_json), + (target_kind, target_path, queue_group, trigger, now(), input_json, + json.dumps(resources), recovery_of_run_id), ) return self.get(cursor.lastrowid, connection=connection), True @@ -162,6 +172,10 @@ class RunStore: ).fetchone() if row is None: return None + requested = set(json.loads(row["resources_json"])) + for active in connection.execute("SELECT resources_json FROM runs WHERE status='running' AND parent_run_id IS NULL"): + if requested & set(json.loads(active["resources_json"])): + return None run_id = row["id"] connection.execute( "UPDATE runs SET status = 'running', started_at = ? WHERE id = ?", @@ -263,6 +277,26 @@ class RunStore: for row in connection.execute("SELECT id, status, finished_at FROM runs") } + def active_runs(self) -> list[Run]: + with self._connect() as connection: + return [self._run(row) for row in connection.execute( + "SELECT * FROM runs WHERE status IN ('queued','running') ORDER BY id" + )] + + def interrupted_without_recovery(self) -> list[Run]: + with self._connect() as connection: + return [self._run(row) for row in connection.execute( + """SELECT r.* FROM runs r WHERE r.status='interrupted' AND r.target_kind='task' + AND r.cancel_requested_at IS NULL + AND NOT EXISTS (SELECT 1 FROM runs child WHERE child.recovery_of_run_id=r.id) + ORDER BY r.id""" + )] + + def recovery_for(self, run_id: int) -> Run | None: + with self._connect() as connection: + row = connection.execute("SELECT * FROM runs WHERE recovery_of_run_id=? ORDER BY id DESC LIMIT 1", (run_id,)).fetchone() + return self._run(row) if row else None + def set_log_path(self, run_id: int, path: str) -> None: with self._connect() as connection: connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id)) diff --git a/python-tools/app/task_runtime.py b/python-tools/app/task_runtime.py new file mode 100644 index 0000000..84c408f --- /dev/null +++ b/python-tools/app/task_runtime.py @@ -0,0 +1,161 @@ +"""Cooperative task checkpoints and optional plain-text progress.""" + +from __future__ import annotations + +import json +import os +import signal +import time +from contextlib import contextmanager +from pathlib import Path + + +PAUSED_EXIT_CODE = 75 +CHECKPOINT_SECONDS = 60 +_pause_requested = False +_defer = 0 +_state: dict = {} +_last_save = 0.0 +_last_progress = 0.0 +_previous_clean = False + + +class TaskPaused(BaseException): + """Unwind at a safe boundary without treating a pause as an ordinary failure.""" + + +def work_directory() -> Path | None: + value = os.environ.get("SERVER_MAINTENANCE_WORK") + return Path(value) if value else None + + +def atomic_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as stream: + temporary.chmod(0o600) + json.dump(value, stream, ensure_ascii=False) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def initialize() -> None: + global _state, _previous_clean, _pause_requested, _last_save + _pause_requested = False + root = work_directory() + if root is not None: + try: + saved = json.loads((root / "checkpoint.json").read_text()) + except FileNotFoundError: + saved = {} + _state = saved.get("data", {}) + _previous_clean = bool(saved.get("clean_shutdown")) + _write_checkpoint(False) + _last_save = time.monotonic() + signal.signal(signal.SIGUSR1, request_pause) + + +def request_pause(_signal=None, _frame=None) -> None: + global _pause_requested + _pause_requested = True + + +def pause_requested() -> bool: + return _pause_requested and work_directory() is not None + + +def checkpoint_data() -> dict: + return _state if work_directory() is not None else {} + + +def was_clean_shutdown() -> bool: + return _previous_clean + + +def _write_checkpoint(clean: bool) -> None: + root = work_directory() + if root is not None: + atomic_json(root / "checkpoint.json", { + "version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(), + }) + + +def checkpoint(data: dict | None = None, *, force: bool = False) -> None: + """Keep the latest state in memory; persist ordinary repeatable work once a minute.""" + global _state, _last_save + if data is not None: + _state = data + if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS: + _write_checkpoint(False) + _last_save = time.monotonic() + check_stop() + + +def check_stop() -> None: + if _pause_requested and work_directory() is not None and not _defer: + _write_checkpoint(True) + raise TaskPaused() + + +@contextmanager +def critical_section(): + """Finish a small consistency-sensitive operation before observing a pause.""" + global _defer + check_stop() + _defer += 1 + try: + yield + finally: + _defer -= 1 + check_stop() + + +def sleep(seconds: float) -> None: + deadline = time.monotonic() + seconds + while (remaining := deadline - time.monotonic()) > 0: + check_stop() + time.sleep(min(remaining, 0.25)) + check_stop() + + +def progress(text: str, *, force: bool = False) -> None: + global _last_progress + root = work_directory() + if root is not None and (force or time.monotonic() - _last_progress >= 1): + path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json"))) + atomic_json(path, {"text": text, "updated_at": time.time()}) + _last_progress = time.monotonic() + + +class Progress: + """Item-count estimates, independently usable for several named steps.""" + + def __init__(self, totals: dict[str, int]): + self.totals = totals + self.done = dict.fromkeys(totals, 0) + self.started: dict[str, float] = {} + self.baseline: dict[str, int] = {} + + def update(self, label: str, done: int, *, force: bool = False) -> None: + self.started.setdefault(label, time.monotonic()) + self.baseline.setdefault(label, done) + self.done[label] = done + lines = [] + for name, total in self.totals.items(): + count = self.done[name] + percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0) + line = f"{name}: {percent}%" + processed = count - self.baseline.get(name, 0) + if processed > 0 and count < total and name in self.started: + seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count))) + estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes" + line += f", ETA {estimate}" + lines.append(line) + progress("\n".join(lines), force=force) diff --git a/python-tools/app/web.py b/python-tools/app/web.py index 332af58..17b0811 100644 --- a/python-tools/app/web.py +++ b/python-tools/app/web.py @@ -14,7 +14,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u from .config import Settings from .discovery import TaskCatalog from .inputs import PendingUpload, validate_inputs, validate_uploads -from .runner import RunManager +from .runner import RunManager, TaskConflictError from .scheduler import BackgroundScheduler from .store import RunStore @@ -24,11 +24,8 @@ def create_app(settings: Settings | None = None) -> Flask: catalog = TaskCatalog(settings.task_root) store = RunStore(settings.database_path) store.initialize() - store.mark_interrupted() manager = RunManager(settings, catalog, store) - manager.cleanup_transient_files() scheduler = BackgroundScheduler(catalog, manager) - scheduler.start() project_root = Path(__file__).resolve().parent.parent app = Flask( @@ -62,6 +59,66 @@ def create_app(settings: Settings | None = None) -> Flask: return "—" return datetime.fromisoformat(value).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z") + def run_links(run): + if run is None: + return None + display_status = run.status + if run.recovery_of_run_id is not None and run.status in {"queued", "running"}: + display_status = "Recovery queued" if run.status == "queued" else "recovering" + return { + "run_id": run.id, + "status": run.status, + "display_status": display_status, + "recovery_of_run_id": run.recovery_of_run_id, + "artifacts_url": url_for("run_artifacts", run_id=run.id), + "detail_url": url_for("run_detail", run_id=run.id), + "output_url": url_for("run_output", run_id=run.id), + "state_url": url_for("run_state", run_id=run.id), + } + + def task_state(task, active_runs=None): + state = manager.task_status(task, active_runs=active_runs) + return { + "active_run": run_links(state["active_run"]), + "blocked_by": run_links(state["blocked_by"]), + "progress": state["progress"], + } + + def live_states(groups, latest): + active = store.active_runs() + tasks = {task.id: task_state(task, active) for group in groups for task in group.tasks} + group_states = {} + for group in groups: + run = latest[("group", group.id)] + active = run if run and run.status in {"queued", "running"} else None + blocker = next(( + tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"] + for task in group.tasks + if tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"] + ), None) + group_states[group.id] = { + "active_run": run_links(active), + "blocked_by": None if active else blocker, + "progress": manager.progress(active) if active else "", + } + return {"tasks": tasks, "groups": group_states} + + def unavailable(error): + if isinstance(error, TaskConflictError): + return task_conflict(error) + message = str(error) + if request.accept_mimetypes.best == "application/json": + return {"error": message}, 503 + return render_template("unavailable.html", message=message, blocking_run=None), 503 + + @app.errorhandler(TaskConflictError) + def task_conflict(error): + blocking_run = run_links(error.run) + message = f"Task blocked by run #{error.run.id} ({blocking_run['display_status']})." + if request.accept_mimetypes.best == "application/json": + return {"error": message, "blocked_by": blocking_run}, 409 + return render_template("unavailable.html", message=message, blocking_run=blocking_run), 409 + def render_index( *, form_values: dict[str, dict[str, list[str]]] | None = None, @@ -80,6 +137,7 @@ def create_app(settings: Settings | None = None) -> Flask: "index.html", groups=groups, latest=latest, + live_states=live_states(groups, latest), form_values=form_values or {}, input_errors=input_errors or {}, ), @@ -90,6 +148,14 @@ def create_app(settings: Settings | None = None) -> Flask: def index(): return render_index() + @app.get("/tasks/state") + def tasks_state(): + groups = catalog.groups() + latest = {("group", group.id): store.last_run("group", group.id) for group in groups} + response = app.json.response(live_states(groups, latest)) + response.headers["Cache-Control"] = "no-store" + return response + @app.post("/tasks//run") def run_task(task_id: str): task = catalog.task(task_id) @@ -114,15 +180,12 @@ def create_app(settings: Settings | None = None) -> Flask: input_errors={task.id: errors}, status=400, ) - run, _ = manager.run_task(task, input_data, uploads) + try: + run, _ = manager.run_task(task, input_data, uploads) + except RuntimeError as error: + return unavailable(error) if request.accept_mimetypes.best == "application/json": - return { - "artifacts_url": url_for("run_artifacts", run_id=run.id), - "detail_url": url_for("run_detail", run_id=run.id), - "output_url": url_for("run_output", run_id=run.id), - "run_id": run.id, - "state_url": url_for("run_state", run_id=run.id), - }, 202 + return run_links(run), 202 return redirect(url_for("run_detail", run_id=run.id)) @app.post("/groups//run") @@ -130,7 +193,12 @@ def create_app(settings: Settings | None = None) -> Flask: group = catalog.group(group_id) if group is None or not group.can_run_as_group: abort(404) - run, _ = manager.run_group(group) + try: + run, _ = manager.run_group(group) + except RuntimeError as error: + return unavailable(error) + if request.accept_mimetypes.best == "application/json": + return run_links(run), 202 return redirect(url_for("run_detail", run_id=run.id)) @app.get("/runs/") @@ -147,6 +215,10 @@ def create_app(settings: Settings | None = None) -> Flask: output=output, output_offset=offset, execution_task=execution_task, + recovery_run=run_links(store.recovery_for(run.id)), + progress=manager.progress(run), + display_status=run_links(run)["display_status"], + live_states={"tasks": {execution_task.id: task_state(execution_task)} if execution_task else {}, "groups": {}}, ) def execution_replay_task(run): @@ -190,9 +262,14 @@ def create_app(settings: Settings | None = None) -> Flask: except OSError: abort(409, description="The reviewed upload is no longer available. Upload the file and review again.") uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream) - executed, created = manager.run_task(task, validated, uploads) + try: + executed, created = manager.run_task(task, validated, uploads) + except RuntimeError as error: + return unavailable(error) if created and executed.status == "queued": manager.discard_review_uploads(run.id) + if request.accept_mimetypes.best == "application/json": + return run_links(executed), 202 return redirect(url_for("run_detail", run_id=executed.id)) @app.post("/runs//cancel") @@ -216,6 +293,9 @@ def create_app(settings: Settings | None = None) -> Flask: return { "output": output, "offset": offset, "reset": reset, "has_more": has_more, "status": run.status, + "display_status": run_links(run)["display_status"], + "recovery_of_run_id": run.recovery_of_run_id, + "progress": manager.progress(run), } def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]: @@ -299,8 +379,10 @@ def create_app(settings: Settings | None = None) -> Flask: if run is None: abort(404) return { - "status": run.status, + **run_links(run), + "progress": manager.progress(run), "finished_at": run.finished_at, + "recovery_run": run_links(store.recovery_for(run.id)), "exit_code": run.exit_code, "cancel_requested_at": run.cancel_requested_at, } @@ -316,4 +398,12 @@ def create_app(settings: Settings | None = None) -> Flask: runs=store.history(target_kind, target_path), ) + try: + manager.startup() + manager.cleanup_transient_files() + scheduler.start() + except BaseException: + scheduler.stop() + manager.shutdown(timeout=5) + raise return app diff --git a/python-tools/app/worker.py b/python-tools/app/worker.py new file mode 100644 index 0000000..f117b86 --- /dev/null +++ b/python-tools/app/worker.py @@ -0,0 +1,65 @@ +"""Execute task code in isolation, optionally entering its embedded recovery hook.""" + +import os +import runpy +import sys +from pathlib import Path +from urllib.error import HTTPError, URLError + +from . import task_runtime as runtime + + +def service_unavailable(error: Exception) -> bool: + """Retry connection failures, not authentication, invalid data or task bugs.""" + while error is not None: + if isinstance(error, HTTPError): + return error.code in {408, 502, 503, 504} + if isinstance(error, (ConnectionError, TimeoutError)): + return True + if isinstance(error, URLError): + return isinstance(error.reason, (OSError, TimeoutError)) + sqlstate = getattr(error, "sqlstate", None) + if sqlstate and (sqlstate.startswith("08") or sqlstate in {"57P01", "57P03"}): + return True + if type(error).__module__.startswith("psycopg") and type(error).__name__ == "OperationalError": + text = str(error).casefold() + return any(reason in text for reason in ("connection refused", "timeout expired", "could not translate host name", "server closed the connection")) + error = error.__cause__ + return False + + +def main() -> None: + path = Path(sys.argv[1]).resolve() + sys.path.insert(0, str(path.parent)) + sys.argv = [str(path)] + runtime.initialize() + try: + if os.environ.get("SERVER_MAINTENANCE_RECOVER") == "1": + namespace = runpy.run_path(str(path), run_name="__task_recovery__") + recover = namespace.get("recover") + if not callable(recover): + raise RuntimeError("This task no longer provides its recovery function") + print("Recovering interrupted task; verifying current state.", flush=True) + attempts = 0 + while True: + try: + recover() + break + except Exception as error: + if not service_unavailable(error): + raise + attempts += 1 + delay = min(30, 2 ** min(attempts, 5)) + message = f"Recovery waiting for an unavailable service; retry in {delay} seconds." + runtime.progress(message, force=True) + print(message, flush=True) + runtime.sleep(delay) + else: + runpy.run_path(str(path), run_name="__main__") + except runtime.TaskPaused: + print("Paused at a safe checkpoint for recovery after restart.", flush=True) + raise SystemExit(runtime.PAUSED_EXIT_CODE) + + +if __name__ == "__main__": + main() diff --git a/python-tools/compose.example.yaml b/python-tools/compose.example.yaml index ca67f2f..f2fce2e 100644 --- a/python-tools/compose.example.yaml +++ b/python-tools/compose.example.yaml @@ -14,3 +14,5 @@ services: - ${PYTHON_TOOLS_VOLUME_ROOT:?Set the persistent volume root}/credentials:/opt/credentials:ro # Add task-specific bind mounts here when required. restart: always + stop_grace_period: 120s + init: true diff --git a/python-tools/static/app.js b/python-tools/static/app.js index 79e55bd..4667e28 100644 --- a/python-tools/static/app.js +++ b/python-tools/static/app.js @@ -3,7 +3,7 @@ const output = document.querySelector("#run-output"); if (output) { let offset = Number(output.dataset.outputOffset || 0); let detailsFinalized = !["queued", "running"].includes( - document.querySelector("#run-status")?.textContent.trim(), + document.querySelector("#run-status")?.dataset.runStatus, ); const refresh = async () => { let delay = 2000; @@ -24,7 +24,12 @@ if (output) { offset = state.offset; const status = document.querySelector("#run-status"); if (status) { - status.textContent = state.status; + status.textContent = state.display_status; + status.dataset.runStatus = state.status; + const progress = document.querySelector("#run-progress"); + if (progress && progress.textContent !== (state.progress || "—")) { + progress.textContent = state.progress || "—"; + } if (state.has_more) { delay = 0; } else if (state.status !== "queued" && state.status !== "running") { @@ -42,6 +47,7 @@ if (output) { document.querySelectorAll(actions).forEach((form) => form.remove()); status.closest("dl").replaceWith(nextDetails); nextDetails.after(...page.querySelectorAll(actions)); + initializeTaskForms(page); detailsFinalized = true; } const artifacts = document.querySelector("#run-artifacts"); @@ -115,98 +121,135 @@ for (const choice of document.querySelectorAll("button[data-execution-choice]")) }); } -for (const form of document.querySelectorAll("form[data-wait-for-result]")) { - const result = form.querySelector(".task-result"); - const controls = [...form.querySelectorAll("input, select, button")]; - let waiting = false; +const activeStatus = (status) => status === "queued" || status === "running"; +const taskControllers = new Map(); +let tasksStateUrl; +let taskStateVersion = 0; +let refreshingTasks = false; +let taskRefreshTimer; - const warnBeforeLeaving = (event) => { - event.preventDefault(); - event.returnValue = ""; - }; - - const setWaiting = (value) => { - waiting = value; - form.classList.toggle("is-waiting", value); - controls.forEach((control) => { - control.disabled = value; - }); - form.querySelectorAll("details.split-menu").forEach((menu) => { - menu.open = false; - }); - if (value) { - window.addEventListener("beforeunload", warnBeforeLeaving); - } else { - window.removeEventListener("beforeunload", warnBeforeLeaving); - } - }; - - const showResult = (message, kind, detailUrl) => { - result.replaceChildren(); - result.className = `task-result ${kind}`; - result.append(message); - if (detailUrl) { - const link = document.createElement("a"); - link.href = detailUrl; - link.textContent = "View details"; - result.append(" ", link); - } - }; - - const downloadArtifacts = async (run) => { - const response = await fetch(run.artifacts_url, { cache: "no-store" }); - if (!response.ok) { - return; - } - const artifacts = (await response.json()).artifacts; - const artifact = artifacts.find((item) => !item.view_url); - if (!artifact) { - return; - } +const downloadArtifacts = async (run) => { + const response = await fetch(run.artifacts_url, { cache: "no-store" }); + if (!response.ok) { + throw new Error("Could not read task downloads. Open the run details to download them."); + } + const artifact = (await response.json()).artifacts.find((item) => !item.view_url); + if (artifact) { const link = document.createElement("a"); link.href = artifact.url; link.download = ""; document.body.append(link); link.click(); link.remove(); + } +}; + +function taskController(form, initialState) { + const result = form.querySelector(".task-result"); + const message = result.querySelector(".task-message"); + const progress = result.querySelector(".task-progress"); + let state = initialState; + let observedRun = state.active_run; + let submitting = false; + let lastResult; + let previousMessage; + + const render = (text, kind = "", run = null, progressText = "", suffix = "", label = "View details") => { + const signature = JSON.stringify([text, kind, run?.detail_url, suffix, label]); + if (signature !== previousMessage) { + message.replaceChildren(text); + if (run) { + const link = document.createElement("a"); + link.href = run.detail_url; + link.textContent = label; + message.append(" ", link, suffix); + } + previousMessage = signature; + } + if (progress.textContent !== progressText) { + progress.textContent = progressText; + } + result.className = `task-result ${kind}`; + result.hidden = !text; }; - const waitForRun = async (run) => { - const started = Date.now(); - while (true) { + const showState = () => { + const active = state.active_run; + const blocker = state.blocked_by; + const disabled = submitting || Boolean(active || blocker); + form.querySelector(".task-controls").disabled = disabled; + form.classList.toggle("is-waiting", submitting || Boolean(active)); + form.setAttribute("aria-busy", String(submitting || Boolean(active))); + form.querySelectorAll("details.split-menu").forEach((menu) => { + menu.inert = disabled; + if (disabled) { + menu.open = false; + } + }); + if (submitting) { + render("Starting...", "running"); + } else if (active) { + const status = active.display_status; + render(`${status[0].toUpperCase()}${status.slice(1)}.`, "running", active, + state.progress, "", `Run #${active.run_id}`); + } else if (blocker) { + render("Blocked by", "blocked", blocker, "", ` (${blocker.display_status}).`, `run #${blocker.run_id}`); + } else if (lastResult) { + render(lastResult.text, lastResult.kind, lastResult.run); + } else { + render(""); + } + }; + + const apply = async (nextState, version) => { + if (submitting || version !== taskStateVersion) { + return; + } + state = nextState; + if (state.active_run) { + observedRun = state.active_run; + } else if (observedRun) { + const run = observedRun; const response = await fetch(run.state_url, { cache: "no-store" }); if (!response.ok) { throw new Error("Could not read task status."); } - const state = await response.json(); - if (state.status !== "queued" && state.status !== "running") { - if (state.status === "succeeded") { - showResult("Completed successfully.", "succeeded", run.detail_url); - if (form.dataset.downloadArtifacts !== undefined) { - await downloadArtifacts(run); - } - } else { - showResult(`Finished with status: ${state.status}.`, "failed", run.detail_url); - } + const completed = await response.json(); + if (submitting || version !== taskStateVersion) { return; } - await new Promise((resolve) => window.setTimeout(resolve, Date.now() - started < 10000 ? 500 : 2000)); + if (activeStatus(completed.status)) { + state = { ...state, active_run: completed, progress: completed.progress }; + } else { + observedRun = null; + lastResult = { + text: completed.status === "succeeded" ? "Completed successfully." : `Finished with status: ${completed.status}.`, + kind: completed.status === "succeeded" ? "succeeded" : "failed", + run, + }; + if (completed.status === "succeeded" && form.dataset.downloadArtifacts !== undefined) { + try { + await downloadArtifacts(run); + } catch (error) { + lastResult = { text: error.message, kind: "failed", run }; + } + } + } } + showState(); }; form.addEventListener("submit", async (event) => { - if (waiting) { - event.preventDefault(); - return; - } - if (!form.reportValidity()) { - return; - } event.preventDefault(); + if (submitting || state.active_run || state.blocked_by || !form.reportValidity()) { + return; + } // Disabled fields are omitted from FormData, so collect values first. const formData = new FormData(form); - setWaiting(true); - showResult("Running...", "running"); + submitting = true; + lastResult = null; + taskStateVersion += 1; + showState(); try { const response = await fetch(form.action, { method: "POST", @@ -214,15 +257,94 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) { headers: { Accept: "application/json" }, }); const data = await response.json(); - if (!response.ok) { - const messages = data.errors ? Object.values(data.errors).join(" ") : "Could not start task."; - throw new Error(messages); + taskStateVersion += 1; + if (response.status === 409 && data.blocked_by) { + state = { active_run: null, blocked_by: data.blocked_by, progress: "" }; + } else if (!response.ok) { + throw new Error(data.errors ? Object.values(data.errors).join(" ") : data.error || "Could not start task."); + } else { + observedRun = data; + state = { active_run: data, blocked_by: null, progress: "" }; + if (form.dataset.waitForResult === undefined) { + window.location.assign(data.detail_url); + } } - await waitForRun(data); } catch (error) { - showResult(error instanceof Error ? error.message : "Could not run task.", "failed"); + lastResult = { text: error instanceof Error ? error.message : "Could not run task.", kind: "failed" }; } finally { - setWaiting(false); + submitting = false; + showState(); + refreshTaskStates(); } }); + showState(); + return { apply }; } + +async function refreshTaskStates() { + if (refreshingTasks || !tasksStateUrl) { + return; + } + refreshingTasks = true; + window.clearTimeout(taskRefreshTimer); + const version = taskStateVersion; + const errorMessage = document.querySelector("#task-status-error"); + try { + const response = await fetch(tasksStateUrl, { cache: "no-store" }); + if (!response.ok) { + throw new Error("Could not read live task status."); + } + const states = await response.json(); + for (const [form, controller] of taskControllers) { + if (!form.isConnected) { + taskControllers.delete(form); + continue; + } + const state = form.dataset.taskId ? states.tasks[form.dataset.taskId] : states.groups[form.dataset.groupId]; + if (state) { + await controller.apply(state, version); + } + } + if (errorMessage) { + errorMessage.hidden = true; + } + } catch (error) { + console.warn("Could not refresh task status:", error); + if (errorMessage) { + errorMessage.textContent = "Live status unavailable. Retrying..."; + errorMessage.hidden = false; + } + } finally { + refreshingTasks = false; + if (taskControllers.size) { + taskRefreshTimer = window.setTimeout(refreshTaskStates, 1000); + } + } +} + +function initializeTaskForms(source = document) { + const data = source.querySelector("#task-states"); + if (!data) { + return; + } + tasksStateUrl = data.dataset.stateUrl; + const states = JSON.parse(data.textContent); + for (const form of document.querySelectorAll("form[data-task-id], form[data-group-id]")) { + if (!taskControllers.has(form)) { + const state = form.dataset.taskId ? states.tasks[form.dataset.taskId] : states.groups[form.dataset.groupId]; + taskControllers.set(form, taskController(form, state)); + } + } + if (taskControllers.size) { + if (!document.querySelector("#task-status-error")) { + const errorMessage = document.createElement("p"); + errorMessage.id = "task-status-error"; + errorMessage.setAttribute("role", "status"); + errorMessage.hidden = true; + document.querySelector("main").prepend(errorMessage); + } + refreshTaskStates(); + } +} + +initializeTaskForms(); diff --git a/python-tools/static/style.css b/python-tools/static/style.css index 6b619da..6c6d5a6 100644 --- a/python-tools/static/style.css +++ b/python-tools/static/style.css @@ -34,6 +34,7 @@ a { .group-heading { align-items: center; display: flex; + flex-wrap: wrap; gap: 1rem; justify-content: space-between; } @@ -69,6 +70,21 @@ a { min-width: 0; } +.task-controls { + border: 0; + margin: 0; + min-width: 0; + padding: 0; +} + +.task-form > .task-controls { + align-items: end; + display: grid; + gap: .75rem; + grid-column: 1 / -1; + grid-template-columns: minmax(0, 1fr) auto; +} + .task-options { display: grid; gap: .75rem; @@ -204,23 +220,24 @@ button { } button:disabled { - cursor: wait; + cursor: not-allowed; } .spinner { border: .15em solid currentColor; border-right-color: transparent; border-radius: 50%; - display: none; + display: inline-block; height: .75em; margin-left: .35em; vertical-align: -.05em; + visibility: hidden; width: .75em; } .is-waiting .spinner { animation: spin .7s linear infinite; - display: inline-block; + visibility: visible; } .task-result { @@ -228,6 +245,26 @@ button:disabled { margin: .5rem 0 0; } +.task-progress, +.run-progress { + overflow-wrap: anywhere; + white-space: pre-wrap; +} + +.task-progress { + display: block; +} + +.task-result, +.group-heading > form { + min-width: 0; + overflow-wrap: anywhere; +} + +.task-result[hidden] { + display: none; +} + .task-result.succeeded { color: #167c3a; } @@ -247,7 +284,8 @@ button:disabled { } @media (max-width: 42rem) { - .task-form { + .task-form, + .task-form > .task-controls { grid-template-columns: 1fr; } diff --git a/python-tools/templates/history.html b/python-tools/templates/history.html index 5704c88..16f3f1f 100644 --- a/python-tools/templates/history.html +++ b/python-tools/templates/history.html @@ -8,6 +8,7 @@
  • Run #{{ run.id }} · {{ run.status }} + {% if run.recovery_of_run_id is not none %}· recovery of #{{ run.recovery_of_run_id }}{% endif %} · {{ run.created_at|timestamp }} {% if run.exit_code is not none %}· exit {{ run.exit_code }}{% endif %}
  • diff --git a/python-tools/templates/index.html b/python-tools/templates/index.html index 8c625b9..30ec603 100644 --- a/python-tools/templates/index.html +++ b/python-tools/templates/index.html @@ -1,6 +1,8 @@ {% extends "base.html" %} +{% from "task_status.html" import task_status, live_state_data %} {% block content %}

    Tasks

    + {{ live_state_data(live_states) }} {% if not groups %}

    No task groups found.

    {% endif %} @@ -10,8 +12,12 @@

    {{ group.display_name }}

    {% if group.can_run_as_group %} -
    - + {% set group_state = live_states.groups[group.id] %} + +
    + +
    + {{ task_status(group_state) }}
    {% endif %}
    @@ -24,6 +30,8 @@
      {% for task in group.tasks %} {% set task_run = latest[("task", task.id)] %} + {% set state = live_states.tasks[task.id] %} + {% set busy = state.active_run or state.blocked_by %}
    • {{ task.display_name }}

      @@ -37,9 +45,10 @@

      {% endif %}
      -
      + {% set values = form_values.get(task.id, {}) %} {% set errors = input_errors.get(task.id, {}) %} +
      {% for field in task.inputs %} {% if not task.execution or field.name != task.execution.field.name %} @@ -101,9 +110,9 @@ -
      +
      ▾
      {% for option in execution.options %} @@ -115,11 +124,12 @@ {% else %} {% endif %}
      - {% if task.wait_for_result %}

      {% endif %} +
      + {{ task_status(state) }}
    • {% else %} diff --git a/python-tools/templates/run.html b/python-tools/templates/run.html index b3e0dc4..a73e12a 100644 --- a/python-tools/templates/run.html +++ b/python-tools/templates/run.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% from "task_status.html" import task_status, live_state_data %} {% block title %}Run {{ run.id }} · Server Maintenance{% endblock %} {% block content %}

      @@ -8,7 +9,10 @@

      {{ run.target_kind|title }} run #{{ run.id }}

      Target
      {{ run.target_path }}
      -
      Status
      {{ run.status }}
      +
      Status
      {{ display_status }}
      +
      Progress
      {{ progress or "—" }}
      + {% if run.recovery_of_run_id is not none %}
      Recovery of
      Run #{{ run.recovery_of_run_id }}
      {% endif %} + {% if recovery_run %}
      Recovery run
      Run #{{ recovery_run.run_id }} · {{ recovery_run.display_status }}
      {% endif %}
      Started
      {{ run.started_at|timestamp }}
      Finished
      {{ run.finished_at|timestamp }}
      Exit code
      {{ run.exit_code if run.exit_code is not none else "—" }}
      @@ -20,8 +24,13 @@ {% endif %} {% if execution_task %} -
      - + {{ live_state_data(live_states) }} + {% set state = live_states.tasks[execution_task.id] %} + +
      + +
      + {{ task_status(state) }}
      {% endif %} {% if children %} diff --git a/python-tools/templates/task_status.html b/python-tools/templates/task_status.html new file mode 100644 index 0000000..3c12ee9 --- /dev/null +++ b/python-tools/templates/task_status.html @@ -0,0 +1,13 @@ +{% macro task_status(state) %} + {% set active = state.active_run %} + {% set blocker = state.blocked_by %} +

      + {% if active %}{{ active.display_status|capitalize }}. Run #{{ active.run_id }}{% elif blocker %}Blocked by run #{{ blocker.run_id }} ({{ blocker.display_status }}).{% endif %} + {{ state.progress if active else '' }} +

      +{% endmacro %} + +{% macro live_state_data(states) %} + + +{% endmacro %} diff --git a/python-tools/templates/unavailable.html b/python-tools/templates/unavailable.html new file mode 100644 index 0000000..fe5dd41 --- /dev/null +++ b/python-tools/templates/unavailable.html @@ -0,0 +1,10 @@ +{% extends "base.html" %} +{% block title %}Task unavailable · Server Maintenance{% endblock %} +{% block content %} +

      Tasks

      +

      Task unavailable

      +

      {{ message }}

      + {% if blocking_run %} +

      View run #{{ blocking_run.run_id }}

      + {% endif %} +{% endblock %} diff --git a/python-tools/tests/test_discovery.py b/python-tools/tests/test_discovery.py index bfeae3a..5b02ef1 100644 --- a/python-tools/tests/test_discovery.py +++ b/python-tools/tests/test_discovery.py @@ -9,6 +9,27 @@ from app.discovery import Group, TaskCatalog from .conftest import write_task +def test_discovers_embedded_recovery_and_literal_resources_without_import(tmp_path): + path=write_task(tmp_path,'tasks','change.py', ''' +"""A recoverable task.""" +RESOURCES = ('shared-files', 'shared-files') +def recover(): + raise RuntimeError('Must not run during discovery') +raise RuntimeError('Must not import during discovery') +''') + task=TaskCatalog(tmp_path).task('tasks/change.py') + assert task.recoverable is True + assert task.resources==('group:tasks','shared-files') + path.write_text('class Example:\n def recover(self): pass\n') + assert TaskCatalog(tmp_path).task('tasks/change.py').recoverable is False + + +def test_resources_do_not_execute_expressions(tmp_path): + write_task(tmp_path,'tasks','change.py',"RESOURCES = tuple(['shared-files'])\n") + with pytest.raises(ValueError): + TaskCatalog(tmp_path).groups() + + def test_discovers_only_direct_non_hidden_python_tasks(tmp_path: Path) -> None: root = tmp_path / "tasks" write_task(root, "media_tasks", "check_one.py", '"""Checks one thing."""\n') diff --git a/python-tools/tests/test_incremental_output.py b/python-tools/tests/test_incremental_output.py index 6c45322..e599aa2 100644 --- a/python-tools/tests/test_incremental_output.py +++ b/python-tools/tests/test_incremental_output.py @@ -21,7 +21,11 @@ def output_run(settings): def test_log_polling_returns_only_appended_bytes_and_status(output_run): client, run_id, path, store = output_run first = client.get(f"/runs/{run_id}/output?offset=0").get_json() - assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"} + assert first == { + "output": "first\n", "offset": 6, "reset": False, "has_more": False, + "status": "running", "display_status": "running", "progress": "", + "recovery_of_run_id": None, + } with path.open("ab") as stream: stream.write("next: \u20ac\n".encode()) store.finish(run_id, status="succeeded") diff --git a/python-tools/tests/test_live_status.py b/python-tools/tests/test_live_status.py new file mode 100644 index 0000000..ee12ae3 --- /dev/null +++ b/python-tools/tests/test_live_status.py @@ -0,0 +1,142 @@ +"""Web contracts for restoring live task state and rejecting resource conflicts.""" + +import json + +import pytest + +from app.runner import RunManager +from app.scheduler import BackgroundScheduler +from app.web import create_app + +from .conftest import write_task + + +@pytest.fixture +def live_app(settings, monkeypatch): + for group, filename, source in ( + ("example", "long.py", 'RESOURCES = ("shared-data",)\n'), + ("example", "sibling.py", ""), + ("related", "change.py", 'RESOURCES = ("shared-data",)\n'), + ("independent", "free.py", ""), + ): + write_task(settings.task_root, group, filename, source + 'print("done")\n') + monkeypatch.setattr(RunManager, "_wake_worker", lambda *_: None) + app = create_app(settings) + yield app + app.extensions["background_scheduler"].stop() + + +def start_live_run(app, *, status="running", recovery=False): + manager = app.extensions["run_manager"] + task = app.extensions["catalog"].task("example/long.py") + run, _ = manager.run_task(task) + if recovery: + old = manager.store.finish(run.id, status="interrupted") + run, _ = manager.store.enqueue( + target_kind="task", target_path=task.id, queue_group=task.group_id, + resources=task.resources, recovery_of_run_id=old.id, + ) + if status == "running": + run = manager.store.claim_next(task.group_id) + progress = manager.settings.state_root / "runs" / f"{run.id}.progress.json" + progress.parent.mkdir(parents=True, exist_ok=True) + progress.write_text(json.dumps({"text": "2 of 5\nETA: 3 minutes"})) + return run + + +@pytest.mark.parametrize("status", ["queued", "running"]) +def test_task_states_restore_active_conflicts_and_progress(live_app, status): + run = start_live_run(live_app, status=status) + client = live_app.test_client() + response = client.get("/tasks/state") + assert response.status_code == 200 + assert response.headers["Cache-Control"] == "no-store" + states = response.get_json() + own = states["tasks"]["example/long.py"] + assert own["active_run"]["run_id"] == run.id + assert own["active_run"]["display_status"] == status + assert own["progress"] == "2 of 5\nETA: 3 minutes" + assert own["blocked_by"] is None + for name in ("example/sibling.py", "related/change.py"): + state = states["tasks"][name] + assert state["active_run"] is None + assert state["blocked_by"]["detail_url"] == f"/runs/{run.id}" + assert states["tasks"]["independent/free.py"] == { + "active_run": None, "blocked_by": None, "progress": "", + } + assert states["groups"]["related"]["blocked_by"]["run_id"] == run.id + assert not {"input_json", "resources_json", "log_path"}.intersection(own["active_run"]) + page = client.get("/") + assert b'fieldset class="task-controls" disabled' in page.data + assert b"2 of 5\nETA: 3 minutes" in page.data + assert f'Run #{run.id}'.encode() in page.data + assert client.post("/tasks/state").status_code == 405 + + +@pytest.mark.parametrize("status", ["queued", "running"]) +def test_recovery_links_and_live_run_progress(live_app, status): + run = start_live_run(live_app, status=status, recovery=True) + client = live_app.test_client() + display_status = "Recovery queued" if status == "queued" else "recovering" + state = client.get(f"/runs/{run.id}/state").get_json() + assert state["status"] == status + assert state["display_status"] == display_status + assert state["recovery_of_run_id"] == run.recovery_of_run_id + assert state["progress"] == "2 of 5\nETA: 3 minutes" + output = client.get(f"/runs/{run.id}/output?offset=0").get_json() + assert output["progress"] == state["progress"] + assert output["display_status"] == display_status + detail = client.get(f"/runs/{run.id}") + assert f'href="/runs/{run.recovery_of_run_id}"'.encode() in detail.data + assert b'id="run-progress" class="run-progress">2 of 5\nETA: 3 minutes' in detail.data + old_detail = client.get(f"/runs/{run.recovery_of_run_id}") + assert b"Recovery run" in old_detail.data + assert f'href="/runs/{run.id}"'.encode() in old_detail.data + old_state = client.get(f"/runs/{run.recovery_of_run_id}/state").get_json() + assert old_state["recovery_run"]["run_id"] == run.id + assert b"recovery of" in client.get("/history/task/example/long.py").data + + +@pytest.mark.parametrize("accept", ["text/html", "application/json"]) +@pytest.mark.parametrize("target", ["/tasks/related/change.py/run", "/groups/related/run"]) +def test_conflicting_submissions_return_409_with_blocking_run(live_app, accept, target): + run = start_live_run(live_app) + response = live_app.test_client().post(target, headers={"Accept": accept}) + assert response.status_code == 409 + if accept == "application/json": + payload = response.get_json() + assert payload["blocked_by"]["run_id"] == run.id + assert payload["blocked_by"]["detail_url"] == f"/runs/{run.id}" + assert f"#{run.id}" in payload["error"] + else: + assert f'href="/runs/{run.id}"'.encode() in response.data + assert live_app.extensions["run_store"].last_run("task", "related/change.py") is None + unrelated = live_app.test_client().post("/tasks/independent/free.py/run", headers={"Accept": accept}) + assert unrelated.status_code == (202 if accept == "application/json" else 302) + + +@pytest.mark.parametrize("accept", ["text/html", "application/json"]) +@pytest.mark.parametrize("target", ["/tasks/example/long.py/run", "/groups/example/run"]) +def test_shutdown_is_reported_without_enqueuing(live_app, accept, target): + live_app.extensions["run_manager"]._closing.set() + response = live_app.test_client().post(target, headers={"Accept": accept}) + assert response.status_code == 503 + assert b"shutting down" in response.data + + +def test_startup_precedes_cleanup_and_scheduler(settings, monkeypatch): + calls = [] + monkeypatch.setattr(RunManager, "startup", lambda *_: calls.append("startup")) + monkeypatch.setattr(RunManager, "cleanup_transient_files", lambda *_: calls.append("cleanup")) + monkeypatch.setattr(BackgroundScheduler, "start", lambda *_: calls.append("scheduler")) + create_app(settings) + assert calls == ["startup", "cleanup", "scheduler"] + + +def test_progress_is_plain_text_and_escaped(live_app): + run = start_live_run(live_app) + path = live_app.config["settings"].state_root / "runs" / f"{run.id}.progress.json" + path.write_text(json.dumps({"text": "\nNext item"})) + for url in ("/", f"/runs/{run.id}"): + page = live_app.test_client().get(url) + assert b"<script>not HTML</script>\nNext item" in page.data diff --git a/python-tools/tests/test_live_status_browser.py b/python-tools/tests/test_live_status_browser.py new file mode 100644 index 0000000..876f6e2 --- /dev/null +++ b/python-tools/tests/test_live_status_browser.py @@ -0,0 +1,261 @@ +"""Optional Playwright checks against disposable, locally served task fixtures.""" + +import json +import os +import threading +from pathlib import Path + +import pytest +from werkzeug.serving import make_server + +from .conftest import write_task +from .test_live_status import live_app as live_app, start_live_run + +playwright = pytest.importorskip("playwright.sync_api") + + +@pytest.fixture(scope="module") +def browser(): + with playwright.sync_playwright() as driver: + executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE", driver.chromium.executable_path) + if not Path(executable).is_file(): + pytest.skip("Install Playwright's Chromium or set PLAYWRIGHT_CHROMIUM_EXECUTABLE") + browser = driver.chromium.launch(executable_path=executable) + yield browser + browser.close() + + +@pytest.fixture +def live_server(live_app): + server = make_server("127.0.0.1", 0, live_app, threaded=True) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield live_app, f"http://127.0.0.1:{server.server_port}" + server.shutdown() + thread.join(timeout=5) + server.server_close() + + +@pytest.fixture +def live_page(browser, live_server): + app, url = live_server + context = browser.new_context(accept_downloads=True) + context.add_init_script("""(() => { + window.leaveListeners = []; + const original = window.addEventListener; + window.addEventListener = function(type, ...args) { + if (type === 'beforeunload') window.leaveListeners.push(type); + return original.call(this, type, ...args); + }; + })();""") + page = context.new_page() + errors = [] + dialogs = [] + page.on("pageerror", lambda error: errors.append(str(error))) + + def dismiss(dialog): + dialogs.append(dialog.type) + dialog.dismiss() + + page.on("dialog", dismiss) + yield app, url, page + context.close() + assert not errors + assert not dialogs + + +def task_form(page, task_id): + return page.locator(f'form[data-task-id="{task_id}"]') + + +@pytest.mark.parametrize("width", [1280, 390, 320]) +@pytest.mark.parametrize("status", ["queued", "running", "recovering", "recovery queued"]) +def test_reload_navigation_and_multiline_progress(live_page, tmp_path, width, status): + app, url, page = live_page + run = start_live_run( + app, status="queued" if status in {"queued", "recovery queued"} else "running", + recovery=status in {"recovering", "recovery queued"}, + ) + page.set_viewport_size({"width": width, "height": 844}) + page.goto(url) + own = task_form(page, "example/long.py") + playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize()) + playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes") + assert own.locator('button[type="submit"]').is_disabled() + for name in ("example/sibling.py", "related/change.py"): + form = task_form(page, name) + playwright.expect(form.locator(".task-result")).to_contain_text("Blocked by") + assert form.locator('button[type="submit"]').is_disabled() + assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled() + page.reload() + playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes") + own.locator(f'.task-result a[href="/runs/{run.id}"]').click() + playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes") + page.reload() + playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes") + playwright.expect(page.locator("#run-status")).to_have_text("Recovery queued" if status == "recovery queued" else status) + assert page.locator("#run-progress").evaluate("el => getComputedStyle(el).whiteSpace") == "pre-wrap" + page.screenshot(path=str(tmp_path / f"run-{status}-{width}.png"), full_page=True) + assert page.evaluate("window.leaveListeners") == [] + page.locator('main a[href="/"]').first.click() + playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize()) + assert page.evaluate("document.documentElement.scrollWidth <= innerWidth") + page.screenshot(path=str(tmp_path / f"live-{status}-{width}.png"), full_page=True) + + +def test_first_render_blocks_conflicts_without_javascript(browser, live_server): + app, url = live_server + start_live_run(app) + context = browser.new_context(java_script_enabled=False) + try: + page = context.new_page() + page.goto(url) + assert task_form(page, "example/long.py").locator('button[type="submit"]').is_disabled() + assert task_form(page, "related/change.py").locator('button[type="submit"]').is_disabled() + assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled() + assert task_form(page, "example/long.py").locator(".task-progress").inner_text() == "2 of 5\nETA: 3 minutes" + finally: + context.close() + + +def test_live_changes_and_racing_conflict_show_blocking_run(live_page): + app, url, page = live_page + page.goto(url) + snapshot = {"value": app.test_client().get("/tasks/state").get_json()} + page.route("**/tasks/state", lambda route: route.fulfill(json=snapshot["value"])) + run = start_live_run(app, status="queued") + related = task_form(page, "related/change.py") + with page.expect_response(lambda response: response.url.endswith("/tasks/related/change.py/run") and response.status == 409): + related.locator('button[type="submit"]').click() + snapshot["value"] = app.test_client().get("/tasks/state").get_json() + playwright.expect(related.locator(".task-result")).to_contain_text(f"Blocked by run #{run.id}") + assert related.locator('button[type="submit"]').is_disabled() + own = task_form(page, "example/long.py") + playwright.expect(own.locator(".task-result")).to_contain_text("Queued") + app.extensions["run_store"].claim_next("example") + snapshot["value"] = app.test_client().get("/tasks/state").get_json() + playwright.expect(own.locator(".task-result")).to_contain_text("Running") + app.extensions["run_store"].finish(run.id, status="succeeded") + snapshot["value"] = app.test_client().get("/tasks/state").get_json() + playwright.expect(own.locator(".task-result")).to_contain_text("Completed successfully.") + playwright.expect(related.locator('button[type="submit"]')).to_be_enabled() + + +def test_long_task_submission_redirects_and_restores_on_return(live_page): + app, url, page = live_page + page.goto(url) + task_form(page, "example/long.py").locator('button[type="submit"]').click() + page.wait_for_url("**/runs/*") + playwright.expect(page.locator("#run-status")).to_have_text("queued") + page.locator('main a[href="/"]').first.click() + playwright.expect(task_form(page, "example/long.py").locator(".task-result")).to_contain_text("Queued") + assert app.extensions["run_store"].last_run("task", "example/long.py").status == "queued" + + +def test_csv_download_after_reload_does_not_repeat(live_page, settings): + app, url, page = live_page + task = write_task(settings.task_root, "exports", "review.py", 'print("done")\n') + (task.parent / "task-inputs.json").write_text(json.dumps({ + "version": 1, "tasks": {task.name: {"wait_for_result": True, "download_artifacts": True}}, + })) + page.goto(url) + form = task_form(page, "exports/review.py") + form.locator('button[type="submit"]').click() + playwright.expect(form.locator(".task-result")).to_contain_text("Queued") + page.reload() + playwright.expect(form.locator(".task-result")).to_contain_text("Queued") + assert page.evaluate("window.leaveListeners") == [] + run = app.extensions["run_store"].last_run("task", "exports/review.py") + artifacts = settings.state_root / "runs" / f"{run.id}.artifacts" + artifacts.mkdir(parents=True, exist_ok=True) + (artifacts / "report.json").write_text('{"ok":true}') + (artifacts / "review.csv").write_text("item,decision\n1,keep\n") + downloads = [] + page.on("download", lambda download: downloads.append(download.suggested_filename)) + with page.expect_download() as downloaded: + app.extensions["run_store"].finish(run.id, status="succeeded") + download = downloaded.value + assert download.suggested_filename == "review.csv" + assert Path(download.path()).read_text() == "item,decision\n1,keep\n" + playwright.expect(form.locator(".task-result")).to_contain_text("Completed successfully.") + assert form.locator('button[type="submit"]').is_enabled() + page.wait_for_timeout(2200) + assert downloads == ["review.csv"] + assert (artifacts / "report.json").is_file() + assert not (artifacts / "review.csv").exists() + + +def test_incremental_output_preserves_selection_and_stops_polling(live_page, settings): + app, url, page = live_page + run = start_live_run(app) + log = settings.state_root / "runs" / f"{run.id}.log" + log.write_text("Select this text\n") + store = app.extensions["run_store"] + store.set_log_path(run.id, str(log)) + polls = [] + artifacts = [] + + def record(request): + if "/output?" in request.url: + polls.append(request.url) + if request.url.endswith("/artifacts"): + artifacts.append(request.url) + + page.on("request", record) + page.goto(f"{url}/runs/{run.id}") + page.evaluate("""() => { + const node = document.querySelector('#run-output').firstChild; + const range = document.createRange(); + range.setStart(node, 0); range.setEnd(node, 16); + getSelection().removeAllRanges(); getSelection().addRange(range); + }""") + with log.open("a") as stream: + stream.write("More output\n") + progress = settings.state_root / "runs" / f"{run.id}.progress.json" + temporary = progress.with_suffix(".tmp") + temporary.write_text(json.dumps({"text": "3 of 5\nETA: 2 minutes"})) + temporary.replace(progress) + playwright.expect(page.locator("#run-output")).to_contain_text("More output") + playwright.expect(page.locator("#run-progress")).to_have_text("3 of 5\nETA: 2 minutes") + assert page.evaluate("getSelection().toString()") == "Select this text" + assert not artifacts + with log.open("a") as stream: + stream.write("Finished\n") + store.finish(run.id, status="succeeded") + playwright.expect(page.locator("#run-output")).to_contain_text("Finished") + playwright.expect(page.locator('form[action$="/cancel"]')).to_have_count(0) + assert page.evaluate("getSelection().toString()") == "Select this text" + finished_polls = len(polls) + page.wait_for_timeout(2300) + assert len(polls) == finished_polls + assert len(artifacts) == 1 + assert all("offset=" in request for request in polls) + + +def test_execute_control_appears_and_tracks_conflicts_after_completion(live_page, settings): + app, url, page = live_page + manifest = settings.task_root / "example" / "task-inputs.json" + manifest.write_text(json.dumps({ + "version": 1, "tasks": {"long.py": {"execution": { + "field": "mode", "dry_run_value": "review", "dry_run_label": "Review", + "execute_value": "apply", "execute_label": "Execute", + }}}, + })) + manager = app.extensions["run_manager"] + catalog = app.extensions["catalog"] + run, _ = manager.run_task(catalog.task("example/long.py"), {"mode": "review"}) + manager.store.claim_next("example") + page.goto(f"{url}/runs/{run.id}") + manager.store.finish(run.id, status="succeeded") + execute = task_form(page, "example/long.py") + playwright.expect(execute.locator('button[type="submit"]')).to_be_visible() + blocker, _ = manager.run_task(catalog.task("example/sibling.py")) + playwright.expect(execute.locator(".task-result")).to_contain_text(f"Blocked by run #{blocker.id}") + assert execute.locator('button[type="submit"]').is_disabled() + manager.store.finish(blocker.id, status="succeeded") + playwright.expect(execute.locator('button[type="submit"]')).to_be_enabled() + execute.locator('button[type="submit"]').click() + page.wait_for_url(lambda current: current.startswith(f"{url}/runs/") and current != f"{url}/runs/{run.id}") + accepted = manager.store.last_run("task", "example/long.py") + assert accepted.id != run.id + assert json.loads(accepted.input_json) == {"mode": "apply"} diff --git a/python-tools/tests/test_recovery.py b/python-tools/tests/test_recovery.py new file mode 100644 index 0000000..1bbf0b4 --- /dev/null +++ b/python-tools/tests/test_recovery.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from app.discovery import TaskCatalog +from app.runner import RunManager, TaskConflictError +from app.store import RunStore + +from .conftest import write_task +from .test_runner import wait_for, wait_for_output + + +COUNTER = '''"""Reconcile a small external ledger before continuing.""" +import json, os +from pathlib import Path +from app import task_runtime as runtime + +def main(): + data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text()) + ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt' + existing = ledger.read_text().splitlines() if ledger.exists() else [] + if not runtime.checkpoint_data(): + runtime.checkpoint({'next': 0}, force=True) + for number in range(len(existing), data.get('count', 50)): + runtime.check_stop() + with ledger.open('a') as stream: + stream.write(str(number) + '\\n') + runtime.checkpoint({'next': number + 1}) + runtime.progress(f'Items: {number + 1}', force=True) + runtime.sleep(.03) + +def recover(): + print('Checking external ledger', flush=True) + main() + +if __name__ == '__main__': + main() +''' + + +def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager): + write_task(manager.settings.task_root, 'count', 'task.py', COUNTER) + task = manager.catalog.task('count/task.py') + run, _ = manager.run_task(task, {'count': 50}) + wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3') + manager.shutdown(timeout=1) + assert manager.store.get(run.id).status == 'interrupted' + work = manager._work_root(run) + saved = json.loads((work / 'checkpoint.json').read_text()) + assert saved['clean_shutdown'] is True + assert (work / 'input.json').exists() + manager.cleanup_transient_files(artifact_max_age_seconds=0) + assert (work / 'input.json').exists() + resumed = RunManager(manager.settings, manager.catalog, manager.store) + resumed.startup() + recovery = manager.store.recovery_for(run.id) + assert recovery is not None + assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded' + assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)] + assert manager.store.get(run.id).status == 'interrupted' + assert not (work / 'input.json').exists() + assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text() + resumed.shutdown(timeout=1) + + +def test_forced_container_style_exit_recovers_from_older_checkpoint(settings): + settings.task_root.mkdir(parents=True) + write_task(settings.task_root, 'count', 'task.py', COUNTER) + control = settings.state_root / 'control.json' + program = ''' +import json,time +from pathlib import Path +from app.config import Settings +from app.discovery import TaskCatalog +from app.runner import RunManager +from app.store import RunStore +s=Settings.from_environment() +c=TaskCatalog(s.task_root) +db=RunStore(s.database_path) +db.initialize() +m=RunManager(s,c,db) +r,_=m.run_task(c.task('count/task.py'),{'count':50}) +while r.id not in m._processes: + time.sleep(.01) +Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid})) +while True: + time.sleep(1) +''' + environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]), + 'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root), + 'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root), + 'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)} + process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True) + child_pid = None + resumed = None + try: + wait_for_output(control, 'pid') + data = json.loads(control.read_text()) + child_pid = data['pid'] + wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n') + process.kill() + process.wait(timeout=3) + os.killpg(child_pid, signal.SIGKILL) + child_pid = None + saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text()) + assert saved['clean_shutdown'] is False + assert saved['data']['next'] == 0 + store = RunStore(settings.database_path) + resumed = RunManager(settings, TaskCatalog(settings.task_root), store) + resumed.startup() + recovery = store.recovery_for(data['run']) + assert recovery is not None + assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded' + assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)] + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=3) + if child_pid is not None: + try: + os.killpg(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + if resumed: + resumed.shutdown(timeout=1) + + +def test_cancelled_tasks_never_auto_recover(manager): + write_task(manager.settings.task_root, 'count', 'task.py', COUNTER) + task = manager.catalog.task('count/task.py') + run, _ = manager.run_task(task, {'count': 100}) + wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n') + manager.request_cancel(run.id) + wait_for(manager, run.id, {'cancelled'}) + manager.shutdown(timeout=1) + new = RunManager(manager.settings, manager.catalog, manager.store) + new.startup() + assert manager.store.recovery_for(run.id) is None + assert not (manager._work_root(run) / 'input.json').exists() + new.shutdown(timeout=1) + + +def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager): + path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():')) + run, _ = manager.run_task(manager.catalog.task('count/task.py')) + wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n') + manager.shutdown(timeout=1) + assert manager.store.get(run.id).status == 'interrupted' + path.write_text(COUNTER) + legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy') + write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER) + # The current task has opted out again; neither it nor a pre-runtime run may resume. + path.write_text(COUNTER.replace('def recover():', 'def unused():')) + new = RunManager(manager.settings, manager.catalog, manager.store) + new.startup() + assert manager.store.recovery_for(run.id) is None + assert manager.store.recovery_for(legacy.id) is None + assert manager.store.active_runs() == [] + new.shutdown(timeout=1) + + +def test_shared_resources_block_cross_group_submissions_and_recoveries(manager): + for group in ('first','second'): + write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER) + first = manager.catalog.task('first/task.py') + second = manager.catalog.task('second/task.py') + run, _ = manager.run_task(first, {'count':50}) + with pytest.raises(TaskConflictError) as error: + manager.run_task(second) + assert error.value.run.id == run.id + manager.shutdown(timeout=1) + new = RunManager(manager.settings, manager.catalog, manager.store) + new.startup() + recovered = manager.store.recovery_for(run.id) + assert recovered is not None + with pytest.raises(TaskConflictError) as error: + new.run_task(second) + assert error.value.run.id == recovered.id + new.shutdown(timeout=1) + + +def test_shutdown_rejects_new_work(manager): + write_task(manager.settings.task_root,'tasks','task.py','print("hello")') + manager.shutdown(timeout=.1) + with pytest.raises(RuntimeError,match='shutting down'): + manager.run_task(manager.catalog.task('tasks/task.py')) diff --git a/python-tools/tests/test_runner.py b/python-tools/tests/test_runner.py index 6cf65c7..0c7be62 100644 --- a/python-tools/tests/test_runner.py +++ b/python-tools/tests/test_runner.py @@ -58,7 +58,11 @@ def test_tasks_in_one_group_are_serialized_and_groups_overlap(manager: RunManage other_task = catalog.task("second_group/other.py") assert first_task is not None and second_task is not None and other_task is not None first, _ = manager.run_task(first_task) - second, _ = manager.run_task(second_task) + second, _ = manager.store.enqueue( + target_kind="task", target_path=second_task.id, queue_group=second_task.group_id, + resources=second_task.resources, + ) + manager._wake_worker(second_task.group_id) other, _ = manager.run_task(other_task) wait_for(manager, first.id, {"running"}) diff --git a/python-tools/tests/test_shutdown_process.py b/python-tools/tests/test_shutdown_process.py new file mode 100644 index 0000000..ccb0ffd --- /dev/null +++ b/python-tools/tests/test_shutdown_process.py @@ -0,0 +1,478 @@ +"""Recovery through the real Waitress entry point, without manager test doubles.""" + +from __future__ import annotations + +import errno +import http.client +import json +import os +import signal +import socket +import sqlite3 +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing +from io import BytesIO +from pathlib import Path + +import pytest +from werkzeug.test import EnvironBuilder + + +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration") +PROJECT_ROOT = Path(__file__).resolve().parents[1] +UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n" +SECRET = "disposable-process-test-secret" + + +LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger.""" +import json +import os +from pathlib import Path +from app import task_runtime as runtime + +RESOURCES = ("shared-ledger",) +state = Path(os.environ["SERVER_MAINTENANCE_STATE"]) + +def values(): + data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text()) + assert data["secret"] == "disposable-process-test-secret" + assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n" + return data + +def main(): + data = values() + ledger = state / "ledger.txt" + existing = ledger.read_text().splitlines() if ledger.exists() else [] + if not runtime.checkpoint_data(): + runtime.checkpoint({"next": 0}, force=True) + items = Path(data["items"]).read_text().splitlines() + for number in range(len(existing), len(items)): + runtime.check_stop() + with ledger.open("a") as stream: + stream.write(items[number] + "\\n") + runtime.checkpoint({"next": number + 1}) + if number == 2: + runtime.progress("Original waiting", force=True) + while not (state / "finish").exists(): + runtime.sleep(.02) + +def recover(): + values() + (state / "observation.json").write_text(json.dumps({ + "checkpoint": runtime.checkpoint_data(), + "clean_shutdown": runtime.was_clean_shutdown(), + "ledger": (state / "ledger.txt").read_text().splitlines(), + })) + runtime.progress("Recovery waiting", force=True) + while not (state / "finish").exists(): + runtime.sleep(.02) + main() + +if __name__ == "__main__": + main() +''' + + +BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline.""" +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from app import task_runtime as runtime + +runtime.checkpoint({"next": 0}, force=True) +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) +runtime.checkpoint({"next": 1}) +Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text( + json.dumps([os.getpid(), child.pid])) +runtime.progress("Blocking", force=True) +time.sleep(60) +''' + + +def wait_until(condition, description, timeout=8): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = condition() + if result: + return result + time.sleep(.02) + pytest.fail(f"Timed out waiting for {description}") + + +def read_json(path): + try: + return json.loads(path.read_text()) + except FileNotFoundError: + return None + + +def process_running(pid): + try: + status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1] + except FileNotFoundError: + return False + return status[0] not in {"Z", "X"} + + +class ProductionApp: + def __init__(self, root): + self.task_root = root / "tasks" + self.state_root = root / "state" + self.credentials_root = root / "credentials" + for path in (self.task_root, self.state_root, self.credentials_root): + path.mkdir() + self.database = self.state_root / "server-maintenance.sqlite3" + self.log_path = root / "server.log" + self.process = None + self.port = None + self.grace = 1 + + def write_task(self, group, filename, source): + path = self.task_root / group / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + return path + + def start(self): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + self.launch(port) + + def ready(): + assert self.process.poll() is None, self.log_path.read_text() + try: + status, _ = self.request("GET", "/tasks/state") + except (ConnectionError, TimeoutError): + return False + return status == 200 + + wait_until(ready, f"Waitress startup; log: {self.log_path}") + + def launch(self, port): + assert self.process is None or self.process.poll() is not None + self.port = port + environment = {key: value for key, value in os.environ.items() + if not key.startswith("SERVER_MAINTENANCE_")} + environment.update({ + "PYTHONPATH": str(PROJECT_ROOT), + "PYTHONUNBUFFERED": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "SERVER_MAINTENANCE_HOST": "127.0.0.1", + "SERVER_MAINTENANCE_PORT": str(self.port), + "SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root), + "SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root), + "SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root), + "SERVER_MAINTENANCE_PYTHON": sys.executable, + "SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace), + }) + with self.log_path.open("ab") as log: + self.process = subprocess.Popen( + [sys.executable, "-m", "app"], cwd=PROJECT_ROOT, + env=environment, stdin=subprocess.DEVNULL, stdout=log, + stderr=subprocess.STDOUT, start_new_session=True, + ) + + def request(self, method, path, data=None): + headers = {"Accept": "application/json"} + body = None + if data is not None: + with closing(EnvironBuilder(method=method, data=data)) as builder: + environ = builder.get_environ() + body = environ["wsgi.input"].read() + headers["Content-Type"] = environ["CONTENT_TYPE"] + connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2) + try: + connection.request(method, path, body=body, headers=headers) + response = connection.getresponse() + payload = response.read() + assert response.getheader("Content-Type", "").startswith("application/json"), payload + return response.status, json.loads(payload) + finally: + connection.close() + + def rows(self): + with closing(sqlite3.connect(self.database)) as connection: + connection.row_factory = sqlite3.Row + return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")] + + def run(self, task, data=None): + status, response = self.request("POST", f"/tasks/{task}/run", data) + assert status == 202, response + return response["run_id"] + + def wait_status(self, run_id, status): + def matching(): + run = next(row for row in self.rows() if row["id"] == run_id) + if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}: + assert run["status"] == status, (run, self.log_path.read_text()) + return run if run["status"] == status else None + return wait_until(matching, f"run {run_id} to become {status}") + + def wait_progress(self, run_id, text): + path = self.state_root / "runs" / f"{run_id}.progress.json" + wait_until(lambda: (read_json(path) or {}).get("text") == text, text) + + def pids(self): + # Match only this disposable instance, including separately-sessioned workers. + marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode() + found = [] + for path in Path("/proc").iterdir(): + if not path.name.isdigit(): + continue + try: + environment = (path / "environ").read_bytes().split(b"\0") + except (FileNotFoundError, ProcessLookupError, PermissionError): + continue + if marker in environment: + found.append(int(path.name)) + return found + + def kill_remaining(self): + def kill_snapshot(): + pids = self.pids() + for pid in pids: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + return not pids + + kill_snapshot() + if self.process is not None: + self.process.wait(timeout=3) + wait_until(kill_snapshot, "disposable instance teardown", timeout=3) + + def stop(self): + started = time.monotonic() + self.process.send_signal(signal.SIGTERM) + try: + code = self.process.wait(timeout=self.grace + 3) + except subprocess.TimeoutExpired: + pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}") + elapsed = time.monotonic() - started + assert code == 0, self.log_path.read_text() + assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text()) + return elapsed + + +@pytest.fixture +def production_app(tmp_path): + app = ProductionApp(tmp_path) + try: + yield app + finally: + app.kill_remaining() + + +@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"]) +def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal): + app = production_app + ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK) + (ledger_task.parent / "task-inputs.json").write_text(json.dumps({ + "version": 1, + "tasks": {"task.py": {"inputs": [ + {"name": "items", "label": "Items", "type": "file", "required": True, + "accept": [".txt"], "maximum_bytes": 1024}, + {"name": "secret", "label": "Secret", "type": "text", "required": True, + "sensitive": True}, + ]}}, + })) + app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n') + app.write_task("maintenance", "task.py", 'print("done")\n') + app.start() + original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")}) + app.wait_progress(original_id, "Original waiting") + runs = app.state_root / "runs" + work = runs / f"{original_id}.work" + saved_input = json.loads((work / "input.json").read_text()) + upload = Path(saved_input["items"]) + assert saved_input["secret"] == SECRET + assert SECRET not in app.rows()[0]["input_json"] + assert read_json(work / "checkpoint.json")["data"] == {"next": 0} + old_pids = app.pids() + if shutdown_signal == signal.SIGTERM: + app.stop() + assert app.rows()[0]["status"] == "interrupted" + assert app.rows()[0]["exit_code"] == 75 + else: + # SIGKILL the entire disposable instance, as container teardown does. + app.kill_remaining() + assert app.process.returncode == -signal.SIGKILL + assert app.rows()[0]["status"] == "running" + wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit") + saved = read_json(work / "checkpoint.json") + clean = shutdown_signal == signal.SIGTERM + assert saved["clean_shutdown"] is clean + assert saved["data"] == {"next": 3 if clean else 0} + assert upload.read_bytes() == UPLOAD + assert json.loads((work / "input.json").read_text()) == saved_input + + stale = time.time() - 45 * 86400 + for path in (work, work / "input.json", upload.parent, upload): + os.utime(path, (stale, stale)) + app.start() + recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id) + app.wait_progress(recovery["id"], "Recovery waiting") + observation = read_json(app.state_root / "tasks/ledger/observation.json") + assert observation == { + "checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean, + "ledger": ["0", "1", "2"], + } + status, states = app.request("GET", "/tasks/state") + assert status == 200 + assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering" + assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"] + status, conflict = app.request("POST", "/tasks/contender/task.py/run") + assert status == 409, conflict + assert conflict["blocked_by"]["run_id"] == recovery["id"] + assert not any(row["target_path"] == "contender/task.py" for row in app.rows()) + + # A real completion invokes cleanup while recovery still needs its ancestor's files. + cleanup_probe = runs / "999999.input.json" + cleanup_probe.write_text("{}\n") + maintenance_id = app.run("maintenance/task.py") + app.wait_status(maintenance_id, "succeeded") + wait_until(lambda: not cleanup_probe.exists(), "completion cleanup") + assert upload.read_bytes() == UPLOAD + assert json.loads((work / "input.json").read_text()) == saved_input + assert app.rows()[0]["status"] == "interrupted" + + (app.state_root / "tasks/ledger/finish").touch() + app.wait_status(recovery["id"], "succeeded") + assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)] + wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup") + app.stop() + assert not app.pids(), "A task escaped registration/shutdown" + + +def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app): + app = production_app + app.write_task("blocking", "task.py", BLOCKING_TASK) + app.write_task("watcher", "internal/task.py", BLOCKING_TASK) + (app.task_root / "watcher/background-tasks.json").write_text(json.dumps({ + "version": 1, + "tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}], + })) + app.start() + run_id = app.run("blocking/task.py") + app.wait_progress(run_id, "Blocking") + background_work = app.state_root / "background/watcher/watcher.work" + wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking", + "background worker to start") + work = app.state_root / "runs" / f"{run_id}.work" + task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json") + assert all(process_running(pid) for pid in task_pids) + elapsed = app.stop() + assert elapsed >= app.grace * .8, "The hard deadline was not exercised" + wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit") + assert not app.pids(), "A foreground/background task escaped shutdown registration" + assert read_json(work / "checkpoint.json")["clean_shutdown"] is False + assert read_json(work / "checkpoint.json")["data"] == {"next": 0} + assert (work / "input.json").exists() + + +def test_production_graceful_pause_does_not_leave_live_descendants(production_app): + app = production_app + source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n") + app.write_task("cooperative", "task.py", source) + app.start() + run_id = app.run("cooperative/task.py") + app.wait_progress(run_id, "Blocking") + work = app.state_root / "runs" / f"{run_id}.work" + worker_pid, child_pid = read_json(work / "pids.json") + app.stop() + assert read_json(work / "checkpoint.json")["clean_shutdown"] is True + assert not process_running(worker_pid) + wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5) + + +def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app): + app = production_app + source = ''' +from app import task_runtime as runtime +runtime.progress("Started", force=True) +while True: + runtime.sleep(.02) +''' + groups = [f"queue_{index}" for index in range(12)] + for group in groups: + app.write_task(group, "task.py", source) + app.start() + first_id = app.run(f"{groups[0]}/task.py") + app.wait_progress(first_id, "Started") + + def submit(group): + try: + return app.request("POST", f"/tasks/{group}/task.py/run")[0] + except (ConnectionError, TimeoutError, http.client.RemoteDisconnected): + # The listener may close before this in-flight request is accepted. + return None + + with ThreadPoolExecutor(max_workers=6) as requests: + pending = [requests.submit(submit, group) for group in groups[1:]] + app.stop() + statuses = [request.result(timeout=3) for request in pending] + assert all(status in {202, 503, None} for status in statuses), statuses + assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration" + assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text() + + +def test_production_bind_failure_stops_workers_started_by_recovery(production_app): + app = production_app + source = ''' +from app import task_runtime as runtime + +def main(): + runtime.progress("Started", force=True) + while True: + runtime.sleep(.02) + +def recover(): + main() + +if __name__ == "__main__": + main() +''' + app.write_task("recoverable", "task.py", source) + app.start() + run_id = app.run("recoverable/task.py") + app.wait_progress(run_id, "Started") + app.stop() + assert app.rows()[0]["status"] == "interrupted" + + # Delay startup cleanup without patching app code, until recovery is running. + cleanup_gate = app.state_root / "tasks/.orphaned-groups.json" + os.mkfifo(cleanup_gate) + with socket.socket() as occupied: + occupied.bind(("127.0.0.1", 0)) + occupied.listen() + app.launch(occupied.getsockname()[1]) + recovery = wait_until( + lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None), + "startup recovery registration", + ) + app.wait_progress(recovery["id"], "Started") + + def release_cleanup(): + try: + descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK) + except OSError as error: + if error.errno == errno.ENXIO: + return False + raise + with os.fdopen(descriptor, "w") as gate: + json.dump({}, gate) + return True + + wait_until(release_cleanup, "startup cleanup gate", timeout=2) + assert app.process.wait(timeout=4) != 0 + assert "Address already in use" in app.log_path.read_text() + assert any(row["recovery_of_run_id"] == run_id for row in app.rows()) + wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5) diff --git a/python-tools/tests/test_task_runtime.py b/python-tools/tests/test_task_runtime.py new file mode 100644 index 0000000..c432a5e --- /dev/null +++ b/python-tools/tests/test_task_runtime.py @@ -0,0 +1,81 @@ +import json +import signal + +import pytest + +from app import task_runtime as runtime + + +@pytest.fixture +def work(tmp_path,monkeypatch): + monkeypatch.setenv('SERVER_MAINTENANCE_WORK',str(tmp_path)) + old=signal.getsignal(signal.SIGUSR1) + runtime.initialize() + yield tmp_path + signal.signal(signal.SIGUSR1,old) + runtime._pause_requested=False + runtime._defer=0 + + +def test_safe_point_saves_latest_memory_state_and_marks_clean(work): + runtime.checkpoint({'next':1},force=True) + runtime.checkpoint({'next':2}) + assert json.loads((work/'checkpoint.json').read_text())['data']=={'next':1} + runtime.request_pause() + with pytest.raises(runtime.TaskPaused): + runtime.check_stop() + saved=json.loads((work/'checkpoint.json').read_text()) + assert saved['data']=={'next':2} + assert saved['clean_shutdown'] is True + runtime.initialize() + assert runtime.was_clean_shutdown() is True + assert json.loads((work/'checkpoint.json').read_text())['clean_shutdown'] is False + + +def test_small_critical_operation_finishes_before_pause(work): + with pytest.raises(runtime.TaskPaused): + with runtime.critical_section(): + runtime.request_pause() + runtime.checkpoint({'committed':True},force=True) + assert runtime.checkpoint_data()=={'committed':True} + saved=json.loads((work/'checkpoint.json').read_text()) + assert saved['data']=={'committed':True} + assert saved['clean_shutdown'] is True + + +def test_sleep_is_interruptible(work): + runtime.request_pause() + with pytest.raises(runtime.TaskPaused): + runtime.sleep(30) + + +def test_multiline_progress_has_per_step_eta_only(work,monkeypatch): + clock=[100.] + monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0]) + progress=runtime.Progress({'Removing files':10,'Linking files':20}) + progress.update('Removing files',0,force=True) + clock[0]+=30 + progress.update('Removing files',5,force=True) + text=json.loads((work/'progress.json').read_text())['text'] + assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%' + assert (work/'progress.json').stat().st_mode&0o777==0o600 + + +def test_runtime_does_not_write_or_pause_direct_scripts_without_work(monkeypatch): + monkeypatch.delenv('SERVER_MAINTENANCE_WORK',raising=False) + runtime.request_pause() + runtime.checkpoint({'ignored':True}) + runtime.progress('hello',force=True) + assert runtime.checkpoint_data()=={} + runtime._pause_requested=False + + +def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch): + clock=[100.] + monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0]) + progress=runtime.Progress({'Items':50}) + progress.update('Items',39,force=True) + assert 'ETA' not in json.loads((work/'progress.json').read_text())['text'] + clock[0]+=60 + progress.update('Items',40,force=True) + assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes' diff --git a/python-tools/tests/test_upload_replay.py b/python-tools/tests/test_upload_replay.py index a4d2145..13db2eb 100644 --- a/python-tools/tests/test_upload_replay.py +++ b/python-tools/tests/test_upload_replay.py @@ -141,3 +141,35 @@ def test_removing_file_input_from_task_cleans_up_unused_review_upload(settings): manifest.write_text(json.dumps(definition)) app.extensions["run_manager"].cleanup_transient_files() assert not original.parent.exists() + + +def test_conflicting_execute_is_linked_and_keeps_review_upload(settings, monkeypatch): + app, client, review_id = start_review(settings) + manager = app.extensions["run_manager"] + original = Path(json.loads(manager.store.get(review_id).input_json)["csv"]) + write_task(settings.task_root, "imports", "other.py", 'print("done")\n') + monkeypatch.setattr(manager, "_wake_worker", lambda *_: None) + blocker, _ = manager.run_task(app.extensions["catalog"].task("imports/other.py")) + page = client.get(f"/runs/{review_id}") + assert b'fieldset class="task-controls" disabled' in page.data + assert f'href="/runs/{blocker.id}"'.encode() in page.data + response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"}) + assert response.status_code == 409 + assert response.get_json()["blocked_by"]["run_id"] == blocker.id + assert original.is_file() + manager.store.finish(blocker.id, status="cancelled") + accepted = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"}) + assert accepted.status_code == 202 + assert accepted.get_json()["detail_url"].startswith("/runs/") + assert accepted.get_json()["run_id"] != review_id + + +def test_shutdown_during_execute_keeps_review_upload(settings): + app, client, review_id = start_review(settings) + manager = app.extensions["run_manager"] + original = Path(json.loads(manager.store.get(review_id).input_json)["csv"]) + manager._closing.set() + response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"}) + assert response.status_code == 503 + assert "shutting down" in response.get_json()["error"] + assert original.is_file() diff --git a/python-tools/tests/test_worker.py b/python-tools/tests/test_worker.py new file mode 100644 index 0000000..c43cc2b --- /dev/null +++ b/python-tools/tests/test_worker.py @@ -0,0 +1,56 @@ +from urllib.error import HTTPError, URLError + +import pytest + +from app.worker import service_unavailable +from app import worker + + +@pytest.mark.parametrize('error', [ + ConnectionRefusedError(), TimeoutError(), URLError(OSError('connection refused')), + HTTPError('http://example.invalid',503,'unavailable',{},None), +]) +def test_service_failures_can_retry_recovery(error): + wrapped=RuntimeError('Service request failed') + wrapped.__cause__=error + assert service_unavailable(wrapped) + + +@pytest.mark.parametrize('error', [ + RuntimeError('Unrecognized file'), FileNotFoundError(), PermissionError(), + HTTPError('http://example.invalid',401,'unauthorized',{},None), + HTTPError('http://example.invalid',400,'bad input',{},None), +]) +def test_task_and_authentication_errors_are_not_automatically_repeated(error): + assert not service_unavailable(error) + + +def test_database_starting_up_is_transient_but_authentication_is_not(): + error=RuntimeError('Database unavailable') + error.sqlstate='57P03' + assert service_unavailable(error) + error.sqlstate='28P01' + assert not service_unavailable(error) + + +def test_recovery_rechecks_live_state_after_lost_service_response(tmp_path, monkeypatch): + ledger = [] + calls = [] + waits = [] + + def recover(): + calls.append(list(ledger)) + if not ledger: + ledger.append('applied') + raise URLError(ConnectionResetError('response lost')) + + monkeypatch.setattr(worker.sys, 'argv', ['worker', str(tmp_path/'task.py')]) + monkeypatch.setenv('SERVER_MAINTENANCE_RECOVER','1') + monkeypatch.setattr(worker.runtime,'initialize',lambda:None) + monkeypatch.setattr(worker.runtime,'progress',lambda *a,**kw:None) + monkeypatch.setattr(worker.runtime,'sleep',waits.append) + monkeypatch.setattr(worker.runpy,'run_path',lambda *a,**kw:{'recover':recover}) + worker.main() + assert ledger==['applied'] + assert calls==[[],['applied']] + assert waits==[2]