diff --git a/python-tools/README.md b/python-tools/README.md index 0a24225..b231e0a 100644 --- a/python-tools/README.md +++ b/python-tools/README.md @@ -103,7 +103,7 @@ Every input has `name`, `label`, and `type`; `required` defaults to `false`. Set `sensitive` to `true` on text or file inputs that must not appear in run history. Text is masked, the database records `[redacted]`, and the original -value is available only through a private execution-input file. Interrupted +value is available only through the run's private work-directory input file. Interrupted recoverable tasks retain that input and their uploads until recovery finishes. For review-first tasks, use `execution`. It produces one split button: click @@ -219,13 +219,14 @@ For simple item-count percentages and per-step estimates: ```python tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20}) -tracker.update("Scanning files", 0, force=True) +tracker.update("Scanning files", 0) # After processing five files: tracker.update("Scanning files", 5) ``` -Each step has its own percentage and an approximate ETA based only on processed -items and elapsed time. Pending steps have no ETA. Tasks may instead publish their +Each step shows completed/total items, a percentage and an approximate ETA based +on items processed since starting or resuming. Stage starts and completion are +published immediately. Pending steps have no ETA. Tasks may instead publish their own text when totals are unknown; no global estimate is required. A group may also include a private `task-config.json` for stable local details diff --git a/python-tools/app/runner.py b/python-tools/app/runner.py index b53c450..dd30da7 100644 --- a/python-tools/app/runner.py +++ b/python-tools/app/runner.py @@ -50,6 +50,7 @@ class RunManager: self._processes: dict[int, subprocess.Popen[bytes]] = {} self._background_processes: set[subprocess.Popen[bytes]] = set() self._lock = threading.Lock() + self._cleanup_lock = threading.Lock() self._closing = threading.Event() def run_task( @@ -58,8 +59,6 @@ class RunManager: input_data: dict[str, str | int | list[str]] | None = None, uploads: Mapping[str, PendingUpload] | None = None, ) -> tuple[Run, bool]: - if self._closing.is_set(): - raise RuntimeError("The application is shutting down; try again after it restarts.") sensitive_names = {field.name for field in task.inputs if field.sensitive} stored_input = self._redact_input(input_data or {}, sensitive_names) with self._lock: @@ -72,33 +71,28 @@ class RunManager: target_kind="task", target_path=task.id, queue_group=task.group_id, input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",), ) + if created: + try: + uploaded_inputs = self._save_uploads(run.id, uploads or {}) + execution_input = {**(input_data or {}), **uploaded_inputs} + run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names)) + self._save_work_input(run, execution_input) + except (OSError, ValueError) as error: + self._delete_uploads(run.id) + self._discard_work_input(run) + return self.store.finish( + run.id, status="failed", message=f"Could not save task input: {error}", + ), True if created: - try: - uploaded_inputs = self._save_uploads(run.id, uploads or {}) - execution_input = {**(input_data or {}), **uploaded_inputs} - stored_input = self._redact_input(execution_input, sensitive_names) - run = self.store.set_input(run.id, stored_input) - if sensitive_names: - self._write_execution_input(run.id, execution_input) - self._save_work_input(run, execution_input) - except (OSError, ValueError) as error: - self._delete_uploads(run.id) - self._delete_execution_input(run.id) - self._discard_work_input(run) - return self.store.finish( - run.id, - status="failed", - message=f"Could not save uploaded file: {error}", - ), True self._wake_worker(task.group_id) return run, created def run_group(self, group: Group) -> tuple[Run, bool]: - if self._closing.is_set(): - raise RuntimeError("The application is shutting down.") resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources)) resources = resources or (f"group:{group.id}",) with self._lock: + if self._closing.is_set(): + raise RuntimeError("The application is shutting down.") for active in self.store.active_runs(): if active.target_kind == "group" and active.target_path == group.id: return active, False @@ -152,10 +146,10 @@ class RunManager: def shutdown(self, timeout: float | None = None) -> None: self._closing.set() - for event in self._wake_events.values(): - event.set() deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout) with self._lock: + for event in self._wake_events.values(): + event.set() processes = [*self._processes.values(), *self._background_processes] for process in processes: if process.poll() is None: @@ -199,7 +193,6 @@ class RunManager: completed = self.store.get(run.id) if completed is not None and completed.status == "cancelled": self._delete_uploads(run.id) - self._delete_execution_input(run.id) self._discard_work_input(completed) return completed @@ -271,7 +264,10 @@ class RunManager: orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS, ) -> None: """Remove transient run data and state for task groups removed long ago.""" + with self._cleanup_lock: + self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds) + def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None: runs_root = self.settings.state_root / "runs" cutoff = time.time() - artifact_max_age_seconds if runs_root.is_dir(): @@ -438,6 +434,8 @@ class RunManager: def _wake_worker(self, group_id: str) -> None: with self._lock: + if self._closing.is_set(): + return event = self._wake_events.setdefault(group_id, threading.Event()) worker = self._workers.get(group_id) if worker is None or not worker.is_alive(): @@ -454,15 +452,20 @@ class RunManager: def _worker_loop(self, group_id: str, event: threading.Event) -> None: while not self._closing.is_set(): event.clear() - run = self.store.claim_next(group_id) + # A queued run is claimable only after its private input and uploads are saved. + with self._lock: + if self._closing.is_set(): + return + run = self.store.claim_next(group_id) if run is None: event.wait() continue try: self._execute_root_run(run) finally: - for waiting in self._wake_events.values(): - waiting.set() + with self._lock: + for waiting in self._wake_events.values(): + waiting.set() def _execute_root_run(self, run: Run) -> None: if run.target_kind == "task": @@ -529,23 +532,17 @@ class RunManager: log_path = self.settings.state_root / "runs" / f"{run.id}.log" log_path.parent.mkdir(parents=True, exist_ok=True) - input_path = self._execution_input_path(run.id) artifact_root = self._artifact_root(run.id) artifact_root.mkdir(parents=True, exist_ok=True) try: work = self._work_root(run) - saved_input = work / "input.json" - if run.recovery_of_run_id is not None: - input_path = saved_input - elif not input_path.exists(): - input_path = self.settings.state_root / "runs" / f"{run.id}.input.json" - input_path.write_text(run.input_json + "\n", encoding="utf-8") - input_path.chmod(0o600) - if not saved_input.exists(): - self._save_work_input(run, json.loads(input_path.read_text())) + input_path = work / "input.json" + if not input_path.is_file(): + if run.recovery_of_run_id is not None: + raise FileNotFoundError("The original recovery input is no longer available") + self._save_work_input(run, json.loads(run.input_json)) except OSError as error: self._delete_uploads(run.id) - self._delete_execution_input(run.id) self._discard_work_input(run) shutil.rmtree(artifact_root, ignore_errors=True) return self.store.finish( @@ -595,8 +592,6 @@ class RunManager: owned = self._processes.pop(run.id, None) if owned is not None: kill_remaining_group(owned) - self._delete_execution_input(run.id) - (self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True) if self.store.is_cancel_requested(run.id) or ( parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) @@ -659,22 +654,6 @@ class RunManager: def _artifact_root(self, run_id: int) -> Path: return self.settings.state_root / "runs" / f"{run_id}.artifacts" - def _execution_input_path(self, run_id: int) -> Path: - return self.settings.state_root / "runs" / f"{run_id}.execution-input.json" - - def _write_execution_input( - self, - run_id: int, - input_data: Mapping[str, str | int | list[str]], - ) -> None: - path = self._execution_input_path(run_id) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(input_data, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8") - path.chmod(0o600) - - def _delete_execution_input(self, run_id: int) -> None: - self._execution_input_path(run_id).unlink(missing_ok=True) - @staticmethod def _redact_input( input_data: Mapping[str, str | int | list[str]], diff --git a/python-tools/app/task_runtime.py b/python-tools/app/task_runtime.py index 84c408f..ccf1af5 100644 --- a/python-tools/app/task_runtime.py +++ b/python-tools/app/task_runtime.py @@ -144,18 +144,21 @@ class Progress: self.baseline: dict[str, int] = {} def update(self, label: str, done: int, *, force: bool = False) -> None: + force = force or label not in self.started or done >= self.totals[label] self.started.setdefault(label, time.monotonic()) self.baseline.setdefault(label, done) self.done[label] = done lines = [] for name, total in self.totals.items(): count = self.done[name] - percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0) - line = f"{name}: {percent}%" + if total: + line = f"{name}: {count}/{total} ({min(100, int(count * 100 / total))}%)" + else: + line = f"{name}: {'no items' if name in self.started else 'waiting'}" processed = count - self.baseline.get(name, 0) if processed > 0 and count < total and name in self.started: seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count))) - estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes" - line += f", ETA {estimate}" + value, unit = (seconds, "second") if seconds < 60 else (round(seconds / 60), "minute") + line += f", ETA {value} {unit}{'' if value == 1 else 's'}" lines.append(line) progress("\n".join(lines), force=force) diff --git a/python-tools/tests/test_runner.py b/python-tools/tests/test_runner.py index 0c7be62..f8b6b0b 100644 --- a/python-tools/tests/test_runner.py +++ b/python-tools/tests/test_runner.py @@ -1,6 +1,8 @@ from __future__ import annotations import time +import threading +from concurrent.futures import ThreadPoolExecutor from dataclasses import replace from io import BytesIO from pathlib import Path @@ -379,3 +381,38 @@ def test_upload_is_removed_when_task_process_cannot_start(manager: RunManager) - assert created is True assert wait_for(broken_manager, run.id, {"failed"}) == "failed" assert not (broken_manager.settings.state_root / "runs" / f"{run.id}.uploads").exists() + + +def test_idle_worker_cannot_start_before_uploaded_input_is_ready(manager, monkeypatch): + write_task(manager.settings.task_root, "imports", "read.py", ''' +import json, os +from pathlib import Path +values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text()) +assert Path(values["document"]).read_text() == "uploaded content" +''') + task = manager.catalog.task("imports/read.py") + manager._wake_worker(task.group_id) + saving = threading.Event() + release = threading.Event() + save_uploads = manager._save_uploads + + def slow_upload(run_id, uploads): + saving.set() + assert release.wait(5) + return save_uploads(run_id, uploads) + + monkeypatch.setattr(manager, '_save_uploads', slow_upload) + upload = PendingUpload('document', '.txt', 1024, BytesIO(b'uploaded content')) + with ThreadPoolExecutor() as pool: + submitted = pool.submit(manager.run_task, task, uploads={'document': upload}) + try: + assert saving.wait(5) + run = manager.store.active_runs()[0] + manager._wake_events[task.group_id].set() + time.sleep(.1) + assert manager.store.get(run.id).status == 'queued' + finally: + release.set() + run, created = submitted.result(timeout=5) + assert created + assert wait_for(manager, run.id, {'succeeded', 'failed'}) == 'succeeded' diff --git a/python-tools/tests/test_runtime_cleanup.py b/python-tools/tests/test_runtime_cleanup.py index 5234c29..07d41b4 100644 --- a/python-tools/tests/test_runtime_cleanup.py +++ b/python-tools/tests/test_runtime_cleanup.py @@ -1,6 +1,8 @@ import json import os import time +import threading +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from app.web import create_app @@ -8,6 +10,32 @@ from .conftest import write_task from .test_runner import wait_for +def test_concurrent_cleanup_passes_do_not_overlap(manager, monkeypatch): + entered = threading.Event() + release = threading.Event() + calls = [] + + def cleanup(*args): + calls.append(args) + if len(calls) == 1: + entered.set() + assert release.wait(5) + + monkeypatch.setattr(manager, '_cleanup_transient_files', cleanup) + with ThreadPoolExecutor() as pool: + first = pool.submit(manager.cleanup_transient_files) + assert entered.wait(5) + second = pool.submit(manager.cleanup_transient_files) + try: + time.sleep(.1) + assert len(calls) == 1 + finally: + release.set() + first.result(timeout=5) + second.result(timeout=5) + assert len(calls) == 2 + + def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch): run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example") manager.store.claim_next("example") diff --git a/python-tools/tests/test_task_runtime.py b/python-tools/tests/test_task_runtime.py index c432a5e..7c1c4f6 100644 --- a/python-tools/tests/test_task_runtime.py +++ b/python-tools/tests/test_task_runtime.py @@ -57,7 +57,7 @@ def test_multiline_progress_has_per_step_eta_only(work,monkeypatch): clock[0]+=30 progress.update('Removing files',5,force=True) text=json.loads((work/'progress.json').read_text())['text'] - assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%' + assert text=='Removing files: 5/10 (50%), ETA 30 seconds\nLinking files: 0/20 (0%)' assert (work/'progress.json').stat().st_mode&0o777==0o600 @@ -78,4 +78,18 @@ def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch): assert 'ETA' not in json.loads((work/'progress.json').read_text())['text'] clock[0]+=60 progress.update('Items',40,force=True) - assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes' + assert json.loads((work/'progress.json').read_text())['text']=='Items: 40/50 (80%), ETA 10 minutes' + + +def test_stage_start_and_completion_bypass_progress_throttle(work, monkeypatch): + monkeypatch.setattr(runtime.time, 'monotonic', lambda: 100.) + progress = runtime.Progress({'Items': 2, 'Empty': 0}) + progress.update('Items', 0) + assert json.loads((work / 'progress.json').read_text())['text'] == ( + 'Items: 0/2 (0%)\nEmpty: waiting' + ) + progress.update('Items', 1) + progress.update('Items', 2) + assert json.loads((work / 'progress.json').read_text())['text'].startswith('Items: 2/2 (100%)') + progress.update('Empty', 0) + assert json.loads((work / 'progress.json').read_text())['text'].endswith('Empty: no items') diff --git a/python-tools/tests/test_web.py b/python-tools/tests/test_web.py index 43f26c1..afdfabc 100644 --- a/python-tools/tests/test_web.py +++ b/python-tools/tests/test_web.py @@ -1,6 +1,7 @@ from __future__ import annotations from io import BytesIO +import json import time from app.web import create_app @@ -254,7 +255,7 @@ print(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text()) assert executed.input_json == '{"mode":"apply"}' assert executed_id != run_id assert app.extensions["run_store"].get(run_id).input_json == '{"mode":"report"}' - assert '"mode":"report"' in client.get(f"/runs/{run_id}/output").get_json()["output"] + assert json.loads(client.get(f"/runs/{run_id}/output").get_json()["output"]) == {"mode": "report"} executed_detail = client.get(f"/runs/{executed_id}") assert b'href="/history/task/examples/migrate.py"' in executed_detail.data history = client.get("/history/task/examples/migrate.py")