Simplify task input handling and tighten runner coordination
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-11-01 16:44:11 +00:00
1 parent 3bfd199b9c
commit 1c713ff635
7 files changed
+130 -67

No files matched your search

+5 -4
View File
@@ -103,7 +103,7 @@ Every input has `name`, `label`, and `type`; `required` defaults to `false`.
Set `sensitive` to `true` on text or file inputs that must not appear in run
history. Text is masked, the database records `[redacted]`, and the original
value is available only through a private execution-input file. Interrupted
value is available only through the run's private work-directory input file. Interrupted
recoverable tasks retain that input and their uploads until recovery finishes.
For review-first tasks, use `execution`. It produces one split button: click
@@ -219,13 +219,14 @@ For simple item-count percentages and per-step estimates:
```python
tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20})
tracker.update("Scanning files", 0, force=True)
tracker.update("Scanning files", 0)
# After processing five files:
tracker.update("Scanning files", 5)
```
Each step has its own percentage and an approximate ETA based only on processed
items and elapsed time. Pending steps have no ETA. Tasks may instead publish their
Each step shows completed/total items, a percentage and an approximate ETA based
on items processed since starting or resuming. Stage starts and completion are
published immediately. Pending steps have no ETA. Tasks may instead publish their
own text when totals are unknown; no global estimate is required.
A group may also include a private `task-config.json` for stable local details
+35 -56
View File
@@ -50,6 +50,7 @@ class RunManager:
self._processes: dict[int, subprocess.Popen[bytes]] = {}
self._background_processes: set[subprocess.Popen[bytes]] = set()
self._lock = threading.Lock()
self._cleanup_lock = threading.Lock()
self._closing = threading.Event()
def run_task(
@@ -58,8 +59,6 @@ class RunManager:
input_data: dict[str, str | int | list[str]] | None = None,
uploads: Mapping[str, PendingUpload] | None = None,
) -> tuple[Run, bool]:
if self._closing.is_set():
raise RuntimeError("The application is shutting down; try again after it restarts.")
sensitive_names = {field.name for field in task.inputs if field.sensitive}
stored_input = self._redact_input(input_data or {}, sensitive_names)
with self._lock:
@@ -72,33 +71,28 @@ class RunManager:
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
)
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
execution_input = {**(input_data or {}), **uploaded_inputs}
run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names))
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id, status="failed", message=f"Could not save task input: {error}",
), True
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
execution_input = {**(input_data or {}), **uploaded_inputs}
stored_input = self._redact_input(execution_input, sensitive_names)
run = self.store.set_input(run.id, stored_input)
if sensitive_names:
self._write_execution_input(run.id, execution_input)
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id,
status="failed",
message=f"Could not save uploaded file: {error}",
), True
self._wake_worker(task.group_id)
return run, created
def run_group(self, group: Group) -> tuple[Run, bool]:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
resources = resources or (f"group:{group.id}",)
with self._lock:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
for active in self.store.active_runs():
if active.target_kind == "group" and active.target_path == group.id:
return active, False
@@ -152,10 +146,10 @@ class RunManager:
def shutdown(self, timeout: float | None = None) -> None:
self._closing.set()
for event in self._wake_events.values():
event.set()
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
with self._lock:
for event in self._wake_events.values():
event.set()
processes = [*self._processes.values(), *self._background_processes]
for process in processes:
if process.poll() is None:
@@ -199,7 +193,6 @@ class RunManager:
completed = self.store.get(run.id)
if completed is not None and completed.status == "cancelled":
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(completed)
return completed
@@ -271,7 +264,10 @@ class RunManager:
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
) -> None:
"""Remove transient run data and state for task groups removed long ago."""
with self._cleanup_lock:
self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds)
def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None:
runs_root = self.settings.state_root / "runs"
cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir():
@@ -438,6 +434,8 @@ class RunManager:
def _wake_worker(self, group_id: str) -> None:
with self._lock:
if self._closing.is_set():
return
event = self._wake_events.setdefault(group_id, threading.Event())
worker = self._workers.get(group_id)
if worker is None or not worker.is_alive():
@@ -454,15 +452,20 @@ class RunManager:
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
while not self._closing.is_set():
event.clear()
run = self.store.claim_next(group_id)
# A queued run is claimable only after its private input and uploads are saved.
with self._lock:
if self._closing.is_set():
return
run = self.store.claim_next(group_id)
if run is None:
event.wait()
continue
try:
self._execute_root_run(run)
finally:
for waiting in self._wake_events.values():
waiting.set()
with self._lock:
for waiting in self._wake_events.values():
waiting.set()
def _execute_root_run(self, run: Run) -> None:
if run.target_kind == "task":
@@ -529,23 +532,17 @@ class RunManager:
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
input_path = self._execution_input_path(run.id)
artifact_root = self._artifact_root(run.id)
artifact_root.mkdir(parents=True, exist_ok=True)
try:
work = self._work_root(run)
saved_input = work / "input.json"
if run.recovery_of_run_id is not None:
input_path = saved_input
elif not input_path.exists():
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
input_path.write_text(run.input_json + "\n", encoding="utf-8")
input_path.chmod(0o600)
if not saved_input.exists():
self._save_work_input(run, json.loads(input_path.read_text()))
input_path = work / "input.json"
if not input_path.is_file():
if run.recovery_of_run_id is not None:
raise FileNotFoundError("The original recovery input is no longer available")
self._save_work_input(run, json.loads(run.input_json))
except OSError as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
@@ -595,8 +592,6 @@ class RunManager:
owned = self._processes.pop(run.id, None)
if owned is not None:
kill_remaining_group(owned)
self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
@@ -659,22 +654,6 @@ class RunManager:
def _artifact_root(self, run_id: int) -> Path:
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
def _execution_input_path(self, run_id: int) -> Path:
return self.settings.state_root / "runs" / f"{run_id}.execution-input.json"
def _write_execution_input(
self,
run_id: int,
input_data: Mapping[str, str | int | list[str]],
) -> None:
path = self._execution_input_path(run_id)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(input_data, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
path.chmod(0o600)
def _delete_execution_input(self, run_id: int) -> None:
self._execution_input_path(run_id).unlink(missing_ok=True)
@staticmethod
def _redact_input(
input_data: Mapping[str, str | int | list[str]],
+7 -4
View File
@@ -144,18 +144,21 @@ class Progress:
self.baseline: dict[str, int] = {}
def update(self, label: str, done: int, *, force: bool = False) -> None:
force = force or label not in self.started or done >= self.totals[label]
self.started.setdefault(label, time.monotonic())
self.baseline.setdefault(label, done)
self.done[label] = done
lines = []
for name, total in self.totals.items():
count = self.done[name]
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
line = f"{name}: {percent}%"
if total:
line = f"{name}: {count}/{total} ({min(100, int(count * 100 / total))}%)"
else:
line = f"{name}: {'no items' if name in self.started else 'waiting'}"
processed = count - self.baseline.get(name, 0)
if processed > 0 and count < total and name in self.started:
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
line += f", ETA {estimate}"
value, unit = (seconds, "second") if seconds < 60 else (round(seconds / 60), "minute")
line += f", ETA {value} {unit}{'' if value == 1 else 's'}"
lines.append(line)
progress("\n".join(lines), force=force)
+37
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import time
import threading
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from io import BytesIO
from pathlib import Path
@@ -379,3 +381,38 @@ def test_upload_is_removed_when_task_process_cannot_start(manager: RunManager) -
assert created is True
assert wait_for(broken_manager, run.id, {"failed"}) == "failed"
assert not (broken_manager.settings.state_root / "runs" / f"{run.id}.uploads").exists()
def test_idle_worker_cannot_start_before_uploaded_input_is_ready(manager, monkeypatch):
write_task(manager.settings.task_root, "imports", "read.py", '''
import json, os
from pathlib import Path
values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
assert Path(values["document"]).read_text() == "uploaded content"
''')
task = manager.catalog.task("imports/read.py")
manager._wake_worker(task.group_id)
saving = threading.Event()
release = threading.Event()
save_uploads = manager._save_uploads
def slow_upload(run_id, uploads):
saving.set()
assert release.wait(5)
return save_uploads(run_id, uploads)
monkeypatch.setattr(manager, '_save_uploads', slow_upload)
upload = PendingUpload('document', '.txt', 1024, BytesIO(b'uploaded content'))
with ThreadPoolExecutor() as pool:
submitted = pool.submit(manager.run_task, task, uploads={'document': upload})
try:
assert saving.wait(5)
run = manager.store.active_runs()[0]
manager._wake_events[task.group_id].set()
time.sleep(.1)
assert manager.store.get(run.id).status == 'queued'
finally:
release.set()
run, created = submitted.result(timeout=5)
assert created
assert wait_for(manager, run.id, {'succeeded', 'failed'}) == 'succeeded'
@@ -1,6 +1,8 @@
import json
import os
import time
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from app.web import create_app
@@ -8,6 +10,32 @@ from .conftest import write_task
from .test_runner import wait_for
def test_concurrent_cleanup_passes_do_not_overlap(manager, monkeypatch):
entered = threading.Event()
release = threading.Event()
calls = []
def cleanup(*args):
calls.append(args)
if len(calls) == 1:
entered.set()
assert release.wait(5)
monkeypatch.setattr(manager, '_cleanup_transient_files', cleanup)
with ThreadPoolExecutor() as pool:
first = pool.submit(manager.cleanup_transient_files)
assert entered.wait(5)
second = pool.submit(manager.cleanup_transient_files)
try:
time.sleep(.1)
assert len(calls) == 1
finally:
release.set()
first.result(timeout=5)
second.result(timeout=5)
assert len(calls) == 2
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
manager.store.claim_next("example")
+16 -2
View File
@@ -57,7 +57,7 @@ def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
clock[0]+=30
progress.update('Removing files',5,force=True)
text=json.loads((work/'progress.json').read_text())['text']
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
assert text=='Removing files: 5/10 (50%), ETA 30 seconds\nLinking files: 0/20 (0%)'
assert (work/'progress.json').stat().st_mode&0o777==0o600
@@ -78,4 +78,18 @@ def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
clock[0]+=60
progress.update('Items',40,force=True)
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
assert json.loads((work/'progress.json').read_text())['text']=='Items: 40/50 (80%), ETA 10 minutes'
def test_stage_start_and_completion_bypass_progress_throttle(work, monkeypatch):
monkeypatch.setattr(runtime.time, 'monotonic', lambda: 100.)
progress = runtime.Progress({'Items': 2, 'Empty': 0})
progress.update('Items', 0)
assert json.loads((work / 'progress.json').read_text())['text'] == (
'Items: 0/2 (0%)\nEmpty: waiting'
)
progress.update('Items', 1)
progress.update('Items', 2)
assert json.loads((work / 'progress.json').read_text())['text'].startswith('Items: 2/2 (100%)')
progress.update('Empty', 0)
assert json.loads((work / 'progress.json').read_text())['text'].endswith('Empty: no items')
+2 -1
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from io import BytesIO
import json
import time
from app.web import create_app
@@ -254,7 +255,7 @@ print(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
assert executed.input_json == '{"mode":"apply"}'
assert executed_id != run_id
assert app.extensions["run_store"].get(run_id).input_json == '{"mode":"report"}'
assert '"mode":"report"' in client.get(f"/runs/{run_id}/output").get_json()["output"]
assert json.loads(client.get(f"/runs/{run_id}/output").get_json()["output"]) == {"mode": "report"}
executed_detail = client.get(f"/runs/{executed_id}")
assert b'href="/history/task/examples/migrate.py"' in executed_detail.data
history = client.get("/history/task/examples/migrate.py")