Simplify task input handling and tighten runner coordination
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
3bfd199b9c
commit
1c713ff635
7 files changed
+130
-67
No files matched your search
@@ -103,7 +103,7 @@ Every input has `name`, `label`, and `type`; `required` defaults to `false`.
|
||||
|
||||
Set `sensitive` to `true` on text or file inputs that must not appear in run
|
||||
history. Text is masked, the database records `[redacted]`, and the original
|
||||
value is available only through a private execution-input file. Interrupted
|
||||
value is available only through the run's private work-directory input file. Interrupted
|
||||
recoverable tasks retain that input and their uploads until recovery finishes.
|
||||
|
||||
For review-first tasks, use `execution`. It produces one split button: click
|
||||
@@ -219,13 +219,14 @@ For simple item-count percentages and per-step estimates:
|
||||
|
||||
```python
|
||||
tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20})
|
||||
tracker.update("Scanning files", 0, force=True)
|
||||
tracker.update("Scanning files", 0)
|
||||
# After processing five files:
|
||||
tracker.update("Scanning files", 5)
|
||||
```
|
||||
|
||||
Each step has its own percentage and an approximate ETA based only on processed
|
||||
items and elapsed time. Pending steps have no ETA. Tasks may instead publish their
|
||||
Each step shows completed/total items, a percentage and an approximate ETA based
|
||||
on items processed since starting or resuming. Stage starts and completion are
|
||||
published immediately. Pending steps have no ETA. Tasks may instead publish their
|
||||
own text when totals are unknown; no global estimate is required.
|
||||
|
||||
A group may also include a private `task-config.json` for stable local details
|
||||
|
||||
+35
-56
@@ -50,6 +50,7 @@ class RunManager:
|
||||
self._processes: dict[int, subprocess.Popen[bytes]] = {}
|
||||
self._background_processes: set[subprocess.Popen[bytes]] = set()
|
||||
self._lock = threading.Lock()
|
||||
self._cleanup_lock = threading.Lock()
|
||||
self._closing = threading.Event()
|
||||
|
||||
def run_task(
|
||||
@@ -58,8 +59,6 @@ class RunManager:
|
||||
input_data: dict[str, str | int | list[str]] | None = None,
|
||||
uploads: Mapping[str, PendingUpload] | None = None,
|
||||
) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down; try again after it restarts.")
|
||||
sensitive_names = {field.name for field in task.inputs if field.sensitive}
|
||||
stored_input = self._redact_input(input_data or {}, sensitive_names)
|
||||
with self._lock:
|
||||
@@ -72,33 +71,28 @@ class RunManager:
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
|
||||
)
|
||||
if created:
|
||||
try:
|
||||
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
||||
execution_input = {**(input_data or {}), **uploaded_inputs}
|
||||
run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names))
|
||||
self._save_work_input(run, execution_input)
|
||||
except (OSError, ValueError) as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._discard_work_input(run)
|
||||
return self.store.finish(
|
||||
run.id, status="failed", message=f"Could not save task input: {error}",
|
||||
), True
|
||||
if created:
|
||||
try:
|
||||
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
||||
execution_input = {**(input_data or {}), **uploaded_inputs}
|
||||
stored_input = self._redact_input(execution_input, sensitive_names)
|
||||
run = self.store.set_input(run.id, stored_input)
|
||||
if sensitive_names:
|
||||
self._write_execution_input(run.id, execution_input)
|
||||
self._save_work_input(run, execution_input)
|
||||
except (OSError, ValueError) as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
status="failed",
|
||||
message=f"Could not save uploaded file: {error}",
|
||||
), True
|
||||
self._wake_worker(task.group_id)
|
||||
return run, created
|
||||
|
||||
def run_group(self, group: Group) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
|
||||
resources = resources or (f"group:{group.id}",)
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
for active in self.store.active_runs():
|
||||
if active.target_kind == "group" and active.target_path == group.id:
|
||||
return active, False
|
||||
@@ -152,10 +146,10 @@ class RunManager:
|
||||
|
||||
def shutdown(self, timeout: float | None = None) -> None:
|
||||
self._closing.set()
|
||||
for event in self._wake_events.values():
|
||||
event.set()
|
||||
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
|
||||
with self._lock:
|
||||
for event in self._wake_events.values():
|
||||
event.set()
|
||||
processes = [*self._processes.values(), *self._background_processes]
|
||||
for process in processes:
|
||||
if process.poll() is None:
|
||||
@@ -199,7 +193,6 @@ class RunManager:
|
||||
completed = self.store.get(run.id)
|
||||
if completed is not None and completed.status == "cancelled":
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(completed)
|
||||
return completed
|
||||
|
||||
@@ -271,7 +264,10 @@ class RunManager:
|
||||
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
|
||||
) -> None:
|
||||
"""Remove transient run data and state for task groups removed long ago."""
|
||||
with self._cleanup_lock:
|
||||
self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds)
|
||||
|
||||
def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None:
|
||||
runs_root = self.settings.state_root / "runs"
|
||||
cutoff = time.time() - artifact_max_age_seconds
|
||||
if runs_root.is_dir():
|
||||
@@ -438,6 +434,8 @@ class RunManager:
|
||||
|
||||
def _wake_worker(self, group_id: str) -> None:
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return
|
||||
event = self._wake_events.setdefault(group_id, threading.Event())
|
||||
worker = self._workers.get(group_id)
|
||||
if worker is None or not worker.is_alive():
|
||||
@@ -454,15 +452,20 @@ class RunManager:
|
||||
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
|
||||
while not self._closing.is_set():
|
||||
event.clear()
|
||||
run = self.store.claim_next(group_id)
|
||||
# A queued run is claimable only after its private input and uploads are saved.
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return
|
||||
run = self.store.claim_next(group_id)
|
||||
if run is None:
|
||||
event.wait()
|
||||
continue
|
||||
try:
|
||||
self._execute_root_run(run)
|
||||
finally:
|
||||
for waiting in self._wake_events.values():
|
||||
waiting.set()
|
||||
with self._lock:
|
||||
for waiting in self._wake_events.values():
|
||||
waiting.set()
|
||||
|
||||
def _execute_root_run(self, run: Run) -> None:
|
||||
if run.target_kind == "task":
|
||||
@@ -529,23 +532,17 @@ class RunManager:
|
||||
|
||||
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
input_path = self._execution_input_path(run.id)
|
||||
artifact_root = self._artifact_root(run.id)
|
||||
artifact_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
work = self._work_root(run)
|
||||
saved_input = work / "input.json"
|
||||
if run.recovery_of_run_id is not None:
|
||||
input_path = saved_input
|
||||
elif not input_path.exists():
|
||||
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
|
||||
input_path.write_text(run.input_json + "\n", encoding="utf-8")
|
||||
input_path.chmod(0o600)
|
||||
if not saved_input.exists():
|
||||
self._save_work_input(run, json.loads(input_path.read_text()))
|
||||
input_path = work / "input.json"
|
||||
if not input_path.is_file():
|
||||
if run.recovery_of_run_id is not None:
|
||||
raise FileNotFoundError("The original recovery input is no longer available")
|
||||
self._save_work_input(run, json.loads(run.input_json))
|
||||
except OSError as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
return self.store.finish(
|
||||
@@ -595,8 +592,6 @@ class RunManager:
|
||||
owned = self._processes.pop(run.id, None)
|
||||
if owned is not None:
|
||||
kill_remaining_group(owned)
|
||||
self._delete_execution_input(run.id)
|
||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
@@ -659,22 +654,6 @@ class RunManager:
|
||||
def _artifact_root(self, run_id: int) -> Path:
|
||||
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
|
||||
|
||||
def _execution_input_path(self, run_id: int) -> Path:
|
||||
return self.settings.state_root / "runs" / f"{run_id}.execution-input.json"
|
||||
|
||||
def _write_execution_input(
|
||||
self,
|
||||
run_id: int,
|
||||
input_data: Mapping[str, str | int | list[str]],
|
||||
) -> None:
|
||||
path = self._execution_input_path(run_id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(input_data, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
|
||||
def _delete_execution_input(self, run_id: int) -> None:
|
||||
self._execution_input_path(run_id).unlink(missing_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def _redact_input(
|
||||
input_data: Mapping[str, str | int | list[str]],
|
||||
|
||||
@@ -144,18 +144,21 @@ class Progress:
|
||||
self.baseline: dict[str, int] = {}
|
||||
|
||||
def update(self, label: str, done: int, *, force: bool = False) -> None:
|
||||
force = force or label not in self.started or done >= self.totals[label]
|
||||
self.started.setdefault(label, time.monotonic())
|
||||
self.baseline.setdefault(label, done)
|
||||
self.done[label] = done
|
||||
lines = []
|
||||
for name, total in self.totals.items():
|
||||
count = self.done[name]
|
||||
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
|
||||
line = f"{name}: {percent}%"
|
||||
if total:
|
||||
line = f"{name}: {count}/{total} ({min(100, int(count * 100 / total))}%)"
|
||||
else:
|
||||
line = f"{name}: {'no items' if name in self.started else 'waiting'}"
|
||||
processed = count - self.baseline.get(name, 0)
|
||||
if processed > 0 and count < total and name in self.started:
|
||||
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
|
||||
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
|
||||
line += f", ETA {estimate}"
|
||||
value, unit = (seconds, "second") if seconds < 60 else (round(seconds / 60), "minute")
|
||||
line += f", ETA {value} {unit}{'' if value == 1 else 's'}"
|
||||
lines.append(line)
|
||||
progress("\n".join(lines), force=force)
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import replace
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
@@ -379,3 +381,38 @@ def test_upload_is_removed_when_task_process_cannot_start(manager: RunManager) -
|
||||
assert created is True
|
||||
assert wait_for(broken_manager, run.id, {"failed"}) == "failed"
|
||||
assert not (broken_manager.settings.state_root / "runs" / f"{run.id}.uploads").exists()
|
||||
|
||||
|
||||
def test_idle_worker_cannot_start_before_uploaded_input_is_ready(manager, monkeypatch):
|
||||
write_task(manager.settings.task_root, "imports", "read.py", '''
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||
assert Path(values["document"]).read_text() == "uploaded content"
|
||||
''')
|
||||
task = manager.catalog.task("imports/read.py")
|
||||
manager._wake_worker(task.group_id)
|
||||
saving = threading.Event()
|
||||
release = threading.Event()
|
||||
save_uploads = manager._save_uploads
|
||||
|
||||
def slow_upload(run_id, uploads):
|
||||
saving.set()
|
||||
assert release.wait(5)
|
||||
return save_uploads(run_id, uploads)
|
||||
|
||||
monkeypatch.setattr(manager, '_save_uploads', slow_upload)
|
||||
upload = PendingUpload('document', '.txt', 1024, BytesIO(b'uploaded content'))
|
||||
with ThreadPoolExecutor() as pool:
|
||||
submitted = pool.submit(manager.run_task, task, uploads={'document': upload})
|
||||
try:
|
||||
assert saving.wait(5)
|
||||
run = manager.store.active_runs()[0]
|
||||
manager._wake_events[task.group_id].set()
|
||||
time.sleep(.1)
|
||||
assert manager.store.get(run.id).status == 'queued'
|
||||
finally:
|
||||
release.set()
|
||||
run, created = submitted.result(timeout=5)
|
||||
assert created
|
||||
assert wait_for(manager, run.id, {'succeeded', 'failed'}) == 'succeeded'
|
||||
@@ -1,6 +1,8 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from app.web import create_app
|
||||
@@ -8,6 +10,32 @@ from .conftest import write_task
|
||||
from .test_runner import wait_for
|
||||
|
||||
|
||||
def test_concurrent_cleanup_passes_do_not_overlap(manager, monkeypatch):
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
calls = []
|
||||
|
||||
def cleanup(*args):
|
||||
calls.append(args)
|
||||
if len(calls) == 1:
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
|
||||
monkeypatch.setattr(manager, '_cleanup_transient_files', cleanup)
|
||||
with ThreadPoolExecutor() as pool:
|
||||
first = pool.submit(manager.cleanup_transient_files)
|
||||
assert entered.wait(5)
|
||||
second = pool.submit(manager.cleanup_transient_files)
|
||||
try:
|
||||
time.sleep(.1)
|
||||
assert len(calls) == 1
|
||||
finally:
|
||||
release.set()
|
||||
first.result(timeout=5)
|
||||
second.result(timeout=5)
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
|
||||
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
|
||||
manager.store.claim_next("example")
|
||||
|
||||
@@ -57,7 +57,7 @@ def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
|
||||
clock[0]+=30
|
||||
progress.update('Removing files',5,force=True)
|
||||
text=json.loads((work/'progress.json').read_text())['text']
|
||||
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
|
||||
assert text=='Removing files: 5/10 (50%), ETA 30 seconds\nLinking files: 0/20 (0%)'
|
||||
assert (work/'progress.json').stat().st_mode&0o777==0o600
|
||||
|
||||
|
||||
@@ -78,4 +78,18 @@ def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
|
||||
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
|
||||
clock[0]+=60
|
||||
progress.update('Items',40,force=True)
|
||||
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
|
||||
assert json.loads((work/'progress.json').read_text())['text']=='Items: 40/50 (80%), ETA 10 minutes'
|
||||
|
||||
|
||||
def test_stage_start_and_completion_bypass_progress_throttle(work, monkeypatch):
|
||||
monkeypatch.setattr(runtime.time, 'monotonic', lambda: 100.)
|
||||
progress = runtime.Progress({'Items': 2, 'Empty': 0})
|
||||
progress.update('Items', 0)
|
||||
assert json.loads((work / 'progress.json').read_text())['text'] == (
|
||||
'Items: 0/2 (0%)\nEmpty: waiting'
|
||||
)
|
||||
progress.update('Items', 1)
|
||||
progress.update('Items', 2)
|
||||
assert json.loads((work / 'progress.json').read_text())['text'].startswith('Items: 2/2 (100%)')
|
||||
progress.update('Empty', 0)
|
||||
assert json.loads((work / 'progress.json').read_text())['text'].endswith('Empty: no items')
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
import json
|
||||
import time
|
||||
|
||||
from app.web import create_app
|
||||
@@ -254,7 +255,7 @@ print(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||
assert executed.input_json == '{"mode":"apply"}'
|
||||
assert executed_id != run_id
|
||||
assert app.extensions["run_store"].get(run_id).input_json == '{"mode":"report"}'
|
||||
assert '"mode":"report"' in client.get(f"/runs/{run_id}/output").get_json()["output"]
|
||||
assert json.loads(client.get(f"/runs/{run_id}/output").get_json()["output"]) == {"mode": "report"}
|
||||
executed_detail = client.get(f"/runs/{executed_id}")
|
||||
assert b'href="/history/task/examples/migrate.py"' in executed_detail.data
|
||||
history = client.get("/history/task/examples/migrate.py")
|
||||
|
||||
Reference in new issue
Block a user