Simplify task input handling and tighten runner coordination
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-11-01 16:44:11 +00:00
1 parent 3bfd199b9c
commit 1c713ff635
7 files changed
+130 -67

No files matched your search

+35 -56
View File
@@ -50,6 +50,7 @@ class RunManager:
self._processes: dict[int, subprocess.Popen[bytes]] = {}
self._background_processes: set[subprocess.Popen[bytes]] = set()
self._lock = threading.Lock()
self._cleanup_lock = threading.Lock()
self._closing = threading.Event()
def run_task(
@@ -58,8 +59,6 @@ class RunManager:
input_data: dict[str, str | int | list[str]] | None = None,
uploads: Mapping[str, PendingUpload] | None = None,
) -> tuple[Run, bool]:
if self._closing.is_set():
raise RuntimeError("The application is shutting down; try again after it restarts.")
sensitive_names = {field.name for field in task.inputs if field.sensitive}
stored_input = self._redact_input(input_data or {}, sensitive_names)
with self._lock:
@@ -72,33 +71,28 @@ class RunManager:
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
)
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
execution_input = {**(input_data or {}), **uploaded_inputs}
run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names))
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id, status="failed", message=f"Could not save task input: {error}",
), True
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
execution_input = {**(input_data or {}), **uploaded_inputs}
stored_input = self._redact_input(execution_input, sensitive_names)
run = self.store.set_input(run.id, stored_input)
if sensitive_names:
self._write_execution_input(run.id, execution_input)
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id,
status="failed",
message=f"Could not save uploaded file: {error}",
), True
self._wake_worker(task.group_id)
return run, created
def run_group(self, group: Group) -> tuple[Run, bool]:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
resources = resources or (f"group:{group.id}",)
with self._lock:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
for active in self.store.active_runs():
if active.target_kind == "group" and active.target_path == group.id:
return active, False
@@ -152,10 +146,10 @@ class RunManager:
def shutdown(self, timeout: float | None = None) -> None:
self._closing.set()
for event in self._wake_events.values():
event.set()
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
with self._lock:
for event in self._wake_events.values():
event.set()
processes = [*self._processes.values(), *self._background_processes]
for process in processes:
if process.poll() is None:
@@ -199,7 +193,6 @@ class RunManager:
completed = self.store.get(run.id)
if completed is not None and completed.status == "cancelled":
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(completed)
return completed
@@ -271,7 +264,10 @@ class RunManager:
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
) -> None:
"""Remove transient run data and state for task groups removed long ago."""
with self._cleanup_lock:
self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds)
def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None:
runs_root = self.settings.state_root / "runs"
cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir():
@@ -438,6 +434,8 @@ class RunManager:
def _wake_worker(self, group_id: str) -> None:
with self._lock:
if self._closing.is_set():
return
event = self._wake_events.setdefault(group_id, threading.Event())
worker = self._workers.get(group_id)
if worker is None or not worker.is_alive():
@@ -454,15 +452,20 @@ class RunManager:
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
while not self._closing.is_set():
event.clear()
run = self.store.claim_next(group_id)
# A queued run is claimable only after its private input and uploads are saved.
with self._lock:
if self._closing.is_set():
return
run = self.store.claim_next(group_id)
if run is None:
event.wait()
continue
try:
self._execute_root_run(run)
finally:
for waiting in self._wake_events.values():
waiting.set()
with self._lock:
for waiting in self._wake_events.values():
waiting.set()
def _execute_root_run(self, run: Run) -> None:
if run.target_kind == "task":
@@ -529,23 +532,17 @@ class RunManager:
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
input_path = self._execution_input_path(run.id)
artifact_root = self._artifact_root(run.id)
artifact_root.mkdir(parents=True, exist_ok=True)
try:
work = self._work_root(run)
saved_input = work / "input.json"
if run.recovery_of_run_id is not None:
input_path = saved_input
elif not input_path.exists():
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
input_path.write_text(run.input_json + "\n", encoding="utf-8")
input_path.chmod(0o600)
if not saved_input.exists():
self._save_work_input(run, json.loads(input_path.read_text()))
input_path = work / "input.json"
if not input_path.is_file():
if run.recovery_of_run_id is not None:
raise FileNotFoundError("The original recovery input is no longer available")
self._save_work_input(run, json.loads(run.input_json))
except OSError as error:
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
@@ -595,8 +592,6 @@ class RunManager:
owned = self._processes.pop(run.id, None)
if owned is not None:
kill_remaining_group(owned)
self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
@@ -659,22 +654,6 @@ class RunManager:
def _artifact_root(self, run_id: int) -> Path:
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
def _execution_input_path(self, run_id: int) -> Path:
return self.settings.state_root / "runs" / f"{run_id}.execution-input.json"
def _write_execution_input(
self,
run_id: int,
input_data: Mapping[str, str | int | list[str]],
) -> None:
path = self._execution_input_path(run_id)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(input_data, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
path.chmod(0o600)
def _delete_execution_input(self, run_id: int) -> None:
self._execution_input_path(run_id).unlink(missing_ok=True)
@staticmethod
def _redact_input(
input_data: Mapping[str, str | int | list[str]],