Protect task diagnostics and stream incremental run output
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 47s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m24s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 38s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 43s

This commit is contained in:
ajp_anton committed 2026-10-26 23:39:50 +00:00
1 parent 0aa67439fd
commit 55bbbe2fd7
8 files changed
+319 -51

No files matched your search

+43 -17
View File
@@ -10,6 +10,7 @@ import subprocess
import threading
import time
from collections.abc import Mapping
from datetime import datetime
from pathlib import Path
from .config import Settings
@@ -127,31 +128,54 @@ class RunManager:
runs_root = self.settings.state_root / "runs"
cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir():
lifetimes = self.store.run_lifetimes()
for path in runs_root.iterdir():
if not path.is_dir():
continue
run_id_text, separator, kind = path.name.partition(".")
if not separator or not run_id_text.isdigit() or kind not in {"uploads", "artifacts"}:
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
continue
run = self.store.get(int(run_id_text))
if kind == "uploads" and (run is None or run.status not in {"queued", "running"}):
run_id = int(run_id_text)
if run_id not in lifetimes:
# A worker may have created this run after the bulk snapshot.
run = self.store.get(run_id)
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
status, finished_at = lifetimes[run_id]
if status in {"queued", "running"}:
continue
if kind in {"input.json", "execution-input.json"} and path.is_file():
path.unlink(missing_ok=True)
elif kind == "uploads" and path.is_dir():
shutil.rmtree(path, ignore_errors=True)
elif kind == "artifacts":
elif kind == "artifacts" and path.is_dir():
if status in {"failed", "interrupted", "cancelled"}:
self._keep_diagnostic_reports(path)
try:
expired = path.stat().st_mtime < cutoff
completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime
except FileNotFoundError:
continue
if expired:
if completed_at < cutoff:
shutil.rmtree(path, ignore_errors=True)
for path in runs_root.glob("*.execution-input.json"):
run_id_text = path.name.removesuffix(".execution-input.json")
if not run_id_text.isdigit():
continue
run = self.store.get(int(run_id_text))
if run is None or run.status not in {"queued", "running"}:
path.unlink(missing_ok=True)
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
@staticmethod
def _keep_diagnostic_reports(root: Path) -> None:
try:
paths = list(root.iterdir())
except FileNotFoundError:
return
for path in paths:
if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json":
try:
json.loads(path.read_text(encoding="utf-8"))
continue
except FileNotFoundError:
continue
except ValueError:
pass
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink(missing_ok=True)
def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None:
"""Delete state only after a group remains absent for a grace period."""
@@ -318,6 +342,7 @@ class RunManager:
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
self.store.set_log_path(run.id, str(log_path))
try:
with log_path.open("wb") as log_file:
@@ -347,11 +372,12 @@ class RunManager:
self._processes.pop(run.id, None)
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
shutil.rmtree(artifact_root, ignore_errors=True)
self._keep_diagnostic_reports(artifact_root)
return self.store.finish(
run.id,
status="cancelled",
@@ -365,7 +391,7 @@ class RunManager:
log_path=str(log_path),
)
if completed.status != "succeeded":
shutil.rmtree(artifact_root, ignore_errors=True)
self._keep_diagnostic_reports(artifact_root)
else:
try:
artifact_root.rmdir()