Simplify task input handling and tighten runner coordination
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
3bfd199b9c
commit
1c713ff635
7 files changed
+130
-67
No files matched your search
+35
-56
@@ -50,6 +50,7 @@ class RunManager:
|
||||
self._processes: dict[int, subprocess.Popen[bytes]] = {}
|
||||
self._background_processes: set[subprocess.Popen[bytes]] = set()
|
||||
self._lock = threading.Lock()
|
||||
self._cleanup_lock = threading.Lock()
|
||||
self._closing = threading.Event()
|
||||
|
||||
def run_task(
|
||||
@@ -58,8 +59,6 @@ class RunManager:
|
||||
input_data: dict[str, str | int | list[str]] | None = None,
|
||||
uploads: Mapping[str, PendingUpload] | None = None,
|
||||
) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down; try again after it restarts.")
|
||||
sensitive_names = {field.name for field in task.inputs if field.sensitive}
|
||||
stored_input = self._redact_input(input_data or {}, sensitive_names)
|
||||
with self._lock:
|
||||
@@ -72,33 +71,28 @@ class RunManager:
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
|
||||
)
|
||||
if created:
|
||||
try:
|
||||
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
||||
execution_input = {**(input_data or {}), **uploaded_inputs}
|
||||
run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names))
|
||||
self._save_work_input(run, execution_input)
|
||||
except (OSError, ValueError) as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._discard_work_input(run)
|
||||
return self.store.finish(
|
||||
run.id, status="failed", message=f"Could not save task input: {error}",
|
||||
), True
|
||||
if created:
|
||||
try:
|
||||
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
||||
execution_input = {**(input_data or {}), **uploaded_inputs}
|
||||
stored_input = self._redact_input(execution_input, sensitive_names)
|
||||
run = self.store.set_input(run.id, stored_input)
|
||||
if sensitive_names:
|
||||
self._write_execution_input(run.id, execution_input)
|
||||
self._save_work_input(run, execution_input)
|
||||
except (OSError, ValueError) as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
status="failed",
|
||||
message=f"Could not save uploaded file: {error}",
|
||||
), True
|
||||
self._wake_worker(task.group_id)
|
||||
return run, created
|
||||
|
||||
def run_group(self, group: Group) -> tuple[Run, bool]:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
|
||||
resources = resources or (f"group:{group.id}",)
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
raise RuntimeError("The application is shutting down.")
|
||||
for active in self.store.active_runs():
|
||||
if active.target_kind == "group" and active.target_path == group.id:
|
||||
return active, False
|
||||
@@ -152,10 +146,10 @@ class RunManager:
|
||||
|
||||
def shutdown(self, timeout: float | None = None) -> None:
|
||||
self._closing.set()
|
||||
for event in self._wake_events.values():
|
||||
event.set()
|
||||
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
|
||||
with self._lock:
|
||||
for event in self._wake_events.values():
|
||||
event.set()
|
||||
processes = [*self._processes.values(), *self._background_processes]
|
||||
for process in processes:
|
||||
if process.poll() is None:
|
||||
@@ -199,7 +193,6 @@ class RunManager:
|
||||
completed = self.store.get(run.id)
|
||||
if completed is not None and completed.status == "cancelled":
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(completed)
|
||||
return completed
|
||||
|
||||
@@ -271,7 +264,10 @@ class RunManager:
|
||||
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
|
||||
) -> None:
|
||||
"""Remove transient run data and state for task groups removed long ago."""
|
||||
with self._cleanup_lock:
|
||||
self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds)
|
||||
|
||||
def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None:
|
||||
runs_root = self.settings.state_root / "runs"
|
||||
cutoff = time.time() - artifact_max_age_seconds
|
||||
if runs_root.is_dir():
|
||||
@@ -438,6 +434,8 @@ class RunManager:
|
||||
|
||||
def _wake_worker(self, group_id: str) -> None:
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return
|
||||
event = self._wake_events.setdefault(group_id, threading.Event())
|
||||
worker = self._workers.get(group_id)
|
||||
if worker is None or not worker.is_alive():
|
||||
@@ -454,15 +452,20 @@ class RunManager:
|
||||
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
|
||||
while not self._closing.is_set():
|
||||
event.clear()
|
||||
run = self.store.claim_next(group_id)
|
||||
# A queued run is claimable only after its private input and uploads are saved.
|
||||
with self._lock:
|
||||
if self._closing.is_set():
|
||||
return
|
||||
run = self.store.claim_next(group_id)
|
||||
if run is None:
|
||||
event.wait()
|
||||
continue
|
||||
try:
|
||||
self._execute_root_run(run)
|
||||
finally:
|
||||
for waiting in self._wake_events.values():
|
||||
waiting.set()
|
||||
with self._lock:
|
||||
for waiting in self._wake_events.values():
|
||||
waiting.set()
|
||||
|
||||
def _execute_root_run(self, run: Run) -> None:
|
||||
if run.target_kind == "task":
|
||||
@@ -529,23 +532,17 @@ class RunManager:
|
||||
|
||||
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
input_path = self._execution_input_path(run.id)
|
||||
artifact_root = self._artifact_root(run.id)
|
||||
artifact_root.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
work = self._work_root(run)
|
||||
saved_input = work / "input.json"
|
||||
if run.recovery_of_run_id is not None:
|
||||
input_path = saved_input
|
||||
elif not input_path.exists():
|
||||
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
|
||||
input_path.write_text(run.input_json + "\n", encoding="utf-8")
|
||||
input_path.chmod(0o600)
|
||||
if not saved_input.exists():
|
||||
self._save_work_input(run, json.loads(input_path.read_text()))
|
||||
input_path = work / "input.json"
|
||||
if not input_path.is_file():
|
||||
if run.recovery_of_run_id is not None:
|
||||
raise FileNotFoundError("The original recovery input is no longer available")
|
||||
self._save_work_input(run, json.loads(run.input_json))
|
||||
except OSError as error:
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
self._discard_work_input(run)
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
return self.store.finish(
|
||||
@@ -595,8 +592,6 @@ class RunManager:
|
||||
owned = self._processes.pop(run.id, None)
|
||||
if owned is not None:
|
||||
kill_remaining_group(owned)
|
||||
self._delete_execution_input(run.id)
|
||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
@@ -659,22 +654,6 @@ class RunManager:
|
||||
def _artifact_root(self, run_id: int) -> Path:
|
||||
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
|
||||
|
||||
def _execution_input_path(self, run_id: int) -> Path:
|
||||
return self.settings.state_root / "runs" / f"{run_id}.execution-input.json"
|
||||
|
||||
def _write_execution_input(
|
||||
self,
|
||||
run_id: int,
|
||||
input_data: Mapping[str, str | int | list[str]],
|
||||
) -> None:
|
||||
path = self._execution_input_path(run_id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(input_data, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
|
||||
path.chmod(0o600)
|
||||
|
||||
def _delete_execution_input(self, run_id: int) -> None:
|
||||
self._execution_input_path(run_id).unlink(missing_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def _redact_input(
|
||||
input_data: Mapping[str, str | int | list[str]],
|
||||
|
||||
Reference in new issue
Block a user