Files
docker-images/python-tools/app/runner.py
T
ajp_anton 1c713ff635
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Simplify task input handling and tighten runner coordination
2026-11-01 16:44:11 +00:00

724 lines
31 KiB
Python

"""Per-group task queues and subprocess execution."""
from __future__ import annotations
import json
import os
import signal
import shutil
import subprocess
import threading
import time
from collections.abc import Mapping
from dataclasses import replace
from datetime import datetime
from pathlib import Path
from .config import Settings
from .discovery import BackgroundTask, Group, Task, TaskCatalog
from .inputs import PendingUpload
from .store import Run, RunStore
from .task_runtime import PAUSED_EXIT_CODE, atomic_json
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
class TaskConflictError(RuntimeError):
def __init__(self, run: Run):
self.run = run
super().__init__(f"Run {run.id} is using resources required by this task.")
def kill_remaining_group(process: subprocess.Popen) -> None:
# The leader can already be gone while a transfer subprocess is still alive.
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
class RunManager:
def __init__(self, settings: Settings, catalog: TaskCatalog, store: RunStore) -> None:
self.settings = settings
self.catalog = catalog
self.store = store
self._wake_events: dict[str, threading.Event] = {}
self._workers: dict[str, threading.Thread] = {}
self._processes: dict[int, subprocess.Popen[bytes]] = {}
self._background_processes: set[subprocess.Popen[bytes]] = set()
self._lock = threading.Lock()
self._cleanup_lock = threading.Lock()
self._closing = threading.Event()
def run_task(
self,
task: Task,
input_data: dict[str, str | int | list[str]] | None = None,
uploads: Mapping[str, PendingUpload] | None = None,
) -> tuple[Run, bool]:
sensitive_names = {field.name for field in task.inputs if field.sensitive}
stored_input = self._redact_input(input_data or {}, sensitive_names)
with self._lock:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
blocker = self.task_status(task)["blocked_by"]
if blocker:
raise TaskConflictError(blocker)
run, created = self.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
)
if created:
try:
uploaded_inputs = self._save_uploads(run.id, uploads or {})
execution_input = {**(input_data or {}), **uploaded_inputs}
run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names))
self._save_work_input(run, execution_input)
except (OSError, ValueError) as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
return self.store.finish(
run.id, status="failed", message=f"Could not save task input: {error}",
), True
if created:
self._wake_worker(task.group_id)
return run, created
def run_group(self, group: Group) -> tuple[Run, bool]:
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
resources = resources or (f"group:{group.id}",)
with self._lock:
if self._closing.is_set():
raise RuntimeError("The application is shutting down.")
for active in self.store.active_runs():
if active.target_kind == "group" and active.target_path == group.id:
return active, False
if active.parent_run_id is None and set(resources).intersection(json.loads(active.resources_json)):
raise TaskConflictError(active)
run, created = self.store.enqueue(
target_kind="group", target_path=group.id, queue_group=group.id, resources=resources,
)
if created:
self._wake_worker(group.id)
return run, created
def startup(self) -> None:
"""Recover only runs with the durable input/context written by this runner."""
self.store.mark_interrupted()
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
for old in self.store.interrupted_without_recovery():
work = self._work_root(old)
if not (work / "input.json").is_file():
continue
task = tasks.get(old.target_path)
if task is None:
continue
if old.started_at and not task.recoverable:
continue
run, created = self.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
input_data=json.loads(old.input_json), resources=task.resources,
recovery_of_run_id=old.id,
)
if created:
self._wake_worker(run.queue_group)
for run in self.store.active_runs():
if run.parent_run_id is None:
self._wake_worker(run.queue_group)
def task_status(self, task: Task, *, active_runs: list[Run] | None = None) -> dict:
active = active_runs if active_runs is not None else self.store.active_runs()
own = next((run for run in active if run.target_kind == "task" and run.target_path == task.id), None)
resources = set(task.resources or (f"group:{task.group_id}",))
blocker = next((run for run in active if run is not own and run.parent_run_id is None
and resources.intersection(json.loads(run.resources_json))), None)
return {"active_run": own, "blocked_by": blocker, "progress": self.progress(own) if own else ""}
def progress(self, run: Run) -> str:
try:
data = json.loads((self.settings.state_root / "runs" / f"{run.id}.progress.json").read_text())
except FileNotFoundError:
return ""
return str(data.get("text", ""))
def shutdown(self, timeout: float | None = None) -> None:
self._closing.set()
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
with self._lock:
for event in self._wake_events.values():
event.set()
processes = [*self._processes.values(), *self._background_processes]
for process in processes:
if process.poll() is None:
try:
os.kill(process.pid, signal.SIGUSR1)
except ProcessLookupError:
pass
while any(process.poll() is None for process in processes) and time.monotonic() < deadline:
time.sleep(.05)
for process in processes:
kill_remaining_group(process)
for process in processes:
process.wait(timeout=5)
for worker in self._workers.values():
worker.join(timeout=max(0, deadline - time.monotonic()))
def _work_root(self, run: Run) -> Path:
while run.recovery_of_run_id is not None:
previous = self.store.get(run.recovery_of_run_id)
if previous is None:
raise RuntimeError("The original recovery run no longer exists")
run = previous
return self.settings.state_root / "runs" / f"{run.id}.work"
def _save_work_input(self, run: Run, values: dict) -> None:
atomic_json(self._work_root(run) / "input.json", values)
def _discard_work_input(self, run: Run) -> None:
for name in ("input.json", "input.json.tmp"):
(self._work_root(run) / name).unlink(missing_ok=True)
def request_cancel(self, run_id: int) -> Run | None:
run = self.store.request_cancel(run_id)
if run is None:
return None
if run.parent_run_id is None and run.target_kind == "group":
child = self.store.active_child(run.id)
if child is not None:
self.request_cancel(child.id)
self._terminate_active_process(run.id)
completed = self.store.get(run.id)
if completed is not None and completed.status == "cancelled":
self._delete_uploads(run.id)
self._discard_work_input(completed)
return completed
def artifacts(self, run_id: int) -> list[str]:
root = self._artifact_root(run_id)
if not root.is_dir():
return []
return [
path.name
for path in sorted(root.iterdir(), key=lambda item: item.name)
if path.is_file() and not path.is_symlink()
]
def artifact_path(self, run_id: int, filename: str) -> Path | None:
path = self._artifact_root(run_id) / filename
if Path(filename).name != filename or path.is_symlink() or not path.is_file():
return None
return path
def delete_artifact(self, run_id: int, filename: str) -> None:
path = self.artifact_path(run_id, filename)
if path is None:
return
path.unlink(missing_ok=True)
try:
path.parent.rmdir()
except OSError:
pass
def review_upload_paths(self, run: Run, task: Task) -> dict[str, Path] | None:
"""Return reusable uploads, or None when a run cannot be executed again."""
if run.status != "succeeded" or run.target_kind != "task" or task.execution is None:
return None
if any(field.sensitive for field in task.inputs):
return None
try:
values = json.loads(run.input_json)
except json.JSONDecodeError:
return None
if not isinstance(values, dict) or values.get(task.execution.field.name) != task.execution.dry_run_value:
return None
root = self._upload_root(run.id)
paths = {}
for field in task.inputs:
if field.type != "file":
continue
value = values.get(field.name)
if not value and not field.required:
continue
if not isinstance(value, str) or not run.finished_at:
return None
if time.time() - datetime.fromisoformat(run.finished_at).timestamp() >= REVIEW_UPLOAD_RETENTION_SECONDS:
return None
path = Path(value)
if root.is_symlink() or path.parent != root or path.is_symlink() or not path.is_file():
return None
if path.name != field.name + path.suffix or (field.accept and path.suffix.lower() not in field.accept):
return None
paths[field.name] = path
return paths
def discard_review_uploads(self, run_id: int) -> None:
self._delete_uploads(run_id)
def cleanup_transient_files(
self,
*,
artifact_max_age_seconds: int = 86_400,
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
) -> None:
"""Remove transient run data and state for task groups removed long ago."""
with self._cleanup_lock:
self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds)
def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None:
runs_root = self.settings.state_root / "runs"
cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir():
lifetimes = self.store.run_lifetimes()
protected = self._recovery_context_ids()
for path in runs_root.iterdir():
run_id_text, separator, kind = path.name.partition(".")
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json", "work", "progress.json"} or not run_id_text.isdigit() or path.is_symlink():
continue
run_id = int(run_id_text)
if run_id not in lifetimes:
# A worker may have created this run after the bulk snapshot.
run = self.store.get(run_id)
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
status, finished_at = lifetimes[run_id]
if status in {"queued", "running"} or run_id in protected:
continue
if kind in {"input.json", "execution-input.json"} and path.is_file():
path.unlink(missing_ok=True)
elif kind == "uploads" and path.is_dir():
run = self.store.get(run_id)
task = self.catalog.task(run.target_path) if run and run.target_kind == "task" else None
if task is None or not self.review_upload_paths(run, task):
shutil.rmtree(path, ignore_errors=True)
elif kind == "artifacts" and path.is_dir():
if status in {"failed", "interrupted", "cancelled"}:
self._keep_diagnostic_reports(path)
try:
completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime
except FileNotFoundError:
continue
if completed_at < cutoff:
shutil.rmtree(path, ignore_errors=True)
elif kind == "work" and path.is_dir():
(path / "input.json").unlink(missing_ok=True)
if path.stat().st_mtime < time.time() - 14 * 86400:
shutil.rmtree(path)
elif kind == "progress.json" and path.is_file() and path.stat().st_mtime < time.time() - 30 * 86400:
path.unlink()
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
def _recovery_context_ids(self) -> set[int]:
protected = set()
tasks = {task.id: task for group in self.catalog.groups() for task in group.tasks}
for run in [*self.store.active_runs(), *self.store.interrupted_without_recovery()]:
task = tasks.get(run.target_path)
if run.status == "interrupted" and (task is None or not task.recoverable or not (self._work_root(run) / "input.json").is_file()):
continue
while True:
protected.add(run.id)
if run.recovery_of_run_id is None:
break
run = self.store.get(run.recovery_of_run_id)
if run is None:
break
return protected
def _promote_review_uploads(self, run: Run, task: Task) -> Run:
if run.recovery_of_run_id is None or task.execution is None or any(f.sensitive for f in task.inputs):
return run
values = json.loads(run.input_json)
if values.get(task.execution.field.name) != task.execution.dry_run_value:
return run
origin = self._work_root(run).name.split(".", 1)[0]
old_root = self._upload_root(int(origin))
for field in task.inputs:
if field.type != "file" or not values.get(field.name):
continue
source = Path(values[field.name])
if source.parent != old_root or source.is_symlink() or not source.is_file():
raise RuntimeError("The original uploaded review input is no longer available")
destination = self._upload_root(run.id) / source.name
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
shutil.copy2(source, destination)
values[field.name] = str(destination)
return self.store.set_input(run.id, values)
@staticmethod
def _keep_diagnostic_reports(root: Path) -> None:
try:
paths = list(root.iterdir())
except FileNotFoundError:
return
for path in paths:
if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json":
try:
json.loads(path.read_text(encoding="utf-8"))
continue
except FileNotFoundError:
continue
except ValueError:
pass
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink(missing_ok=True)
def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None:
"""Delete state only after a group remains absent for a grace period."""
if retention_seconds < 0 or not self.settings.task_root.is_dir():
return
root = self.settings.state_root / "tasks"
if not root.is_dir():
return
marker_path = root / ORPHANED_TASK_STATE_FILENAME
try:
markers = json.loads(marker_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
markers = {}
if not isinstance(markers, dict):
markers = {}
active_groups = {group.id for group in self.catalog.groups()}
current_time = time.time()
updated: dict[str, float] = {}
for path in root.iterdir():
if path.is_symlink() or not path.is_dir():
continue
if path.name in active_groups:
continue
first_missing_at = markers.get(path.name, current_time)
if not isinstance(first_missing_at, (int, float)):
first_missing_at = current_time
if current_time - float(first_missing_at) >= retention_seconds:
shutil.rmtree(path, ignore_errors=True)
else:
updated[path.name] = float(first_missing_at)
if updated:
temporary = marker_path.with_suffix(marker_path.suffix + ".tmp")
temporary.write_text(json.dumps(updated, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(marker_path)
else:
marker_path.unlink(missing_ok=True)
def run_background_task(self, task: BackgroundTask) -> None:
"""Run a declared internal task without creating high-volume run history."""
if self._closing.is_set():
return
log_path = self.settings.state_root / "background" / f"{task.id}.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_WORK"] = str(self.settings.state_root / "background" / f"{task.id}.work")
try:
with log_path.open("wb") as log_file:
with self._lock:
if self._closing.is_set():
return
process = subprocess.Popen(
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
)
self._background_processes.add(process)
try:
exit_code = process.wait()
if exit_code not in (0, PAUSED_EXIT_CODE):
log_file.write(f"Background task exited with status {exit_code}.\n".encode())
finally:
kill_remaining_group(process)
with self._lock:
self._background_processes.discard(process)
except OSError as error:
log_path.write_text(f"Could not start background task: {error}\n", encoding="utf-8")
def _wake_worker(self, group_id: str) -> None:
with self._lock:
if self._closing.is_set():
return
event = self._wake_events.setdefault(group_id, threading.Event())
worker = self._workers.get(group_id)
if worker is None or not worker.is_alive():
worker = threading.Thread(
target=self._worker_loop,
args=(group_id, event),
name=f"task-group-{group_id}",
daemon=True,
)
self._workers[group_id] = worker
worker.start()
event.set()
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
while not self._closing.is_set():
event.clear()
# A queued run is claimable only after its private input and uploads are saved.
with self._lock:
if self._closing.is_set():
return
run = self.store.claim_next(group_id)
if run is None:
event.wait()
continue
try:
self._execute_root_run(run)
finally:
with self._lock:
for waiting in self._wake_events.values():
waiting.set()
def _execute_root_run(self, run: Run) -> None:
if run.target_kind == "task":
task = self.catalog.task(run.target_path)
if task is None:
self.store.finish(
run.id,
status="failed",
message="Task is no longer available.",
)
return
self._execute_task(run, task)
return
group = self.catalog.group(run.target_path)
if group is None:
self.store.finish(
run.id,
status="failed",
message="Group is no longer available.",
)
return
self._execute_group(run, group)
def _execute_group(self, parent_run: Run, group: Group) -> None:
for task in group.tasks:
if self.store.is_cancel_requested(parent_run.id):
self.store.finish(
parent_run.id,
status="cancelled",
message="Group cancellation requested.",
)
return
child = self.store.create_child(parent_run, task.id)
completed_child = self._execute_task(child, task, parent_run_id=parent_run.id)
if completed_child.status != "succeeded":
parent_status = (
"cancelled"
if completed_child.status == "cancelled"
or self.store.is_cancel_requested(parent_run.id)
else "failed"
)
self.store.finish(
parent_run.id,
status=parent_status,
message=f"Stopped after {task.id}: {completed_child.status}.",
)
return
self.store.finish(parent_run.id, status="succeeded")
def _execute_task(
self,
run: Run,
task: Task,
*,
parent_run_id: int | None = None,
) -> Run:
if self._closing.is_set():
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
return self.store.finish(run.id, status="cancelled", message="Cancelled before execution.")
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
log_path.parent.mkdir(parents=True, exist_ok=True)
artifact_root = self._artifact_root(run.id)
artifact_root.mkdir(parents=True, exist_ok=True)
try:
work = self._work_root(run)
input_path = work / "input.json"
if not input_path.is_file():
if run.recovery_of_run_id is not None:
raise FileNotFoundError("The original recovery input is no longer available")
self._save_work_input(run, json.loads(run.input_json))
except OSError as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
run.id,
status="failed",
message=f"Could not prepare task input: {error}",
)
environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
environment["SERVER_MAINTENANCE_WORK"] = str(work)
environment["SERVER_MAINTENANCE_PROGRESS"] = str(self.settings.state_root / "runs" / f"{run.id}.progress.json")
if run.recovery_of_run_id is not None:
old = self.store.get(run.recovery_of_run_id)
if old and old.started_at:
environment["SERVER_MAINTENANCE_RECOVER"] = "1"
self.store.set_log_path(run.id, str(log_path))
try:
with log_path.open("wb") as log_file:
with self._lock:
if self._closing.is_set():
return self.store.finish(run.id, status="interrupted", message="Application shutdown before execution.")
process = subprocess.Popen(
[self.settings.python_executable, "-m", "app.worker", str(task.path)],
cwd=task.path.parent, stdin=subprocess.DEVNULL, stdout=log_file,
stderr=subprocess.STDOUT, env=environment, start_new_session=True,
)
self._processes[run.id] = process
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._terminate_active_process(run.id)
exit_code = process.wait()
except OSError as error:
self._delete_uploads(run.id)
self._discard_work_input(run)
shutil.rmtree(artifact_root, ignore_errors=True)
return self.store.finish(
run.id,
status="failed",
message=f"Could not start task: {error}",
log_path=str(log_path),
)
finally:
with self._lock:
owned = self._processes.pop(run.id, None)
if owned is not None:
kill_remaining_group(owned)
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._delete_uploads(run.id)
self._discard_work_input(run)
self._keep_diagnostic_reports(artifact_root)
return self.store.finish(
run.id,
status="cancelled",
exit_code=exit_code,
log_path=str(log_path),
)
if exit_code == PAUSED_EXIT_CODE or (self._closing.is_set() and exit_code != 0):
return self.store.finish(
run.id, status="interrupted", exit_code=exit_code, log_path=str(log_path),
message="Application shutdown; saved state will be checked by recovery after restart.",
)
status = "succeeded" if exit_code == 0 else "failed"
message = None
if status == "succeeded":
try:
run = self._promote_review_uploads(run, task)
except (RuntimeError, OSError) as error:
status, message = "failed", f"Could not preserve review input: {error}"
candidate = replace(run, status=status, finished_at=datetime.now().astimezone().isoformat())
if not self.review_upload_paths(candidate, task):
self._delete_uploads(run.id)
if status != "succeeded":
self._keep_diagnostic_reports(artifact_root)
else:
try:
artifact_root.rmdir()
except OSError:
pass
completed = self.store.finish(run.id, status=status, exit_code=exit_code,
message=message, log_path=str(log_path))
self._discard_work_input(run)
self.cleanup_transient_files()
return completed
def _environment(self, group_id: str) -> dict[str, str]:
task_state = self.settings.state_root / "tasks" / group_id
task_state.mkdir(parents=True, exist_ok=True)
environment = {
"PATH": "/usr/local/bin:/usr/bin:/bin",
"PYTHONUNBUFFERED": "1",
"PYTHONPATH": str(Path(__file__).resolve().parent.parent),
"SERVER_MAINTENANCE_CREDENTIALS": str(self.settings.credentials_root),
"SERVER_MAINTENANCE_STATE_ROOT": str(self.settings.state_root),
"SERVER_MAINTENANCE_STATE": str(task_state),
}
if self.settings.timezone_name:
environment["TZ"] = self.settings.timezone_name
return environment
def _upload_root(self, run_id: int) -> Path:
return self.settings.state_root / "runs" / f"{run_id}.uploads"
def _artifact_root(self, run_id: int) -> Path:
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
@staticmethod
def _redact_input(
input_data: Mapping[str, str | int | list[str]],
sensitive_names: set[str],
) -> dict[str, str | int | list[str]]:
return {
name: "[redacted]" if name in sensitive_names else value
for name, value in input_data.items()
}
def _save_uploads(
self,
run_id: int,
uploads: Mapping[str, PendingUpload],
) -> dict[str, str]:
if not uploads:
return {}
root = self._upload_root(run_id)
root.mkdir(parents=True, exist_ok=False)
saved = {}
try:
for field_name, upload in uploads.items():
suffix = upload.extension or ".upload"
destination = root / f"{field_name}{suffix}"
temporary = destination.with_suffix(destination.suffix + ".part")
total = 0
with temporary.open("xb") as output:
while chunk := upload.stream.read(64 * 1024):
total += len(chunk)
if upload.maximum_bytes is not None and total > upload.maximum_bytes:
raise ValueError(f"{field_name} exceeds its size limit")
output.write(chunk)
temporary.replace(destination)
destination.chmod(0o600)
saved[field_name] = str(destination)
except Exception:
shutil.rmtree(root, ignore_errors=True)
raise
return saved
def _delete_uploads(self, run_id: int) -> None:
shutil.rmtree(self._upload_root(run_id), ignore_errors=True)
def _terminate_active_process(self, run_id: int) -> None:
with self._lock:
process = self._processes.get(run_id)
if process is None or process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
threading.Thread(
target=self._kill_after_grace,
args=(process,),
name=f"task-cancel-{run_id}",
daemon=True,
).start()
def _kill_after_grace(self, process: subprocess.Popen[bytes]) -> None:
try:
process.wait(timeout=self.settings.cancel_grace_seconds)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass