Protect task diagnostics and stream incremental run output
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 47s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m24s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 38s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 43s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 47s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m24s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 38s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 43s
This commit is contained in:
1 parent
0aa67439fd
commit
55bbbe2fd7
8 files changed
+319
-51
No files matched your search
+43
-17
@@ -10,6 +10,7 @@ import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from .config import Settings
|
||||
@@ -127,31 +128,54 @@ class RunManager:
|
||||
runs_root = self.settings.state_root / "runs"
|
||||
cutoff = time.time() - artifact_max_age_seconds
|
||||
if runs_root.is_dir():
|
||||
lifetimes = self.store.run_lifetimes()
|
||||
for path in runs_root.iterdir():
|
||||
if not path.is_dir():
|
||||
continue
|
||||
run_id_text, separator, kind = path.name.partition(".")
|
||||
if not separator or not run_id_text.isdigit() or kind not in {"uploads", "artifacts"}:
|
||||
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
|
||||
continue
|
||||
run = self.store.get(int(run_id_text))
|
||||
if kind == "uploads" and (run is None or run.status not in {"queued", "running"}):
|
||||
run_id = int(run_id_text)
|
||||
if run_id not in lifetimes:
|
||||
# A worker may have created this run after the bulk snapshot.
|
||||
run = self.store.get(run_id)
|
||||
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
|
||||
status, finished_at = lifetimes[run_id]
|
||||
if status in {"queued", "running"}:
|
||||
continue
|
||||
if kind in {"input.json", "execution-input.json"} and path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
elif kind == "uploads" and path.is_dir():
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif kind == "artifacts":
|
||||
elif kind == "artifacts" and path.is_dir():
|
||||
if status in {"failed", "interrupted", "cancelled"}:
|
||||
self._keep_diagnostic_reports(path)
|
||||
try:
|
||||
expired = path.stat().st_mtime < cutoff
|
||||
completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if expired:
|
||||
if completed_at < cutoff:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
for path in runs_root.glob("*.execution-input.json"):
|
||||
run_id_text = path.name.removesuffix(".execution-input.json")
|
||||
if not run_id_text.isdigit():
|
||||
continue
|
||||
run = self.store.get(int(run_id_text))
|
||||
if run is None or run.status not in {"queued", "running"}:
|
||||
path.unlink(missing_ok=True)
|
||||
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
|
||||
|
||||
@staticmethod
|
||||
def _keep_diagnostic_reports(root: Path) -> None:
|
||||
try:
|
||||
paths = list(root.iterdir())
|
||||
except FileNotFoundError:
|
||||
return
|
||||
for path in paths:
|
||||
if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json":
|
||||
try:
|
||||
json.loads(path.read_text(encoding="utf-8"))
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except ValueError:
|
||||
pass
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None:
|
||||
"""Delete state only after a group remains absent for a grace period."""
|
||||
|
||||
@@ -318,6 +342,7 @@ class RunManager:
|
||||
environment = self._environment(task.group_id)
|
||||
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
|
||||
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
|
||||
self.store.set_log_path(run.id, str(log_path))
|
||||
|
||||
try:
|
||||
with log_path.open("wb") as log_file:
|
||||
@@ -347,11 +372,12 @@ class RunManager:
|
||||
self._processes.pop(run.id, None)
|
||||
self._delete_uploads(run.id)
|
||||
self._delete_execution_input(run.id)
|
||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||
|
||||
if self.store.is_cancel_requested(run.id) or (
|
||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||
):
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
self._keep_diagnostic_reports(artifact_root)
|
||||
return self.store.finish(
|
||||
run.id,
|
||||
status="cancelled",
|
||||
@@ -365,7 +391,7 @@ class RunManager:
|
||||
log_path=str(log_path),
|
||||
)
|
||||
if completed.status != "succeeded":
|
||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||
self._keep_diagnostic_reports(artifact_root)
|
||||
else:
|
||||
try:
|
||||
artifact_root.rmdir()
|
||||
|
||||
@@ -256,6 +256,17 @@ class RunStore:
|
||||
with self._connect() as owned_connection:
|
||||
return self.get(run_id, connection=owned_connection)
|
||||
|
||||
def run_lifetimes(self) -> dict[int, tuple[str, str | None]]:
|
||||
with self._connect() as connection:
|
||||
return {
|
||||
row["id"]: (row["status"], row["finished_at"])
|
||||
for row in connection.execute("SELECT id, status, finished_at FROM runs")
|
||||
}
|
||||
|
||||
def set_log_path(self, run_id: int, path: str) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id))
|
||||
|
||||
def last_run(self, target_kind: str, target_path: str) -> Run | None:
|
||||
with self._connect() as connection:
|
||||
row = connection.execute(
|
||||
|
||||
+33
-9
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import ipaddress
|
||||
import json
|
||||
from datetime import datetime
|
||||
@@ -137,11 +138,13 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
if run is None:
|
||||
abort(404)
|
||||
execution_task = execution_replay_task(run)
|
||||
output, offset, _, _ = read_run_output(run)
|
||||
return render_template(
|
||||
"run.html",
|
||||
run=run,
|
||||
children=store.children(run.id),
|
||||
output=read_run_output(run),
|
||||
output=output,
|
||||
output_offset=offset,
|
||||
execution_task=execution_task,
|
||||
)
|
||||
|
||||
@@ -195,15 +198,36 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
run = store.get(run_id)
|
||||
if run is None:
|
||||
abort(404)
|
||||
return {"output": read_run_output(run)}
|
||||
raw_offset = request.args.get("offset")
|
||||
if raw_offset is None:
|
||||
return {"output": read_run_output(run)[0]}
|
||||
if not raw_offset.isdecimal() or len(raw_offset) > 19 or int(raw_offset) > 2**63 - 1:
|
||||
abort(400)
|
||||
output, offset, reset, has_more = read_run_output(run, int(raw_offset), 1024 * 1024)
|
||||
return {
|
||||
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
|
||||
"status": run.status,
|
||||
}
|
||||
|
||||
def read_run_output(run) -> str:
|
||||
def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
|
||||
if not run.log_path:
|
||||
return ""
|
||||
return "", 0, offset != 0, False
|
||||
try:
|
||||
return Path(run.log_path).read_text(encoding="utf-8", errors="replace")
|
||||
with Path(run.log_path).open("rb") as stream:
|
||||
stream.seek(0, 2)
|
||||
size = stream.tell()
|
||||
reset = offset > size
|
||||
if reset:
|
||||
offset = 0
|
||||
stream.seek(offset)
|
||||
data = stream.read() if maximum_bytes is None else stream.read(maximum_bytes)
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
return "", 0, offset != 0, False
|
||||
has_more = offset + len(data) < size
|
||||
decoder = codecs.getincrementaldecoder("utf-8")("replace")
|
||||
output = decoder.decode(data, final=run.status not in {"queued", "running"} and not has_more)
|
||||
pending, _ = decoder.getstate()
|
||||
return output, offset + len(data) - len(pending), reset, has_more
|
||||
|
||||
@app.get("/runs/<int:run_id>/artifacts")
|
||||
def run_artifacts(run_id: int):
|
||||
@@ -224,7 +248,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
@app.get("/runs/<int:run_id>/artifacts/<filename>/view")
|
||||
def view_artifact(run_id: int, filename: str):
|
||||
run = store.get(run_id)
|
||||
if run is None or run.status != "succeeded" or not filename.casefold().endswith(".json"):
|
||||
if run is None or run.status in {"queued", "running"} or not filename.casefold().endswith(".json"):
|
||||
abort(404)
|
||||
artifact = manager.artifact_path(run.id, filename)
|
||||
if artifact is None:
|
||||
@@ -234,7 +258,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
@app.get("/runs/<int:run_id>/artifacts/<filename>")
|
||||
def download_artifact(run_id: int, filename: str):
|
||||
run = store.get(run_id)
|
||||
if run is None or run.status != "succeeded":
|
||||
if run is None or run.status in {"queued", "running"} or (run.status != "succeeded" and not filename.casefold().endswith(".json")):
|
||||
abort(404)
|
||||
artifact = manager.artifact_path(run.id, filename)
|
||||
if artifact is None:
|
||||
@@ -250,11 +274,11 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
def delete_after_transfer():
|
||||
try:
|
||||
yield from original_response
|
||||
manager.delete_artifact(run.id, artifact.name)
|
||||
finally:
|
||||
close = getattr(original_response, "close", None)
|
||||
if close is not None:
|
||||
close()
|
||||
manager.delete_artifact(run.id, artifact.name)
|
||||
|
||||
response.response = delete_after_transfer()
|
||||
response.direct_passthrough = False
|
||||
|
||||
Reference in new issue
Block a user