Protect task diagnostics and stream incremental run output
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 47s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m24s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 38s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 43s

This commit is contained in:
ajp_anton committed 2026-10-26 23:39:50 +00:00
1 parent 0aa67439fd
commit 55bbbe2fd7
8 files changed
+315 -47

No files matched your search

+12 -6
View File
@@ -8,9 +8,9 @@ with `python -m app`.
The application discovers direct `*.py` files in each non-hidden directory The application discovers direct `*.py` files in each non-hidden directory
under `/opt/tasks`. Each directory is a task group. Tasks in one group run under `/opt/tasks`. Each directory is a task group. Tasks in one group run
sequentially; different groups may run concurrently. It records run history and sequentially; different groups may run concurrently. It records run history in
combined output in SQLite under `/var/lib/server-maintenance`, and supports SQLite and combined output in log files under `/var/lib/server-maintenance`,
cancellation of active task process groups. and supports cancellation of active task process groups.
The image contains application code only. Mount these paths at runtime: The image contains application code only. Mount these paths at runtime:
@@ -77,6 +77,7 @@ readable. `description` is optional on a task and otherwise comes from the
first non-empty line of its module docstring. The order of keys in `tasks` first non-empty line of its module docstring. The order of keys in `tasks`
controls the listed tasks' display order. Include a task as `{}` when only controls the listed tasks' display order. Include a task as `{}` when only
ordering is needed. Unlisted direct Python files follow alphabetically. ordering is needed. Unlisted direct Python files follow alphabetically.
Use `\n` for a line break or `\n\n` for a paragraph gap in descriptions.
Each task declaration may contain `title`, `description`, `inputs`, Each task declaration may contain `title`, `description`, `inputs`,
`wait_for_result`, `download_artifacts`, and `execution`. Set `wait_for_result`, `download_artifacts`, and `execution`. Set
@@ -143,9 +144,14 @@ Tasks can write files to the directory named by
`SERVER_MAINTENANCE_ARTIFACTS`. Run-detail pages show them when the task `SERVER_MAINTENANCE_ARTIFACTS`. Run-detail pages show them when the task
finishes. JSON artifacts offer an **Open** link that renders the JSON in a new finishes. JSON artifacts offer an **Open** link that renders the JSON in a new
tab and leaves the artifact available; other artifacts offer a download link. tab and leaves the artifact available; other artifacts offer a download link.
The server deletes an artifact after its download transfer ends; any artifact The server deletes an artifact after its download stream completes; interrupted
not downloaded is removed after 24 hours. A browser cannot confirm that a downloads remain available for retry. Active runs' artifacts never expire.
downloaded file was retained on the client device. After a run finishes, artifacts expire after 24 hours. Failed or cancelled
runs keep valid JSON diagnostic reports, but discard incomplete downloads.
Ordinary input files are removed when execution ends; saved inputs remain in
SQLite for history and review replay. Run pages fetch only appended log bytes.
A browser cannot confirm that a downloaded file was retained on the client device.
## Background tasks ## Background tasks
+43 -17
View File
@@ -10,6 +10,7 @@ import subprocess
import threading import threading
import time import time
from collections.abc import Mapping from collections.abc import Mapping
from datetime import datetime
from pathlib import Path from pathlib import Path
from .config import Settings from .config import Settings
@@ -127,31 +128,54 @@ class RunManager:
runs_root = self.settings.state_root / "runs" runs_root = self.settings.state_root / "runs"
cutoff = time.time() - artifact_max_age_seconds cutoff = time.time() - artifact_max_age_seconds
if runs_root.is_dir(): if runs_root.is_dir():
lifetimes = self.store.run_lifetimes()
for path in runs_root.iterdir(): for path in runs_root.iterdir():
if not path.is_dir():
continue
run_id_text, separator, kind = path.name.partition(".") run_id_text, separator, kind = path.name.partition(".")
if not separator or not run_id_text.isdigit() or kind not in {"uploads", "artifacts"}: if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
continue continue
run = self.store.get(int(run_id_text)) run_id = int(run_id_text)
if kind == "uploads" and (run is None or run.status not in {"queued", "running"}): if run_id not in lifetimes:
# A worker may have created this run after the bulk snapshot.
run = self.store.get(run_id)
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
status, finished_at = lifetimes[run_id]
if status in {"queued", "running"}:
continue
if kind in {"input.json", "execution-input.json"} and path.is_file():
path.unlink(missing_ok=True)
elif kind == "uploads" and path.is_dir():
shutil.rmtree(path, ignore_errors=True) shutil.rmtree(path, ignore_errors=True)
elif kind == "artifacts": elif kind == "artifacts" and path.is_dir():
if status in {"failed", "interrupted", "cancelled"}:
self._keep_diagnostic_reports(path)
try: try:
expired = path.stat().st_mtime < cutoff completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime
except FileNotFoundError: except FileNotFoundError:
continue continue
if expired: if completed_at < cutoff:
shutil.rmtree(path, ignore_errors=True) shutil.rmtree(path, ignore_errors=True)
for path in runs_root.glob("*.execution-input.json"):
run_id_text = path.name.removesuffix(".execution-input.json")
if not run_id_text.isdigit():
continue
run = self.store.get(int(run_id_text))
if run is None or run.status not in {"queued", "running"}:
path.unlink(missing_ok=True)
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds) self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
@staticmethod
def _keep_diagnostic_reports(root: Path) -> None:
try:
paths = list(root.iterdir())
except FileNotFoundError:
return
for path in paths:
if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json":
try:
json.loads(path.read_text(encoding="utf-8"))
continue
except FileNotFoundError:
continue
except ValueError:
pass
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink(missing_ok=True)
def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None: def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None:
"""Delete state only after a group remains absent for a grace period.""" """Delete state only after a group remains absent for a grace period."""
@@ -318,6 +342,7 @@ class RunManager:
environment = self._environment(task.group_id) environment = self._environment(task.group_id)
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path) environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root) environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
self.store.set_log_path(run.id, str(log_path))
try: try:
with log_path.open("wb") as log_file: with log_path.open("wb") as log_file:
@@ -347,11 +372,12 @@ class RunManager:
self._processes.pop(run.id, None) self._processes.pop(run.id, None)
self._delete_uploads(run.id) self._delete_uploads(run.id)
self._delete_execution_input(run.id) self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
if self.store.is_cancel_requested(run.id) or ( if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
): ):
shutil.rmtree(artifact_root, ignore_errors=True) self._keep_diagnostic_reports(artifact_root)
return self.store.finish( return self.store.finish(
run.id, run.id,
status="cancelled", status="cancelled",
@@ -365,7 +391,7 @@ class RunManager:
log_path=str(log_path), log_path=str(log_path),
) )
if completed.status != "succeeded": if completed.status != "succeeded":
shutil.rmtree(artifact_root, ignore_errors=True) self._keep_diagnostic_reports(artifact_root)
else: else:
try: try:
artifact_root.rmdir() artifact_root.rmdir()
+11
View File
@@ -256,6 +256,17 @@ class RunStore:
with self._connect() as owned_connection: with self._connect() as owned_connection:
return self.get(run_id, connection=owned_connection) return self.get(run_id, connection=owned_connection)
def run_lifetimes(self) -> dict[int, tuple[str, str | None]]:
with self._connect() as connection:
return {
row["id"]: (row["status"], row["finished_at"])
for row in connection.execute("SELECT id, status, finished_at FROM runs")
}
def set_log_path(self, run_id: int, path: str) -> None:
with self._connect() as connection:
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id))
def last_run(self, target_kind: str, target_path: str) -> Run | None: def last_run(self, target_kind: str, target_path: str) -> Run | None:
with self._connect() as connection: with self._connect() as connection:
row = connection.execute( row = connection.execute(
+33 -9
View File
@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import codecs
import ipaddress import ipaddress
import json import json
from datetime import datetime from datetime import datetime
@@ -137,11 +138,13 @@ def create_app(settings: Settings | None = None) -> Flask:
if run is None: if run is None:
abort(404) abort(404)
execution_task = execution_replay_task(run) execution_task = execution_replay_task(run)
output, offset, _, _ = read_run_output(run)
return render_template( return render_template(
"run.html", "run.html",
run=run, run=run,
children=store.children(run.id), children=store.children(run.id),
output=read_run_output(run), output=output,
output_offset=offset,
execution_task=execution_task, execution_task=execution_task,
) )
@@ -195,15 +198,36 @@ def create_app(settings: Settings | None = None) -> Flask:
run = store.get(run_id) run = store.get(run_id)
if run is None: if run is None:
abort(404) abort(404)
return {"output": read_run_output(run)} raw_offset = request.args.get("offset")
if raw_offset is None:
return {"output": read_run_output(run)[0]}
if not raw_offset.isdecimal() or len(raw_offset) > 19 or int(raw_offset) > 2**63 - 1:
abort(400)
output, offset, reset, has_more = read_run_output(run, int(raw_offset), 1024 * 1024)
return {
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
"status": run.status,
}
def read_run_output(run) -> str: def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
if not run.log_path: if not run.log_path:
return "" return "", 0, offset != 0, False
try: try:
return Path(run.log_path).read_text(encoding="utf-8", errors="replace") with Path(run.log_path).open("rb") as stream:
stream.seek(0, 2)
size = stream.tell()
reset = offset > size
if reset:
offset = 0
stream.seek(offset)
data = stream.read() if maximum_bytes is None else stream.read(maximum_bytes)
except FileNotFoundError: except FileNotFoundError:
return "" return "", 0, offset != 0, False
has_more = offset + len(data) < size
decoder = codecs.getincrementaldecoder("utf-8")("replace")
output = decoder.decode(data, final=run.status not in {"queued", "running"} and not has_more)
pending, _ = decoder.getstate()
return output, offset + len(data) - len(pending), reset, has_more
@app.get("/runs/<int:run_id>/artifacts") @app.get("/runs/<int:run_id>/artifacts")
def run_artifacts(run_id: int): def run_artifacts(run_id: int):
@@ -224,7 +248,7 @@ def create_app(settings: Settings | None = None) -> Flask:
@app.get("/runs/<int:run_id>/artifacts/<filename>/view") @app.get("/runs/<int:run_id>/artifacts/<filename>/view")
def view_artifact(run_id: int, filename: str): def view_artifact(run_id: int, filename: str):
run = store.get(run_id) run = store.get(run_id)
if run is None or run.status != "succeeded" or not filename.casefold().endswith(".json"): if run is None or run.status in {"queued", "running"} or not filename.casefold().endswith(".json"):
abort(404) abort(404)
artifact = manager.artifact_path(run.id, filename) artifact = manager.artifact_path(run.id, filename)
if artifact is None: if artifact is None:
@@ -234,7 +258,7 @@ def create_app(settings: Settings | None = None) -> Flask:
@app.get("/runs/<int:run_id>/artifacts/<filename>") @app.get("/runs/<int:run_id>/artifacts/<filename>")
def download_artifact(run_id: int, filename: str): def download_artifact(run_id: int, filename: str):
run = store.get(run_id) run = store.get(run_id)
if run is None or run.status != "succeeded": if run is None or run.status in {"queued", "running"} or (run.status != "succeeded" and not filename.casefold().endswith(".json")):
abort(404) abort(404)
artifact = manager.artifact_path(run.id, filename) artifact = manager.artifact_path(run.id, filename)
if artifact is None: if artifact is None:
@@ -250,11 +274,11 @@ def create_app(settings: Settings | None = None) -> Flask:
def delete_after_transfer(): def delete_after_transfer():
try: try:
yield from original_response yield from original_response
manager.delete_artifact(run.id, artifact.name)
finally: finally:
close = getattr(original_response, "close", None) close = getattr(original_response, "close", None)
if close is not None: if close is not None:
close() close()
manager.delete_artifact(run.id, artifact.name)
response.response = delete_after_transfer() response.response = delete_after_transfer()
response.direct_passthrough = False response.direct_passthrough = False
+33 -14
View File
@@ -1,25 +1,33 @@
const output = document.querySelector("#run-output"); const output = document.querySelector("#run-output");
if (output) { if (output) {
let refreshTimer; let offset = Number(output.dataset.outputOffset || 0);
let detailsFinalized = !["queued", "running"].includes( let detailsFinalized = !["queued", "running"].includes(
document.querySelector("#run-status")?.textContent.trim(), document.querySelector("#run-status")?.textContent.trim(),
); );
const refresh = async () => { const refresh = async () => {
const response = await fetch(output.dataset.outputUrl, { cache: "no-store" }); let delay = 2000;
if (response.ok) { try {
const nextOutput = (await response.json()).output; const url = new URL(output.dataset.outputUrl, window.location.href);
if (nextOutput !== output.textContent) { url.searchParams.set("offset", String(offset));
output.textContent = nextOutput; const response = await fetch(url, { cache: "no-store" });
if (!response.ok) {
return;
} }
const state = await response.json();
if (state.reset) {
output.replaceChildren();
} }
if (state.output) {
output.append(document.createTextNode(state.output));
}
offset = state.offset;
const status = document.querySelector("#run-status"); const status = document.querySelector("#run-status");
if (status) { if (status) {
const stateResponse = await fetch(status.dataset.stateUrl, { cache: "no-store" });
if (stateResponse.ok) {
const state = await stateResponse.json();
status.textContent = state.status; status.textContent = state.status;
if (state.status !== "queued" && state.status !== "running") { if (state.has_more) {
delay = 0;
} else if (state.status !== "queued" && state.status !== "running") {
if (!detailsFinalized) { if (!detailsFinalized) {
const pageResponse = await fetch(window.location.href, { cache: "no-store" }); const pageResponse = await fetch(window.location.href, { cache: "no-store" });
if (!pageResponse.ok) { if (!pageResponse.ok) {
@@ -36,13 +44,22 @@ if (output) {
nextDetails.after(...page.querySelectorAll(actions)); nextDetails.after(...page.querySelectorAll(actions));
detailsFinalized = true; detailsFinalized = true;
} }
window.clearInterval(refreshTimer); const artifacts = document.querySelector("#run-artifacts");
if (artifacts) {
await refreshArtifacts(artifacts);
} }
delay = null;
}
}
} catch (error) {
console.warn("Could not refresh run details:", error);
} finally {
if (delay !== null) {
window.setTimeout(refresh, delay);
} }
} }
}; };
refresh(); refresh();
refreshTimer = window.setInterval(refresh, 2000);
} }
const refreshArtifacts = async (container) => { const refreshArtifacts = async (container) => {
@@ -73,8 +90,9 @@ const refreshArtifacts = async (container) => {
}; };
for (const container of document.querySelectorAll("#run-artifacts")) { for (const container of document.querySelectorAll("#run-artifacts")) {
if (!output) {
refreshArtifacts(container); refreshArtifacts(container);
window.setInterval(() => refreshArtifacts(container), 2000); }
} }
for (const choice of document.querySelectorAll("button[data-execution-choice]")) { for (const choice of document.querySelectorAll("button[data-execution-choice]")) {
@@ -154,6 +172,7 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) {
}; };
const waitForRun = async (run) => { const waitForRun = async (run) => {
const started = Date.now();
while (true) { while (true) {
const response = await fetch(run.state_url, { cache: "no-store" }); const response = await fetch(run.state_url, { cache: "no-store" });
if (!response.ok) { if (!response.ok) {
@@ -171,7 +190,7 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) {
} }
return; return;
} }
await new Promise((resolve) => window.setTimeout(resolve, 500)); await new Promise((resolve) => window.setTimeout(resolve, Date.now() - started < 10000 ? 500 : 2000));
} }
}; };
+1 -1
View File
@@ -37,5 +37,5 @@
<ul></ul> <ul></ul>
</section> </section>
<h2>Output</h2> <h2>Output</h2>
<pre id="run-output" data-output-url="{{ url_for('run_output', run_id=run.id) }}">{{ output }}</pre> <pre id="run-output" data-output-url="{{ url_for('run_output', run_id=run.id) }}" data-output-offset="{{ output_offset }}">{{ output }}</pre>
{% endblock %} {% endblock %}
@@ -0,0 +1,87 @@
import pytest
from app.web import create_app
from .conftest import write_task
from .test_runner import wait_for, wait_for_output
@pytest.fixture
def output_run(settings):
app = create_app(settings)
store = app.extensions["run_store"]
run, _ = store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
store.claim_next("example")
path = settings.state_root / "test.log"
path.write_bytes(b"first\n")
with store._connect() as connection:
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (str(path), run.id))
return app.test_client(), run.id, path, store
def test_log_polling_returns_only_appended_bytes_and_status(output_run):
client, run_id, path, store = output_run
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"}
with path.open("ab") as stream:
stream.write("next: \u20ac\n".encode())
store.finish(run_id, status="succeeded")
next_chunk = client.get(f"/runs/{run_id}/output?offset=6").get_json()
assert next_chunk["output"] == "next: \u20ac\n"
assert next_chunk["offset"] == path.stat().st_size
assert next_chunk["status"] == "succeeded"
assert client.get(f"/runs/{run_id}/output").get_json() == {"output": "first\nnext: \u20ac\n"}
def test_incomplete_utf8_is_deferred_until_the_next_poll(output_run):
client, run_id, path, _ = output_run
path.write_bytes(b"ok\xe2\x82")
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
assert first["output"] == "ok"
assert first["offset"] == 2
with path.open("ab") as stream:
stream.write(b"\xac")
second = client.get(f"/runs/{run_id}/output?offset=2").get_json()
assert second["output"] == "\u20ac"
assert second["offset"] == 5
def test_truncated_log_resets_the_browser_offset(output_run):
client, run_id, path, _ = output_run
path.write_bytes(b"new")
result = client.get(f"/runs/{run_id}/output?offset=6").get_json()
assert result["reset"] is True
assert result["output"] == "new"
assert result["offset"] == 3
def test_large_log_is_drained_in_bounded_chunks(output_run):
client, run_id, path, store = output_run
path.write_bytes(b"x" * (1024 * 1024 - 1) + "\u20ac".encode())
store.finish(run_id, status="succeeded")
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
assert first["has_more"] is True
assert first["offset"] == 1024 * 1024 - 1
second = client.get(f"/runs/{run_id}/output?offset={first['offset']}").get_json()
assert second["output"] == "\u20ac"
assert second["has_more"] is False
@pytest.mark.parametrize("offset", ["-1", "abc", "1.5", "9" * 100, str(2**63)])
def test_invalid_log_offset_is_rejected(output_run, offset):
client, run_id, _, _ = output_run
assert client.get(f"/runs/{run_id}/output?offset={offset}").status_code == 400
def test_running_task_output_is_visible_before_completion(settings):
write_task(settings.task_root, "example", "long.py", 'import time\nprint("working", flush=True)\ntime.sleep(30)\n')
app = create_app(settings)
manager = app.extensions["run_manager"]
run, _ = manager.run_task(app.extensions["catalog"].task("example/long.py"))
try:
wait_for_output(settings.state_root / "runs" / f"{run.id}.log", "working")
response = app.test_client().get(f"/runs/{run.id}/output?offset=0").get_json()
assert response["status"] == "running"
assert response["output"] == "working\n"
finally:
manager.request_cancel(run.id)
wait_for(manager, run.id, {"cancelled"})
@@ -0,0 +1,95 @@
import json
import os
import time
from datetime import UTC, datetime, timedelta
from app.web import create_app
from .conftest import write_task
from .test_runner import wait_for
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
manager.store.claim_next("example")
root = manager.settings.state_root / "runs"
for name in (f"{run.id}.artifacts", f"{run.id}.uploads"):
path = root / name
path.mkdir(parents=True)
(path / "data").write_text("active")
os.utime(path, (time.time() - 3 * 86400,) * 2)
input_path = root / f"{run.id}.input.json"
input_path.write_text("{}")
monkeypatch.setattr(manager.store, "run_lifetimes", lambda: {})
manager.cleanup_transient_files()
assert input_path.exists()
assert (root / f"{run.id}.artifacts/data").exists()
assert (root / f"{run.id}.uploads/data").exists()
def test_artifact_retention_starts_at_completion_not_directory_creation(manager):
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
manager.store.finish(run.id, status="succeeded")
root = manager.settings.state_root / "runs"
artifact = root / f"{run.id}.artifacts"
artifact.mkdir(parents=True)
(artifact / "report.json").write_text("{}")
os.utime(artifact, (time.time() - 3 * 86400,) * 2)
(root / f"{run.id}.input.json").write_text("{}")
manager.cleanup_transient_files()
assert (artifact / "report.json").exists()
assert not (root / f"{run.id}.input.json").exists()
old = (datetime.now(UTC) - timedelta(days=3)).isoformat()
with manager.store._connect() as connection:
connection.execute("UPDATE runs SET finished_at = ? WHERE id = ?", (old, run.id))
manager.cleanup_transient_files()
assert not artifact.exists()
def test_failed_task_keeps_valid_diagnostics_but_not_incomplete_downloads(manager):
write_task(manager.settings.task_root, "example", "fail.py", '''
import os
from pathlib import Path
root = Path(os.environ["SERVER_MAINTENANCE_ARTIFACTS"])
(root / "report.json").write_text('{"completed_steps": 2}')
(root / "broken.json").write_text('{')
(root / "partial.zip").write_bytes(b"partial")
raise SystemExit(1)
''')
task = manager.catalog.task("example/fail.py")
run, _ = manager.run_task(task)
wait_for(manager, run.id, {"failed"})
assert manager.artifacts(run.id) == ["report.json"]
assert not (manager.settings.state_root / "runs" / f"{run.id}.input.json").exists()
def test_failed_json_report_is_accessible(settings):
app = create_app(settings)
manager = app.extensions["run_manager"]
run, _ = manager.store.enqueue(target_kind="task", target_path="example/fail.py", queue_group="example")
manager.store.finish(run.id, status="failed")
root = settings.state_root / "runs" / f"{run.id}.artifacts"
root.mkdir(parents=True)
(root / "report.json").write_text('{"completed_steps":2}')
client = app.test_client()
response = client.get(f"/runs/{run.id}/artifacts/report.json/view")
assert response.status_code == 200
assert json.loads(response.data) == {"completed_steps": 2}
def test_aborted_artifact_download_can_be_retried(settings):
app = create_app(settings)
manager = app.extensions["run_manager"]
run, _ = manager.store.enqueue(target_kind="task", target_path="example/archive.py", queue_group="example")
manager.store.finish(run.id, status="succeeded")
root = settings.state_root / "runs" / f"{run.id}.artifacts"
root.mkdir(parents=True)
path = root / "review.zip"
path.write_bytes(b"x" * 100_000)
client = app.test_client()
response = client.get(f"/runs/{run.id}/artifacts/review.zip", buffered=False)
next(iter(response.response))
response.close()
assert path.exists()
retry = client.get(f"/runs/{run.id}/artifacts/review.zip")
assert len(retry.data) == 100_000
assert not path.exists()