From 55bbbe2fd7661fdf722ae29ec551745e92d81cc8 Mon Sep 17 00:00:00 2001 From: ajp_anton Date: Mon, 26 Oct 2026 23:39:50 +0000 Subject: [PATCH] Protect task diagnostics and stream incremental run output --- python-tools/README.md | 18 ++-- python-tools/app/runner.py | 60 ++++++++---- python-tools/app/store.py | 11 +++ python-tools/app/web.py | 42 ++++++-- python-tools/static/app.js | 55 +++++++---- python-tools/templates/run.html | 2 +- python-tools/tests/test_incremental_output.py | 87 +++++++++++++++++ python-tools/tests/test_runtime_cleanup.py | 95 +++++++++++++++++++ 8 files changed, 319 insertions(+), 51 deletions(-) create mode 100644 python-tools/tests/test_incremental_output.py create mode 100644 python-tools/tests/test_runtime_cleanup.py diff --git a/python-tools/README.md b/python-tools/README.md index db77465..576eaa1 100644 --- a/python-tools/README.md +++ b/python-tools/README.md @@ -8,9 +8,9 @@ with `python -m app`. The application discovers direct `*.py` files in each non-hidden directory under `/opt/tasks`. Each directory is a task group. Tasks in one group run -sequentially; different groups may run concurrently. It records run history and -combined output in SQLite under `/var/lib/server-maintenance`, and supports -cancellation of active task process groups. +sequentially; different groups may run concurrently. It records run history in +SQLite and combined output in log files under `/var/lib/server-maintenance`, +and supports cancellation of active task process groups. The image contains application code only. Mount these paths at runtime: @@ -77,6 +77,7 @@ readable. `description` is optional on a task and otherwise comes from the first non-empty line of its module docstring. The order of keys in `tasks` controls the listed tasks' display order. Include a task as `{}` when only ordering is needed. Unlisted direct Python files follow alphabetically. +Use `\n` for a line break or `\n\n` for a paragraph gap in descriptions. Each task declaration may contain `title`, `description`, `inputs`, `wait_for_result`, `download_artifacts`, and `execution`. Set @@ -143,9 +144,14 @@ Tasks can write files to the directory named by `SERVER_MAINTENANCE_ARTIFACTS`. Run-detail pages show them when the task finishes. JSON artifacts offer an **Open** link that renders the JSON in a new tab and leaves the artifact available; other artifacts offer a download link. -The server deletes an artifact after its download transfer ends; any artifact -not downloaded is removed after 24 hours. A browser cannot confirm that a -downloaded file was retained on the client device. +The server deletes an artifact after its download stream completes; interrupted +downloads remain available for retry. Active runs' artifacts never expire. +After a run finishes, artifacts expire after 24 hours. Failed or cancelled +runs keep valid JSON diagnostic reports, but discard incomplete downloads. +Ordinary input files are removed when execution ends; saved inputs remain in +SQLite for history and review replay. Run pages fetch only appended log bytes. + +A browser cannot confirm that a downloaded file was retained on the client device. ## Background tasks diff --git a/python-tools/app/runner.py b/python-tools/app/runner.py index a48a5a9..ee63027 100644 --- a/python-tools/app/runner.py +++ b/python-tools/app/runner.py @@ -10,6 +10,7 @@ import subprocess import threading import time from collections.abc import Mapping +from datetime import datetime from pathlib import Path from .config import Settings @@ -127,31 +128,54 @@ class RunManager: runs_root = self.settings.state_root / "runs" cutoff = time.time() - artifact_max_age_seconds if runs_root.is_dir(): + lifetimes = self.store.run_lifetimes() for path in runs_root.iterdir(): - if not path.is_dir(): - continue run_id_text, separator, kind = path.name.partition(".") - if not separator or not run_id_text.isdigit() or kind not in {"uploads", "artifacts"}: + if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink(): continue - run = self.store.get(int(run_id_text)) - if kind == "uploads" and (run is None or run.status not in {"queued", "running"}): + run_id = int(run_id_text) + if run_id not in lifetimes: + # A worker may have created this run after the bulk snapshot. + run = self.store.get(run_id) + lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None) + status, finished_at = lifetimes[run_id] + if status in {"queued", "running"}: + continue + if kind in {"input.json", "execution-input.json"} and path.is_file(): + path.unlink(missing_ok=True) + elif kind == "uploads" and path.is_dir(): shutil.rmtree(path, ignore_errors=True) - elif kind == "artifacts": + elif kind == "artifacts" and path.is_dir(): + if status in {"failed", "interrupted", "cancelled"}: + self._keep_diagnostic_reports(path) try: - expired = path.stat().st_mtime < cutoff + completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime except FileNotFoundError: continue - if expired: + if completed_at < cutoff: shutil.rmtree(path, ignore_errors=True) - for path in runs_root.glob("*.execution-input.json"): - run_id_text = path.name.removesuffix(".execution-input.json") - if not run_id_text.isdigit(): - continue - run = self.store.get(int(run_id_text)) - if run is None or run.status not in {"queued", "running"}: - path.unlink(missing_ok=True) self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds) + @staticmethod + def _keep_diagnostic_reports(root: Path) -> None: + try: + paths = list(root.iterdir()) + except FileNotFoundError: + return + for path in paths: + if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json": + try: + json.loads(path.read_text(encoding="utf-8")) + continue + except FileNotFoundError: + continue + except ValueError: + pass + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None: """Delete state only after a group remains absent for a grace period.""" @@ -318,6 +342,7 @@ class RunManager: environment = self._environment(task.group_id) environment["SERVER_MAINTENANCE_INPUT"] = str(input_path) environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root) + self.store.set_log_path(run.id, str(log_path)) try: with log_path.open("wb") as log_file: @@ -347,11 +372,12 @@ class RunManager: self._processes.pop(run.id, None) self._delete_uploads(run.id) self._delete_execution_input(run.id) + (self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True) if self.store.is_cancel_requested(run.id) or ( parent_run_id is not None and self.store.is_cancel_requested(parent_run_id) ): - shutil.rmtree(artifact_root, ignore_errors=True) + self._keep_diagnostic_reports(artifact_root) return self.store.finish( run.id, status="cancelled", @@ -365,7 +391,7 @@ class RunManager: log_path=str(log_path), ) if completed.status != "succeeded": - shutil.rmtree(artifact_root, ignore_errors=True) + self._keep_diagnostic_reports(artifact_root) else: try: artifact_root.rmdir() diff --git a/python-tools/app/store.py b/python-tools/app/store.py index 2bfc392..bb587b7 100644 --- a/python-tools/app/store.py +++ b/python-tools/app/store.py @@ -256,6 +256,17 @@ class RunStore: with self._connect() as owned_connection: return self.get(run_id, connection=owned_connection) + def run_lifetimes(self) -> dict[int, tuple[str, str | None]]: + with self._connect() as connection: + return { + row["id"]: (row["status"], row["finished_at"]) + for row in connection.execute("SELECT id, status, finished_at FROM runs") + } + + def set_log_path(self, run_id: int, path: str) -> None: + with self._connect() as connection: + connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id)) + def last_run(self, target_kind: str, target_path: str) -> Run | None: with self._connect() as connection: row = connection.execute( diff --git a/python-tools/app/web.py b/python-tools/app/web.py index 76dd47e..f0e3c7d 100644 --- a/python-tools/app/web.py +++ b/python-tools/app/web.py @@ -2,6 +2,7 @@ from __future__ import annotations +import codecs import ipaddress import json from datetime import datetime @@ -137,11 +138,13 @@ def create_app(settings: Settings | None = None) -> Flask: if run is None: abort(404) execution_task = execution_replay_task(run) + output, offset, _, _ = read_run_output(run) return render_template( "run.html", run=run, children=store.children(run.id), - output=read_run_output(run), + output=output, + output_offset=offset, execution_task=execution_task, ) @@ -195,15 +198,36 @@ def create_app(settings: Settings | None = None) -> Flask: run = store.get(run_id) if run is None: abort(404) - return {"output": read_run_output(run)} + raw_offset = request.args.get("offset") + if raw_offset is None: + return {"output": read_run_output(run)[0]} + if not raw_offset.isdecimal() or len(raw_offset) > 19 or int(raw_offset) > 2**63 - 1: + abort(400) + output, offset, reset, has_more = read_run_output(run, int(raw_offset), 1024 * 1024) + return { + "output": output, "offset": offset, "reset": reset, "has_more": has_more, + "status": run.status, + } - def read_run_output(run) -> str: + def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]: if not run.log_path: - return "" + return "", 0, offset != 0, False try: - return Path(run.log_path).read_text(encoding="utf-8", errors="replace") + with Path(run.log_path).open("rb") as stream: + stream.seek(0, 2) + size = stream.tell() + reset = offset > size + if reset: + offset = 0 + stream.seek(offset) + data = stream.read() if maximum_bytes is None else stream.read(maximum_bytes) except FileNotFoundError: - return "" + return "", 0, offset != 0, False + has_more = offset + len(data) < size + decoder = codecs.getincrementaldecoder("utf-8")("replace") + output = decoder.decode(data, final=run.status not in {"queued", "running"} and not has_more) + pending, _ = decoder.getstate() + return output, offset + len(data) - len(pending), reset, has_more @app.get("/runs//artifacts") def run_artifacts(run_id: int): @@ -224,7 +248,7 @@ def create_app(settings: Settings | None = None) -> Flask: @app.get("/runs//artifacts//view") def view_artifact(run_id: int, filename: str): run = store.get(run_id) - if run is None or run.status != "succeeded" or not filename.casefold().endswith(".json"): + if run is None or run.status in {"queued", "running"} or not filename.casefold().endswith(".json"): abort(404) artifact = manager.artifact_path(run.id, filename) if artifact is None: @@ -234,7 +258,7 @@ def create_app(settings: Settings | None = None) -> Flask: @app.get("/runs//artifacts/") def download_artifact(run_id: int, filename: str): run = store.get(run_id) - if run is None or run.status != "succeeded": + if run is None or run.status in {"queued", "running"} or (run.status != "succeeded" and not filename.casefold().endswith(".json")): abort(404) artifact = manager.artifact_path(run.id, filename) if artifact is None: @@ -250,11 +274,11 @@ def create_app(settings: Settings | None = None) -> Flask: def delete_after_transfer(): try: yield from original_response + manager.delete_artifact(run.id, artifact.name) finally: close = getattr(original_response, "close", None) if close is not None: close() - manager.delete_artifact(run.id, artifact.name) response.response = delete_after_transfer() response.direct_passthrough = False diff --git a/python-tools/static/app.js b/python-tools/static/app.js index f51ece6..79e55bd 100644 --- a/python-tools/static/app.js +++ b/python-tools/static/app.js @@ -1,25 +1,33 @@ const output = document.querySelector("#run-output"); if (output) { - let refreshTimer; + let offset = Number(output.dataset.outputOffset || 0); let detailsFinalized = !["queued", "running"].includes( document.querySelector("#run-status")?.textContent.trim(), ); const refresh = async () => { - const response = await fetch(output.dataset.outputUrl, { cache: "no-store" }); - if (response.ok) { - const nextOutput = (await response.json()).output; - if (nextOutput !== output.textContent) { - output.textContent = nextOutput; + let delay = 2000; + try { + const url = new URL(output.dataset.outputUrl, window.location.href); + url.searchParams.set("offset", String(offset)); + const response = await fetch(url, { cache: "no-store" }); + if (!response.ok) { + return; } - } - const status = document.querySelector("#run-status"); - if (status) { - const stateResponse = await fetch(status.dataset.stateUrl, { cache: "no-store" }); - if (stateResponse.ok) { - const state = await stateResponse.json(); + const state = await response.json(); + if (state.reset) { + output.replaceChildren(); + } + if (state.output) { + output.append(document.createTextNode(state.output)); + } + offset = state.offset; + const status = document.querySelector("#run-status"); + if (status) { status.textContent = state.status; - if (state.status !== "queued" && state.status !== "running") { + if (state.has_more) { + delay = 0; + } else if (state.status !== "queued" && state.status !== "running") { if (!detailsFinalized) { const pageResponse = await fetch(window.location.href, { cache: "no-store" }); if (!pageResponse.ok) { @@ -36,13 +44,22 @@ if (output) { nextDetails.after(...page.querySelectorAll(actions)); detailsFinalized = true; } - window.clearInterval(refreshTimer); + const artifacts = document.querySelector("#run-artifacts"); + if (artifacts) { + await refreshArtifacts(artifacts); + } + delay = null; } } + } catch (error) { + console.warn("Could not refresh run details:", error); + } finally { + if (delay !== null) { + window.setTimeout(refresh, delay); + } } }; refresh(); - refreshTimer = window.setInterval(refresh, 2000); } const refreshArtifacts = async (container) => { @@ -73,8 +90,9 @@ const refreshArtifacts = async (container) => { }; for (const container of document.querySelectorAll("#run-artifacts")) { - refreshArtifacts(container); - window.setInterval(() => refreshArtifacts(container), 2000); + if (!output) { + refreshArtifacts(container); + } } for (const choice of document.querySelectorAll("button[data-execution-choice]")) { @@ -154,6 +172,7 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) { }; const waitForRun = async (run) => { + const started = Date.now(); while (true) { const response = await fetch(run.state_url, { cache: "no-store" }); if (!response.ok) { @@ -171,7 +190,7 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) { } return; } - await new Promise((resolve) => window.setTimeout(resolve, 500)); + await new Promise((resolve) => window.setTimeout(resolve, Date.now() - started < 10000 ? 500 : 2000)); } }; diff --git a/python-tools/templates/run.html b/python-tools/templates/run.html index eefe5f0..b3e0dc4 100644 --- a/python-tools/templates/run.html +++ b/python-tools/templates/run.html @@ -37,5 +37,5 @@

    Output

    -
    {{ output }}
    +
    {{ output }}
    {% endblock %} diff --git a/python-tools/tests/test_incremental_output.py b/python-tools/tests/test_incremental_output.py new file mode 100644 index 0000000..6c45322 --- /dev/null +++ b/python-tools/tests/test_incremental_output.py @@ -0,0 +1,87 @@ +import pytest + +from app.web import create_app +from .conftest import write_task +from .test_runner import wait_for, wait_for_output + + +@pytest.fixture +def output_run(settings): + app = create_app(settings) + store = app.extensions["run_store"] + run, _ = store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example") + store.claim_next("example") + path = settings.state_root / "test.log" + path.write_bytes(b"first\n") + with store._connect() as connection: + connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (str(path), run.id)) + return app.test_client(), run.id, path, store + + +def test_log_polling_returns_only_appended_bytes_and_status(output_run): + client, run_id, path, store = output_run + first = client.get(f"/runs/{run_id}/output?offset=0").get_json() + assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"} + with path.open("ab") as stream: + stream.write("next: \u20ac\n".encode()) + store.finish(run_id, status="succeeded") + next_chunk = client.get(f"/runs/{run_id}/output?offset=6").get_json() + assert next_chunk["output"] == "next: \u20ac\n" + assert next_chunk["offset"] == path.stat().st_size + assert next_chunk["status"] == "succeeded" + assert client.get(f"/runs/{run_id}/output").get_json() == {"output": "first\nnext: \u20ac\n"} + + +def test_incomplete_utf8_is_deferred_until_the_next_poll(output_run): + client, run_id, path, _ = output_run + path.write_bytes(b"ok\xe2\x82") + first = client.get(f"/runs/{run_id}/output?offset=0").get_json() + assert first["output"] == "ok" + assert first["offset"] == 2 + with path.open("ab") as stream: + stream.write(b"\xac") + second = client.get(f"/runs/{run_id}/output?offset=2").get_json() + assert second["output"] == "\u20ac" + assert second["offset"] == 5 + + +def test_truncated_log_resets_the_browser_offset(output_run): + client, run_id, path, _ = output_run + path.write_bytes(b"new") + result = client.get(f"/runs/{run_id}/output?offset=6").get_json() + assert result["reset"] is True + assert result["output"] == "new" + assert result["offset"] == 3 + + +def test_large_log_is_drained_in_bounded_chunks(output_run): + client, run_id, path, store = output_run + path.write_bytes(b"x" * (1024 * 1024 - 1) + "\u20ac".encode()) + store.finish(run_id, status="succeeded") + first = client.get(f"/runs/{run_id}/output?offset=0").get_json() + assert first["has_more"] is True + assert first["offset"] == 1024 * 1024 - 1 + second = client.get(f"/runs/{run_id}/output?offset={first['offset']}").get_json() + assert second["output"] == "\u20ac" + assert second["has_more"] is False + + +@pytest.mark.parametrize("offset", ["-1", "abc", "1.5", "9" * 100, str(2**63)]) +def test_invalid_log_offset_is_rejected(output_run, offset): + client, run_id, _, _ = output_run + assert client.get(f"/runs/{run_id}/output?offset={offset}").status_code == 400 + + +def test_running_task_output_is_visible_before_completion(settings): + write_task(settings.task_root, "example", "long.py", 'import time\nprint("working", flush=True)\ntime.sleep(30)\n') + app = create_app(settings) + manager = app.extensions["run_manager"] + run, _ = manager.run_task(app.extensions["catalog"].task("example/long.py")) + try: + wait_for_output(settings.state_root / "runs" / f"{run.id}.log", "working") + response = app.test_client().get(f"/runs/{run.id}/output?offset=0").get_json() + assert response["status"] == "running" + assert response["output"] == "working\n" + finally: + manager.request_cancel(run.id) + wait_for(manager, run.id, {"cancelled"}) diff --git a/python-tools/tests/test_runtime_cleanup.py b/python-tools/tests/test_runtime_cleanup.py new file mode 100644 index 0000000..5234c29 --- /dev/null +++ b/python-tools/tests/test_runtime_cleanup.py @@ -0,0 +1,95 @@ +import json +import os +import time +from datetime import UTC, datetime, timedelta + +from app.web import create_app +from .conftest import write_task +from .test_runner import wait_for + + +def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch): + run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example") + manager.store.claim_next("example") + root = manager.settings.state_root / "runs" + for name in (f"{run.id}.artifacts", f"{run.id}.uploads"): + path = root / name + path.mkdir(parents=True) + (path / "data").write_text("active") + os.utime(path, (time.time() - 3 * 86400,) * 2) + input_path = root / f"{run.id}.input.json" + input_path.write_text("{}") + monkeypatch.setattr(manager.store, "run_lifetimes", lambda: {}) + manager.cleanup_transient_files() + assert input_path.exists() + assert (root / f"{run.id}.artifacts/data").exists() + assert (root / f"{run.id}.uploads/data").exists() + + +def test_artifact_retention_starts_at_completion_not_directory_creation(manager): + run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example") + manager.store.finish(run.id, status="succeeded") + root = manager.settings.state_root / "runs" + artifact = root / f"{run.id}.artifacts" + artifact.mkdir(parents=True) + (artifact / "report.json").write_text("{}") + os.utime(artifact, (time.time() - 3 * 86400,) * 2) + (root / f"{run.id}.input.json").write_text("{}") + manager.cleanup_transient_files() + assert (artifact / "report.json").exists() + assert not (root / f"{run.id}.input.json").exists() + old = (datetime.now(UTC) - timedelta(days=3)).isoformat() + with manager.store._connect() as connection: + connection.execute("UPDATE runs SET finished_at = ? WHERE id = ?", (old, run.id)) + manager.cleanup_transient_files() + assert not artifact.exists() + + +def test_failed_task_keeps_valid_diagnostics_but_not_incomplete_downloads(manager): + write_task(manager.settings.task_root, "example", "fail.py", ''' +import os +from pathlib import Path +root = Path(os.environ["SERVER_MAINTENANCE_ARTIFACTS"]) +(root / "report.json").write_text('{"completed_steps": 2}') +(root / "broken.json").write_text('{') +(root / "partial.zip").write_bytes(b"partial") +raise SystemExit(1) +''') + task = manager.catalog.task("example/fail.py") + run, _ = manager.run_task(task) + wait_for(manager, run.id, {"failed"}) + assert manager.artifacts(run.id) == ["report.json"] + assert not (manager.settings.state_root / "runs" / f"{run.id}.input.json").exists() + + +def test_failed_json_report_is_accessible(settings): + app = create_app(settings) + manager = app.extensions["run_manager"] + run, _ = manager.store.enqueue(target_kind="task", target_path="example/fail.py", queue_group="example") + manager.store.finish(run.id, status="failed") + root = settings.state_root / "runs" / f"{run.id}.artifacts" + root.mkdir(parents=True) + (root / "report.json").write_text('{"completed_steps":2}') + client = app.test_client() + response = client.get(f"/runs/{run.id}/artifacts/report.json/view") + assert response.status_code == 200 + assert json.loads(response.data) == {"completed_steps": 2} + + +def test_aborted_artifact_download_can_be_retried(settings): + app = create_app(settings) + manager = app.extensions["run_manager"] + run, _ = manager.store.enqueue(target_kind="task", target_path="example/archive.py", queue_group="example") + manager.store.finish(run.id, status="succeeded") + root = settings.state_root / "runs" / f"{run.id}.artifacts" + root.mkdir(parents=True) + path = root / "review.zip" + path.write_bytes(b"x" * 100_000) + client = app.test_client() + response = client.get(f"/runs/{run.id}/artifacts/review.zip", buffered=False) + next(iter(response.response)) + response.close() + assert path.exists() + retry = client.get(f"/runs/{run.id}/artifacts/review.zip") + assert len(retry.data) == 100_000 + assert not path.exists()