Protect task diagnostics and stream incremental run output
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 47s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m24s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 38s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 43s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 47s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m24s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 38s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 43s
This commit is contained in:
1 parent
0aa67439fd
commit
55bbbe2fd7
8 files changed
+319
-51
No files matched your search
+12
-6
@@ -8,9 +8,9 @@ with `python -m app`.
|
|||||||
|
|
||||||
The application discovers direct `*.py` files in each non-hidden directory
|
The application discovers direct `*.py` files in each non-hidden directory
|
||||||
under `/opt/tasks`. Each directory is a task group. Tasks in one group run
|
under `/opt/tasks`. Each directory is a task group. Tasks in one group run
|
||||||
sequentially; different groups may run concurrently. It records run history and
|
sequentially; different groups may run concurrently. It records run history in
|
||||||
combined output in SQLite under `/var/lib/server-maintenance`, and supports
|
SQLite and combined output in log files under `/var/lib/server-maintenance`,
|
||||||
cancellation of active task process groups.
|
and supports cancellation of active task process groups.
|
||||||
|
|
||||||
The image contains application code only. Mount these paths at runtime:
|
The image contains application code only. Mount these paths at runtime:
|
||||||
|
|
||||||
@@ -77,6 +77,7 @@ readable. `description` is optional on a task and otherwise comes from the
|
|||||||
first non-empty line of its module docstring. The order of keys in `tasks`
|
first non-empty line of its module docstring. The order of keys in `tasks`
|
||||||
controls the listed tasks' display order. Include a task as `{}` when only
|
controls the listed tasks' display order. Include a task as `{}` when only
|
||||||
ordering is needed. Unlisted direct Python files follow alphabetically.
|
ordering is needed. Unlisted direct Python files follow alphabetically.
|
||||||
|
Use `\n` for a line break or `\n\n` for a paragraph gap in descriptions.
|
||||||
|
|
||||||
Each task declaration may contain `title`, `description`, `inputs`,
|
Each task declaration may contain `title`, `description`, `inputs`,
|
||||||
`wait_for_result`, `download_artifacts`, and `execution`. Set
|
`wait_for_result`, `download_artifacts`, and `execution`. Set
|
||||||
@@ -143,9 +144,14 @@ Tasks can write files to the directory named by
|
|||||||
`SERVER_MAINTENANCE_ARTIFACTS`. Run-detail pages show them when the task
|
`SERVER_MAINTENANCE_ARTIFACTS`. Run-detail pages show them when the task
|
||||||
finishes. JSON artifacts offer an **Open** link that renders the JSON in a new
|
finishes. JSON artifacts offer an **Open** link that renders the JSON in a new
|
||||||
tab and leaves the artifact available; other artifacts offer a download link.
|
tab and leaves the artifact available; other artifacts offer a download link.
|
||||||
The server deletes an artifact after its download transfer ends; any artifact
|
The server deletes an artifact after its download stream completes; interrupted
|
||||||
not downloaded is removed after 24 hours. A browser cannot confirm that a
|
downloads remain available for retry. Active runs' artifacts never expire.
|
||||||
downloaded file was retained on the client device.
|
After a run finishes, artifacts expire after 24 hours. Failed or cancelled
|
||||||
|
runs keep valid JSON diagnostic reports, but discard incomplete downloads.
|
||||||
|
Ordinary input files are removed when execution ends; saved inputs remain in
|
||||||
|
SQLite for history and review replay. Run pages fetch only appended log bytes.
|
||||||
|
|
||||||
|
A browser cannot confirm that a downloaded file was retained on the client device.
|
||||||
|
|
||||||
## Background tasks
|
## Background tasks
|
||||||
|
|
||||||
|
|||||||
+43
-17
@@ -10,6 +10,7 @@ import subprocess
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .config import Settings
|
from .config import Settings
|
||||||
@@ -127,31 +128,54 @@ class RunManager:
|
|||||||
runs_root = self.settings.state_root / "runs"
|
runs_root = self.settings.state_root / "runs"
|
||||||
cutoff = time.time() - artifact_max_age_seconds
|
cutoff = time.time() - artifact_max_age_seconds
|
||||||
if runs_root.is_dir():
|
if runs_root.is_dir():
|
||||||
|
lifetimes = self.store.run_lifetimes()
|
||||||
for path in runs_root.iterdir():
|
for path in runs_root.iterdir():
|
||||||
if not path.is_dir():
|
|
||||||
continue
|
|
||||||
run_id_text, separator, kind = path.name.partition(".")
|
run_id_text, separator, kind = path.name.partition(".")
|
||||||
if not separator or not run_id_text.isdigit() or kind not in {"uploads", "artifacts"}:
|
if not separator or kind not in {"uploads", "artifacts", "input.json", "execution-input.json"} or not run_id_text.isdigit() or path.is_symlink():
|
||||||
continue
|
continue
|
||||||
run = self.store.get(int(run_id_text))
|
run_id = int(run_id_text)
|
||||||
if kind == "uploads" and (run is None or run.status not in {"queued", "running"}):
|
if run_id not in lifetimes:
|
||||||
|
# A worker may have created this run after the bulk snapshot.
|
||||||
|
run = self.store.get(run_id)
|
||||||
|
lifetimes[run_id] = (run.status, run.finished_at) if run else (None, None)
|
||||||
|
status, finished_at = lifetimes[run_id]
|
||||||
|
if status in {"queued", "running"}:
|
||||||
|
continue
|
||||||
|
if kind in {"input.json", "execution-input.json"} and path.is_file():
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
elif kind == "uploads" and path.is_dir():
|
||||||
shutil.rmtree(path, ignore_errors=True)
|
shutil.rmtree(path, ignore_errors=True)
|
||||||
elif kind == "artifacts":
|
elif kind == "artifacts" and path.is_dir():
|
||||||
|
if status in {"failed", "interrupted", "cancelled"}:
|
||||||
|
self._keep_diagnostic_reports(path)
|
||||||
try:
|
try:
|
||||||
expired = path.stat().st_mtime < cutoff
|
completed_at = datetime.fromisoformat(finished_at).timestamp() if finished_at else path.stat().st_mtime
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
continue
|
continue
|
||||||
if expired:
|
if completed_at < cutoff:
|
||||||
shutil.rmtree(path, ignore_errors=True)
|
shutil.rmtree(path, ignore_errors=True)
|
||||||
for path in runs_root.glob("*.execution-input.json"):
|
|
||||||
run_id_text = path.name.removesuffix(".execution-input.json")
|
|
||||||
if not run_id_text.isdigit():
|
|
||||||
continue
|
|
||||||
run = self.store.get(int(run_id_text))
|
|
||||||
if run is None or run.status not in {"queued", "running"}:
|
|
||||||
path.unlink(missing_ok=True)
|
|
||||||
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
|
self._cleanup_orphaned_task_state(orphaned_task_state_retention_seconds)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _keep_diagnostic_reports(root: Path) -> None:
|
||||||
|
try:
|
||||||
|
paths = list(root.iterdir())
|
||||||
|
except FileNotFoundError:
|
||||||
|
return
|
||||||
|
for path in paths:
|
||||||
|
if path.is_file() and not path.is_symlink() and path.suffix.casefold() == ".json":
|
||||||
|
try:
|
||||||
|
json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
continue
|
||||||
|
except FileNotFoundError:
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
if path.is_dir() and not path.is_symlink():
|
||||||
|
shutil.rmtree(path)
|
||||||
|
else:
|
||||||
|
path.unlink(missing_ok=True)
|
||||||
|
|
||||||
def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None:
|
def _cleanup_orphaned_task_state(self, retention_seconds: int) -> None:
|
||||||
"""Delete state only after a group remains absent for a grace period."""
|
"""Delete state only after a group remains absent for a grace period."""
|
||||||
|
|
||||||
@@ -318,6 +342,7 @@ class RunManager:
|
|||||||
environment = self._environment(task.group_id)
|
environment = self._environment(task.group_id)
|
||||||
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
|
environment["SERVER_MAINTENANCE_INPUT"] = str(input_path)
|
||||||
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
|
environment["SERVER_MAINTENANCE_ARTIFACTS"] = str(artifact_root)
|
||||||
|
self.store.set_log_path(run.id, str(log_path))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with log_path.open("wb") as log_file:
|
with log_path.open("wb") as log_file:
|
||||||
@@ -347,11 +372,12 @@ class RunManager:
|
|||||||
self._processes.pop(run.id, None)
|
self._processes.pop(run.id, None)
|
||||||
self._delete_uploads(run.id)
|
self._delete_uploads(run.id)
|
||||||
self._delete_execution_input(run.id)
|
self._delete_execution_input(run.id)
|
||||||
|
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||||
|
|
||||||
if self.store.is_cancel_requested(run.id) or (
|
if self.store.is_cancel_requested(run.id) or (
|
||||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||||
):
|
):
|
||||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
self._keep_diagnostic_reports(artifact_root)
|
||||||
return self.store.finish(
|
return self.store.finish(
|
||||||
run.id,
|
run.id,
|
||||||
status="cancelled",
|
status="cancelled",
|
||||||
@@ -365,7 +391,7 @@ class RunManager:
|
|||||||
log_path=str(log_path),
|
log_path=str(log_path),
|
||||||
)
|
)
|
||||||
if completed.status != "succeeded":
|
if completed.status != "succeeded":
|
||||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
self._keep_diagnostic_reports(artifact_root)
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
artifact_root.rmdir()
|
artifact_root.rmdir()
|
||||||
|
|||||||
@@ -256,6 +256,17 @@ class RunStore:
|
|||||||
with self._connect() as owned_connection:
|
with self._connect() as owned_connection:
|
||||||
return self.get(run_id, connection=owned_connection)
|
return self.get(run_id, connection=owned_connection)
|
||||||
|
|
||||||
|
def run_lifetimes(self) -> dict[int, tuple[str, str | None]]:
|
||||||
|
with self._connect() as connection:
|
||||||
|
return {
|
||||||
|
row["id"]: (row["status"], row["finished_at"])
|
||||||
|
for row in connection.execute("SELECT id, status, finished_at FROM runs")
|
||||||
|
}
|
||||||
|
|
||||||
|
def set_log_path(self, run_id: int, path: str) -> None:
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (path, run_id))
|
||||||
|
|
||||||
def last_run(self, target_kind: str, target_path: str) -> Run | None:
|
def last_run(self, target_kind: str, target_path: str) -> Run | None:
|
||||||
with self._connect() as connection:
|
with self._connect() as connection:
|
||||||
row = connection.execute(
|
row = connection.execute(
|
||||||
|
|||||||
+33
-9
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import codecs
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
@@ -137,11 +138,13 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
if run is None:
|
if run is None:
|
||||||
abort(404)
|
abort(404)
|
||||||
execution_task = execution_replay_task(run)
|
execution_task = execution_replay_task(run)
|
||||||
|
output, offset, _, _ = read_run_output(run)
|
||||||
return render_template(
|
return render_template(
|
||||||
"run.html",
|
"run.html",
|
||||||
run=run,
|
run=run,
|
||||||
children=store.children(run.id),
|
children=store.children(run.id),
|
||||||
output=read_run_output(run),
|
output=output,
|
||||||
|
output_offset=offset,
|
||||||
execution_task=execution_task,
|
execution_task=execution_task,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -195,15 +198,36 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
run = store.get(run_id)
|
run = store.get(run_id)
|
||||||
if run is None:
|
if run is None:
|
||||||
abort(404)
|
abort(404)
|
||||||
return {"output": read_run_output(run)}
|
raw_offset = request.args.get("offset")
|
||||||
|
if raw_offset is None:
|
||||||
|
return {"output": read_run_output(run)[0]}
|
||||||
|
if not raw_offset.isdecimal() or len(raw_offset) > 19 or int(raw_offset) > 2**63 - 1:
|
||||||
|
abort(400)
|
||||||
|
output, offset, reset, has_more = read_run_output(run, int(raw_offset), 1024 * 1024)
|
||||||
|
return {
|
||||||
|
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
|
||||||
|
"status": run.status,
|
||||||
|
}
|
||||||
|
|
||||||
def read_run_output(run) -> str:
|
def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
|
||||||
if not run.log_path:
|
if not run.log_path:
|
||||||
return ""
|
return "", 0, offset != 0, False
|
||||||
try:
|
try:
|
||||||
return Path(run.log_path).read_text(encoding="utf-8", errors="replace")
|
with Path(run.log_path).open("rb") as stream:
|
||||||
|
stream.seek(0, 2)
|
||||||
|
size = stream.tell()
|
||||||
|
reset = offset > size
|
||||||
|
if reset:
|
||||||
|
offset = 0
|
||||||
|
stream.seek(offset)
|
||||||
|
data = stream.read() if maximum_bytes is None else stream.read(maximum_bytes)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
return ""
|
return "", 0, offset != 0, False
|
||||||
|
has_more = offset + len(data) < size
|
||||||
|
decoder = codecs.getincrementaldecoder("utf-8")("replace")
|
||||||
|
output = decoder.decode(data, final=run.status not in {"queued", "running"} and not has_more)
|
||||||
|
pending, _ = decoder.getstate()
|
||||||
|
return output, offset + len(data) - len(pending), reset, has_more
|
||||||
|
|
||||||
@app.get("/runs/<int:run_id>/artifacts")
|
@app.get("/runs/<int:run_id>/artifacts")
|
||||||
def run_artifacts(run_id: int):
|
def run_artifacts(run_id: int):
|
||||||
@@ -224,7 +248,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
@app.get("/runs/<int:run_id>/artifacts/<filename>/view")
|
@app.get("/runs/<int:run_id>/artifacts/<filename>/view")
|
||||||
def view_artifact(run_id: int, filename: str):
|
def view_artifact(run_id: int, filename: str):
|
||||||
run = store.get(run_id)
|
run = store.get(run_id)
|
||||||
if run is None or run.status != "succeeded" or not filename.casefold().endswith(".json"):
|
if run is None or run.status in {"queued", "running"} or not filename.casefold().endswith(".json"):
|
||||||
abort(404)
|
abort(404)
|
||||||
artifact = manager.artifact_path(run.id, filename)
|
artifact = manager.artifact_path(run.id, filename)
|
||||||
if artifact is None:
|
if artifact is None:
|
||||||
@@ -234,7 +258,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
@app.get("/runs/<int:run_id>/artifacts/<filename>")
|
@app.get("/runs/<int:run_id>/artifacts/<filename>")
|
||||||
def download_artifact(run_id: int, filename: str):
|
def download_artifact(run_id: int, filename: str):
|
||||||
run = store.get(run_id)
|
run = store.get(run_id)
|
||||||
if run is None or run.status != "succeeded":
|
if run is None or run.status in {"queued", "running"} or (run.status != "succeeded" and not filename.casefold().endswith(".json")):
|
||||||
abort(404)
|
abort(404)
|
||||||
artifact = manager.artifact_path(run.id, filename)
|
artifact = manager.artifact_path(run.id, filename)
|
||||||
if artifact is None:
|
if artifact is None:
|
||||||
@@ -250,11 +274,11 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
def delete_after_transfer():
|
def delete_after_transfer():
|
||||||
try:
|
try:
|
||||||
yield from original_response
|
yield from original_response
|
||||||
|
manager.delete_artifact(run.id, artifact.name)
|
||||||
finally:
|
finally:
|
||||||
close = getattr(original_response, "close", None)
|
close = getattr(original_response, "close", None)
|
||||||
if close is not None:
|
if close is not None:
|
||||||
close()
|
close()
|
||||||
manager.delete_artifact(run.id, artifact.name)
|
|
||||||
|
|
||||||
response.response = delete_after_transfer()
|
response.response = delete_after_transfer()
|
||||||
response.direct_passthrough = False
|
response.direct_passthrough = False
|
||||||
|
|||||||
+37
-18
@@ -1,25 +1,33 @@
|
|||||||
const output = document.querySelector("#run-output");
|
const output = document.querySelector("#run-output");
|
||||||
|
|
||||||
if (output) {
|
if (output) {
|
||||||
let refreshTimer;
|
let offset = Number(output.dataset.outputOffset || 0);
|
||||||
let detailsFinalized = !["queued", "running"].includes(
|
let detailsFinalized = !["queued", "running"].includes(
|
||||||
document.querySelector("#run-status")?.textContent.trim(),
|
document.querySelector("#run-status")?.textContent.trim(),
|
||||||
);
|
);
|
||||||
const refresh = async () => {
|
const refresh = async () => {
|
||||||
const response = await fetch(output.dataset.outputUrl, { cache: "no-store" });
|
let delay = 2000;
|
||||||
if (response.ok) {
|
try {
|
||||||
const nextOutput = (await response.json()).output;
|
const url = new URL(output.dataset.outputUrl, window.location.href);
|
||||||
if (nextOutput !== output.textContent) {
|
url.searchParams.set("offset", String(offset));
|
||||||
output.textContent = nextOutput;
|
const response = await fetch(url, { cache: "no-store" });
|
||||||
|
if (!response.ok) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
const state = await response.json();
|
||||||
const status = document.querySelector("#run-status");
|
if (state.reset) {
|
||||||
if (status) {
|
output.replaceChildren();
|
||||||
const stateResponse = await fetch(status.dataset.stateUrl, { cache: "no-store" });
|
}
|
||||||
if (stateResponse.ok) {
|
if (state.output) {
|
||||||
const state = await stateResponse.json();
|
output.append(document.createTextNode(state.output));
|
||||||
|
}
|
||||||
|
offset = state.offset;
|
||||||
|
const status = document.querySelector("#run-status");
|
||||||
|
if (status) {
|
||||||
status.textContent = state.status;
|
status.textContent = state.status;
|
||||||
if (state.status !== "queued" && state.status !== "running") {
|
if (state.has_more) {
|
||||||
|
delay = 0;
|
||||||
|
} else if (state.status !== "queued" && state.status !== "running") {
|
||||||
if (!detailsFinalized) {
|
if (!detailsFinalized) {
|
||||||
const pageResponse = await fetch(window.location.href, { cache: "no-store" });
|
const pageResponse = await fetch(window.location.href, { cache: "no-store" });
|
||||||
if (!pageResponse.ok) {
|
if (!pageResponse.ok) {
|
||||||
@@ -36,13 +44,22 @@ if (output) {
|
|||||||
nextDetails.after(...page.querySelectorAll(actions));
|
nextDetails.after(...page.querySelectorAll(actions));
|
||||||
detailsFinalized = true;
|
detailsFinalized = true;
|
||||||
}
|
}
|
||||||
window.clearInterval(refreshTimer);
|
const artifacts = document.querySelector("#run-artifacts");
|
||||||
|
if (artifacts) {
|
||||||
|
await refreshArtifacts(artifacts);
|
||||||
|
}
|
||||||
|
delay = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("Could not refresh run details:", error);
|
||||||
|
} finally {
|
||||||
|
if (delay !== null) {
|
||||||
|
window.setTimeout(refresh, delay);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
refresh();
|
refresh();
|
||||||
refreshTimer = window.setInterval(refresh, 2000);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const refreshArtifacts = async (container) => {
|
const refreshArtifacts = async (container) => {
|
||||||
@@ -73,8 +90,9 @@ const refreshArtifacts = async (container) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
for (const container of document.querySelectorAll("#run-artifacts")) {
|
for (const container of document.querySelectorAll("#run-artifacts")) {
|
||||||
refreshArtifacts(container);
|
if (!output) {
|
||||||
window.setInterval(() => refreshArtifacts(container), 2000);
|
refreshArtifacts(container);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const choice of document.querySelectorAll("button[data-execution-choice]")) {
|
for (const choice of document.querySelectorAll("button[data-execution-choice]")) {
|
||||||
@@ -154,6 +172,7 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const waitForRun = async (run) => {
|
const waitForRun = async (run) => {
|
||||||
|
const started = Date.now();
|
||||||
while (true) {
|
while (true) {
|
||||||
const response = await fetch(run.state_url, { cache: "no-store" });
|
const response = await fetch(run.state_url, { cache: "no-store" });
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
@@ -171,7 +190,7 @@ for (const form of document.querySelectorAll("form[data-wait-for-result]")) {
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await new Promise((resolve) => window.setTimeout(resolve, 500));
|
await new Promise((resolve) => window.setTimeout(resolve, Date.now() - started < 10000 ? 500 : 2000));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -37,5 +37,5 @@
|
|||||||
<ul></ul>
|
<ul></ul>
|
||||||
</section>
|
</section>
|
||||||
<h2>Output</h2>
|
<h2>Output</h2>
|
||||||
<pre id="run-output" data-output-url="{{ url_for('run_output', run_id=run.id) }}">{{ output }}</pre>
|
<pre id="run-output" data-output-url="{{ url_for('run_output', run_id=run.id) }}" data-output-offset="{{ output_offset }}">{{ output }}</pre>
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.web import create_app
|
||||||
|
from .conftest import write_task
|
||||||
|
from .test_runner import wait_for, wait_for_output
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def output_run(settings):
|
||||||
|
app = create_app(settings)
|
||||||
|
store = app.extensions["run_store"]
|
||||||
|
run, _ = store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
|
||||||
|
store.claim_next("example")
|
||||||
|
path = settings.state_root / "test.log"
|
||||||
|
path.write_bytes(b"first\n")
|
||||||
|
with store._connect() as connection:
|
||||||
|
connection.execute("UPDATE runs SET log_path = ? WHERE id = ?", (str(path), run.id))
|
||||||
|
return app.test_client(), run.id, path, store
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_polling_returns_only_appended_bytes_and_status(output_run):
|
||||||
|
client, run_id, path, store = output_run
|
||||||
|
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
|
||||||
|
assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"}
|
||||||
|
with path.open("ab") as stream:
|
||||||
|
stream.write("next: \u20ac\n".encode())
|
||||||
|
store.finish(run_id, status="succeeded")
|
||||||
|
next_chunk = client.get(f"/runs/{run_id}/output?offset=6").get_json()
|
||||||
|
assert next_chunk["output"] == "next: \u20ac\n"
|
||||||
|
assert next_chunk["offset"] == path.stat().st_size
|
||||||
|
assert next_chunk["status"] == "succeeded"
|
||||||
|
assert client.get(f"/runs/{run_id}/output").get_json() == {"output": "first\nnext: \u20ac\n"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_incomplete_utf8_is_deferred_until_the_next_poll(output_run):
|
||||||
|
client, run_id, path, _ = output_run
|
||||||
|
path.write_bytes(b"ok\xe2\x82")
|
||||||
|
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
|
||||||
|
assert first["output"] == "ok"
|
||||||
|
assert first["offset"] == 2
|
||||||
|
with path.open("ab") as stream:
|
||||||
|
stream.write(b"\xac")
|
||||||
|
second = client.get(f"/runs/{run_id}/output?offset=2").get_json()
|
||||||
|
assert second["output"] == "\u20ac"
|
||||||
|
assert second["offset"] == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_truncated_log_resets_the_browser_offset(output_run):
|
||||||
|
client, run_id, path, _ = output_run
|
||||||
|
path.write_bytes(b"new")
|
||||||
|
result = client.get(f"/runs/{run_id}/output?offset=6").get_json()
|
||||||
|
assert result["reset"] is True
|
||||||
|
assert result["output"] == "new"
|
||||||
|
assert result["offset"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_large_log_is_drained_in_bounded_chunks(output_run):
|
||||||
|
client, run_id, path, store = output_run
|
||||||
|
path.write_bytes(b"x" * (1024 * 1024 - 1) + "\u20ac".encode())
|
||||||
|
store.finish(run_id, status="succeeded")
|
||||||
|
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
|
||||||
|
assert first["has_more"] is True
|
||||||
|
assert first["offset"] == 1024 * 1024 - 1
|
||||||
|
second = client.get(f"/runs/{run_id}/output?offset={first['offset']}").get_json()
|
||||||
|
assert second["output"] == "\u20ac"
|
||||||
|
assert second["has_more"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("offset", ["-1", "abc", "1.5", "9" * 100, str(2**63)])
|
||||||
|
def test_invalid_log_offset_is_rejected(output_run, offset):
|
||||||
|
client, run_id, _, _ = output_run
|
||||||
|
assert client.get(f"/runs/{run_id}/output?offset={offset}").status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_running_task_output_is_visible_before_completion(settings):
|
||||||
|
write_task(settings.task_root, "example", "long.py", 'import time\nprint("working", flush=True)\ntime.sleep(30)\n')
|
||||||
|
app = create_app(settings)
|
||||||
|
manager = app.extensions["run_manager"]
|
||||||
|
run, _ = manager.run_task(app.extensions["catalog"].task("example/long.py"))
|
||||||
|
try:
|
||||||
|
wait_for_output(settings.state_root / "runs" / f"{run.id}.log", "working")
|
||||||
|
response = app.test_client().get(f"/runs/{run.id}/output?offset=0").get_json()
|
||||||
|
assert response["status"] == "running"
|
||||||
|
assert response["output"] == "working\n"
|
||||||
|
finally:
|
||||||
|
manager.request_cancel(run.id)
|
||||||
|
wait_for(manager, run.id, {"cancelled"})
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from app.web import create_app
|
||||||
|
from .conftest import write_task
|
||||||
|
from .test_runner import wait_for
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
|
||||||
|
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
|
||||||
|
manager.store.claim_next("example")
|
||||||
|
root = manager.settings.state_root / "runs"
|
||||||
|
for name in (f"{run.id}.artifacts", f"{run.id}.uploads"):
|
||||||
|
path = root / name
|
||||||
|
path.mkdir(parents=True)
|
||||||
|
(path / "data").write_text("active")
|
||||||
|
os.utime(path, (time.time() - 3 * 86400,) * 2)
|
||||||
|
input_path = root / f"{run.id}.input.json"
|
||||||
|
input_path.write_text("{}")
|
||||||
|
monkeypatch.setattr(manager.store, "run_lifetimes", lambda: {})
|
||||||
|
manager.cleanup_transient_files()
|
||||||
|
assert input_path.exists()
|
||||||
|
assert (root / f"{run.id}.artifacts/data").exists()
|
||||||
|
assert (root / f"{run.id}.uploads/data").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_retention_starts_at_completion_not_directory_creation(manager):
|
||||||
|
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
|
||||||
|
manager.store.finish(run.id, status="succeeded")
|
||||||
|
root = manager.settings.state_root / "runs"
|
||||||
|
artifact = root / f"{run.id}.artifacts"
|
||||||
|
artifact.mkdir(parents=True)
|
||||||
|
(artifact / "report.json").write_text("{}")
|
||||||
|
os.utime(artifact, (time.time() - 3 * 86400,) * 2)
|
||||||
|
(root / f"{run.id}.input.json").write_text("{}")
|
||||||
|
manager.cleanup_transient_files()
|
||||||
|
assert (artifact / "report.json").exists()
|
||||||
|
assert not (root / f"{run.id}.input.json").exists()
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=3)).isoformat()
|
||||||
|
with manager.store._connect() as connection:
|
||||||
|
connection.execute("UPDATE runs SET finished_at = ? WHERE id = ?", (old, run.id))
|
||||||
|
manager.cleanup_transient_files()
|
||||||
|
assert not artifact.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_task_keeps_valid_diagnostics_but_not_incomplete_downloads(manager):
|
||||||
|
write_task(manager.settings.task_root, "example", "fail.py", '''
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
root = Path(os.environ["SERVER_MAINTENANCE_ARTIFACTS"])
|
||||||
|
(root / "report.json").write_text('{"completed_steps": 2}')
|
||||||
|
(root / "broken.json").write_text('{')
|
||||||
|
(root / "partial.zip").write_bytes(b"partial")
|
||||||
|
raise SystemExit(1)
|
||||||
|
''')
|
||||||
|
task = manager.catalog.task("example/fail.py")
|
||||||
|
run, _ = manager.run_task(task)
|
||||||
|
wait_for(manager, run.id, {"failed"})
|
||||||
|
assert manager.artifacts(run.id) == ["report.json"]
|
||||||
|
assert not (manager.settings.state_root / "runs" / f"{run.id}.input.json").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_json_report_is_accessible(settings):
|
||||||
|
app = create_app(settings)
|
||||||
|
manager = app.extensions["run_manager"]
|
||||||
|
run, _ = manager.store.enqueue(target_kind="task", target_path="example/fail.py", queue_group="example")
|
||||||
|
manager.store.finish(run.id, status="failed")
|
||||||
|
root = settings.state_root / "runs" / f"{run.id}.artifacts"
|
||||||
|
root.mkdir(parents=True)
|
||||||
|
(root / "report.json").write_text('{"completed_steps":2}')
|
||||||
|
client = app.test_client()
|
||||||
|
response = client.get(f"/runs/{run.id}/artifacts/report.json/view")
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert json.loads(response.data) == {"completed_steps": 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_aborted_artifact_download_can_be_retried(settings):
|
||||||
|
app = create_app(settings)
|
||||||
|
manager = app.extensions["run_manager"]
|
||||||
|
run, _ = manager.store.enqueue(target_kind="task", target_path="example/archive.py", queue_group="example")
|
||||||
|
manager.store.finish(run.id, status="succeeded")
|
||||||
|
root = settings.state_root / "runs" / f"{run.id}.artifacts"
|
||||||
|
root.mkdir(parents=True)
|
||||||
|
path = root / "review.zip"
|
||||||
|
path.write_bytes(b"x" * 100_000)
|
||||||
|
client = app.test_client()
|
||||||
|
response = client.get(f"/runs/{run.id}/artifacts/review.zip", buffered=False)
|
||||||
|
next(iter(response.response))
|
||||||
|
response.close()
|
||||||
|
assert path.exists()
|
||||||
|
retry = client.get(f"/runs/{run.id}/artifacts/review.zip")
|
||||||
|
assert len(retry.data) == 100_000
|
||||||
|
assert not path.exists()
|
||||||
Reference in new issue
Block a user