Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-10-29 22:12:04 +00:00
1 parent 034fecbd51
commit 3bfd199b9c
28 files changed
+2356 -172

No files matched your search

+142
View File
@@ -0,0 +1,142 @@
"""Web contracts for restoring live task state and rejecting resource conflicts."""
import json
import pytest
from app.runner import RunManager
from app.scheduler import BackgroundScheduler
from app.web import create_app
from .conftest import write_task
@pytest.fixture
def live_app(settings, monkeypatch):
for group, filename, source in (
("example", "long.py", 'RESOURCES = ("shared-data",)\n'),
("example", "sibling.py", ""),
("related", "change.py", 'RESOURCES = ("shared-data",)\n'),
("independent", "free.py", ""),
):
write_task(settings.task_root, group, filename, source + 'print("done")\n')
monkeypatch.setattr(RunManager, "_wake_worker", lambda *_: None)
app = create_app(settings)
yield app
app.extensions["background_scheduler"].stop()
def start_live_run(app, *, status="running", recovery=False):
manager = app.extensions["run_manager"]
task = app.extensions["catalog"].task("example/long.py")
run, _ = manager.run_task(task)
if recovery:
old = manager.store.finish(run.id, status="interrupted")
run, _ = manager.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
resources=task.resources, recovery_of_run_id=old.id,
)
if status == "running":
run = manager.store.claim_next(task.group_id)
progress = manager.settings.state_root / "runs" / f"{run.id}.progress.json"
progress.parent.mkdir(parents=True, exist_ok=True)
progress.write_text(json.dumps({"text": "2 of 5\nETA: 3 minutes"}))
return run
@pytest.mark.parametrize("status", ["queued", "running"])
def test_task_states_restore_active_conflicts_and_progress(live_app, status):
run = start_live_run(live_app, status=status)
client = live_app.test_client()
response = client.get("/tasks/state")
assert response.status_code == 200
assert response.headers["Cache-Control"] == "no-store"
states = response.get_json()
own = states["tasks"]["example/long.py"]
assert own["active_run"]["run_id"] == run.id
assert own["active_run"]["display_status"] == status
assert own["progress"] == "2 of 5\nETA: 3 minutes"
assert own["blocked_by"] is None
for name in ("example/sibling.py", "related/change.py"):
state = states["tasks"][name]
assert state["active_run"] is None
assert state["blocked_by"]["detail_url"] == f"/runs/{run.id}"
assert states["tasks"]["independent/free.py"] == {
"active_run": None, "blocked_by": None, "progress": "",
}
assert states["groups"]["related"]["blocked_by"]["run_id"] == run.id
assert not {"input_json", "resources_json", "log_path"}.intersection(own["active_run"])
page = client.get("/")
assert b'fieldset class="task-controls" disabled' in page.data
assert b"2 of 5\nETA: 3 minutes" in page.data
assert f'Run #{run.id}'.encode() in page.data
assert client.post("/tasks/state").status_code == 405
@pytest.mark.parametrize("status", ["queued", "running"])
def test_recovery_links_and_live_run_progress(live_app, status):
run = start_live_run(live_app, status=status, recovery=True)
client = live_app.test_client()
display_status = "Recovery queued" if status == "queued" else "recovering"
state = client.get(f"/runs/{run.id}/state").get_json()
assert state["status"] == status
assert state["display_status"] == display_status
assert state["recovery_of_run_id"] == run.recovery_of_run_id
assert state["progress"] == "2 of 5\nETA: 3 minutes"
output = client.get(f"/runs/{run.id}/output?offset=0").get_json()
assert output["progress"] == state["progress"]
assert output["display_status"] == display_status
detail = client.get(f"/runs/{run.id}")
assert f'href="/runs/{run.recovery_of_run_id}"'.encode() in detail.data
assert b'id="run-progress" class="run-progress">2 of 5\nETA: 3 minutes' in detail.data
old_detail = client.get(f"/runs/{run.recovery_of_run_id}")
assert b"Recovery run" in old_detail.data
assert f'href="/runs/{run.id}"'.encode() in old_detail.data
old_state = client.get(f"/runs/{run.recovery_of_run_id}/state").get_json()
assert old_state["recovery_run"]["run_id"] == run.id
assert b"recovery of" in client.get("/history/task/example/long.py").data
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
@pytest.mark.parametrize("target", ["/tasks/related/change.py/run", "/groups/related/run"])
def test_conflicting_submissions_return_409_with_blocking_run(live_app, accept, target):
run = start_live_run(live_app)
response = live_app.test_client().post(target, headers={"Accept": accept})
assert response.status_code == 409
if accept == "application/json":
payload = response.get_json()
assert payload["blocked_by"]["run_id"] == run.id
assert payload["blocked_by"]["detail_url"] == f"/runs/{run.id}"
assert f"#{run.id}" in payload["error"]
else:
assert f'href="/runs/{run.id}"'.encode() in response.data
assert live_app.extensions["run_store"].last_run("task", "related/change.py") is None
unrelated = live_app.test_client().post("/tasks/independent/free.py/run", headers={"Accept": accept})
assert unrelated.status_code == (202 if accept == "application/json" else 302)
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
@pytest.mark.parametrize("target", ["/tasks/example/long.py/run", "/groups/example/run"])
def test_shutdown_is_reported_without_enqueuing(live_app, accept, target):
live_app.extensions["run_manager"]._closing.set()
response = live_app.test_client().post(target, headers={"Accept": accept})
assert response.status_code == 503
assert b"shutting down" in response.data
def test_startup_precedes_cleanup_and_scheduler(settings, monkeypatch):
calls = []
monkeypatch.setattr(RunManager, "startup", lambda *_: calls.append("startup"))
monkeypatch.setattr(RunManager, "cleanup_transient_files", lambda *_: calls.append("cleanup"))
monkeypatch.setattr(BackgroundScheduler, "start", lambda *_: calls.append("scheduler"))
create_app(settings)
assert calls == ["startup", "cleanup", "scheduler"]
def test_progress_is_plain_text_and_escaped(live_app):
run = start_live_run(live_app)
path = live_app.config["settings"].state_root / "runs" / f"{run.id}.progress.json"
path.write_text(json.dumps({"text": "<script>not HTML</script>\nNext item"}))
for url in ("/", f"/runs/{run.id}"):
page = live_app.test_client().get(url)
assert b"&lt;script&gt;not HTML&lt;/script&gt;\nNext item" in page.data