Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -0,0 +1,142 @@
|
||||
"""Web contracts for restoring live task state and rejecting resource conflicts."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.runner import RunManager
|
||||
from app.scheduler import BackgroundScheduler
|
||||
from app.web import create_app
|
||||
|
||||
from .conftest import write_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_app(settings, monkeypatch):
|
||||
for group, filename, source in (
|
||||
("example", "long.py", 'RESOURCES = ("shared-data",)\n'),
|
||||
("example", "sibling.py", ""),
|
||||
("related", "change.py", 'RESOURCES = ("shared-data",)\n'),
|
||||
("independent", "free.py", ""),
|
||||
):
|
||||
write_task(settings.task_root, group, filename, source + 'print("done")\n')
|
||||
monkeypatch.setattr(RunManager, "_wake_worker", lambda *_: None)
|
||||
app = create_app(settings)
|
||||
yield app
|
||||
app.extensions["background_scheduler"].stop()
|
||||
|
||||
|
||||
def start_live_run(app, *, status="running", recovery=False):
|
||||
manager = app.extensions["run_manager"]
|
||||
task = app.extensions["catalog"].task("example/long.py")
|
||||
run, _ = manager.run_task(task)
|
||||
if recovery:
|
||||
old = manager.store.finish(run.id, status="interrupted")
|
||||
run, _ = manager.store.enqueue(
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
resources=task.resources, recovery_of_run_id=old.id,
|
||||
)
|
||||
if status == "running":
|
||||
run = manager.store.claim_next(task.group_id)
|
||||
progress = manager.settings.state_root / "runs" / f"{run.id}.progress.json"
|
||||
progress.parent.mkdir(parents=True, exist_ok=True)
|
||||
progress.write_text(json.dumps({"text": "2 of 5\nETA: 3 minutes"}))
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["queued", "running"])
|
||||
def test_task_states_restore_active_conflicts_and_progress(live_app, status):
|
||||
run = start_live_run(live_app, status=status)
|
||||
client = live_app.test_client()
|
||||
response = client.get("/tasks/state")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
states = response.get_json()
|
||||
own = states["tasks"]["example/long.py"]
|
||||
assert own["active_run"]["run_id"] == run.id
|
||||
assert own["active_run"]["display_status"] == status
|
||||
assert own["progress"] == "2 of 5\nETA: 3 minutes"
|
||||
assert own["blocked_by"] is None
|
||||
for name in ("example/sibling.py", "related/change.py"):
|
||||
state = states["tasks"][name]
|
||||
assert state["active_run"] is None
|
||||
assert state["blocked_by"]["detail_url"] == f"/runs/{run.id}"
|
||||
assert states["tasks"]["independent/free.py"] == {
|
||||
"active_run": None, "blocked_by": None, "progress": "",
|
||||
}
|
||||
assert states["groups"]["related"]["blocked_by"]["run_id"] == run.id
|
||||
assert not {"input_json", "resources_json", "log_path"}.intersection(own["active_run"])
|
||||
page = client.get("/")
|
||||
assert b'fieldset class="task-controls" disabled' in page.data
|
||||
assert b"2 of 5\nETA: 3 minutes" in page.data
|
||||
assert f'Run #{run.id}'.encode() in page.data
|
||||
assert client.post("/tasks/state").status_code == 405
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["queued", "running"])
|
||||
def test_recovery_links_and_live_run_progress(live_app, status):
|
||||
run = start_live_run(live_app, status=status, recovery=True)
|
||||
client = live_app.test_client()
|
||||
display_status = "Recovery queued" if status == "queued" else "recovering"
|
||||
state = client.get(f"/runs/{run.id}/state").get_json()
|
||||
assert state["status"] == status
|
||||
assert state["display_status"] == display_status
|
||||
assert state["recovery_of_run_id"] == run.recovery_of_run_id
|
||||
assert state["progress"] == "2 of 5\nETA: 3 minutes"
|
||||
output = client.get(f"/runs/{run.id}/output?offset=0").get_json()
|
||||
assert output["progress"] == state["progress"]
|
||||
assert output["display_status"] == display_status
|
||||
detail = client.get(f"/runs/{run.id}")
|
||||
assert f'href="/runs/{run.recovery_of_run_id}"'.encode() in detail.data
|
||||
assert b'id="run-progress" class="run-progress">2 of 5\nETA: 3 minutes' in detail.data
|
||||
old_detail = client.get(f"/runs/{run.recovery_of_run_id}")
|
||||
assert b"Recovery run" in old_detail.data
|
||||
assert f'href="/runs/{run.id}"'.encode() in old_detail.data
|
||||
old_state = client.get(f"/runs/{run.recovery_of_run_id}/state").get_json()
|
||||
assert old_state["recovery_run"]["run_id"] == run.id
|
||||
assert b"recovery of" in client.get("/history/task/example/long.py").data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
|
||||
@pytest.mark.parametrize("target", ["/tasks/related/change.py/run", "/groups/related/run"])
|
||||
def test_conflicting_submissions_return_409_with_blocking_run(live_app, accept, target):
|
||||
run = start_live_run(live_app)
|
||||
response = live_app.test_client().post(target, headers={"Accept": accept})
|
||||
assert response.status_code == 409
|
||||
if accept == "application/json":
|
||||
payload = response.get_json()
|
||||
assert payload["blocked_by"]["run_id"] == run.id
|
||||
assert payload["blocked_by"]["detail_url"] == f"/runs/{run.id}"
|
||||
assert f"#{run.id}" in payload["error"]
|
||||
else:
|
||||
assert f'href="/runs/{run.id}"'.encode() in response.data
|
||||
assert live_app.extensions["run_store"].last_run("task", "related/change.py") is None
|
||||
unrelated = live_app.test_client().post("/tasks/independent/free.py/run", headers={"Accept": accept})
|
||||
assert unrelated.status_code == (202 if accept == "application/json" else 302)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
|
||||
@pytest.mark.parametrize("target", ["/tasks/example/long.py/run", "/groups/example/run"])
|
||||
def test_shutdown_is_reported_without_enqueuing(live_app, accept, target):
|
||||
live_app.extensions["run_manager"]._closing.set()
|
||||
response = live_app.test_client().post(target, headers={"Accept": accept})
|
||||
assert response.status_code == 503
|
||||
assert b"shutting down" in response.data
|
||||
|
||||
|
||||
def test_startup_precedes_cleanup_and_scheduler(settings, monkeypatch):
|
||||
calls = []
|
||||
monkeypatch.setattr(RunManager, "startup", lambda *_: calls.append("startup"))
|
||||
monkeypatch.setattr(RunManager, "cleanup_transient_files", lambda *_: calls.append("cleanup"))
|
||||
monkeypatch.setattr(BackgroundScheduler, "start", lambda *_: calls.append("scheduler"))
|
||||
create_app(settings)
|
||||
assert calls == ["startup", "cleanup", "scheduler"]
|
||||
|
||||
|
||||
def test_progress_is_plain_text_and_escaped(live_app):
|
||||
run = start_live_run(live_app)
|
||||
path = live_app.config["settings"].state_root / "runs" / f"{run.id}.progress.json"
|
||||
path.write_text(json.dumps({"text": "<script>not HTML</script>\nNext item"}))
|
||||
for url in ("/", f"/runs/{run.id}"):
|
||||
page = live_app.test_client().get(url)
|
||||
assert b"<script>not HTML</script>\nNext item" in page.data
|
||||
Reference in new issue
Block a user