Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -9,6 +9,27 @@ from app.discovery import Group, TaskCatalog
|
||||
from .conftest import write_task
|
||||
|
||||
|
||||
def test_discovers_embedded_recovery_and_literal_resources_without_import(tmp_path):
|
||||
path=write_task(tmp_path,'tasks','change.py', '''
|
||||
"""A recoverable task."""
|
||||
RESOURCES = ('shared-files', 'shared-files')
|
||||
def recover():
|
||||
raise RuntimeError('Must not run during discovery')
|
||||
raise RuntimeError('Must not import during discovery')
|
||||
''')
|
||||
task=TaskCatalog(tmp_path).task('tasks/change.py')
|
||||
assert task.recoverable is True
|
||||
assert task.resources==('group:tasks','shared-files')
|
||||
path.write_text('class Example:\n def recover(self): pass\n')
|
||||
assert TaskCatalog(tmp_path).task('tasks/change.py').recoverable is False
|
||||
|
||||
|
||||
def test_resources_do_not_execute_expressions(tmp_path):
|
||||
write_task(tmp_path,'tasks','change.py',"RESOURCES = tuple(['shared-files'])\n")
|
||||
with pytest.raises(ValueError):
|
||||
TaskCatalog(tmp_path).groups()
|
||||
|
||||
|
||||
def test_discovers_only_direct_non_hidden_python_tasks(tmp_path: Path) -> None:
|
||||
root = tmp_path / "tasks"
|
||||
write_task(root, "media_tasks", "check_one.py", '"""Checks one thing."""\n')
|
||||
|
||||
@@ -21,7 +21,11 @@ def output_run(settings):
|
||||
def test_log_polling_returns_only_appended_bytes_and_status(output_run):
|
||||
client, run_id, path, store = output_run
|
||||
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
|
||||
assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"}
|
||||
assert first == {
|
||||
"output": "first\n", "offset": 6, "reset": False, "has_more": False,
|
||||
"status": "running", "display_status": "running", "progress": "",
|
||||
"recovery_of_run_id": None,
|
||||
}
|
||||
with path.open("ab") as stream:
|
||||
stream.write("next: \u20ac\n".encode())
|
||||
store.finish(run_id, status="succeeded")
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Web contracts for restoring live task state and rejecting resource conflicts."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.runner import RunManager
|
||||
from app.scheduler import BackgroundScheduler
|
||||
from app.web import create_app
|
||||
|
||||
from .conftest import write_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_app(settings, monkeypatch):
|
||||
for group, filename, source in (
|
||||
("example", "long.py", 'RESOURCES = ("shared-data",)\n'),
|
||||
("example", "sibling.py", ""),
|
||||
("related", "change.py", 'RESOURCES = ("shared-data",)\n'),
|
||||
("independent", "free.py", ""),
|
||||
):
|
||||
write_task(settings.task_root, group, filename, source + 'print("done")\n')
|
||||
monkeypatch.setattr(RunManager, "_wake_worker", lambda *_: None)
|
||||
app = create_app(settings)
|
||||
yield app
|
||||
app.extensions["background_scheduler"].stop()
|
||||
|
||||
|
||||
def start_live_run(app, *, status="running", recovery=False):
|
||||
manager = app.extensions["run_manager"]
|
||||
task = app.extensions["catalog"].task("example/long.py")
|
||||
run, _ = manager.run_task(task)
|
||||
if recovery:
|
||||
old = manager.store.finish(run.id, status="interrupted")
|
||||
run, _ = manager.store.enqueue(
|
||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||
resources=task.resources, recovery_of_run_id=old.id,
|
||||
)
|
||||
if status == "running":
|
||||
run = manager.store.claim_next(task.group_id)
|
||||
progress = manager.settings.state_root / "runs" / f"{run.id}.progress.json"
|
||||
progress.parent.mkdir(parents=True, exist_ok=True)
|
||||
progress.write_text(json.dumps({"text": "2 of 5\nETA: 3 minutes"}))
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["queued", "running"])
|
||||
def test_task_states_restore_active_conflicts_and_progress(live_app, status):
|
||||
run = start_live_run(live_app, status=status)
|
||||
client = live_app.test_client()
|
||||
response = client.get("/tasks/state")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
states = response.get_json()
|
||||
own = states["tasks"]["example/long.py"]
|
||||
assert own["active_run"]["run_id"] == run.id
|
||||
assert own["active_run"]["display_status"] == status
|
||||
assert own["progress"] == "2 of 5\nETA: 3 minutes"
|
||||
assert own["blocked_by"] is None
|
||||
for name in ("example/sibling.py", "related/change.py"):
|
||||
state = states["tasks"][name]
|
||||
assert state["active_run"] is None
|
||||
assert state["blocked_by"]["detail_url"] == f"/runs/{run.id}"
|
||||
assert states["tasks"]["independent/free.py"] == {
|
||||
"active_run": None, "blocked_by": None, "progress": "",
|
||||
}
|
||||
assert states["groups"]["related"]["blocked_by"]["run_id"] == run.id
|
||||
assert not {"input_json", "resources_json", "log_path"}.intersection(own["active_run"])
|
||||
page = client.get("/")
|
||||
assert b'fieldset class="task-controls" disabled' in page.data
|
||||
assert b"2 of 5\nETA: 3 minutes" in page.data
|
||||
assert f'Run #{run.id}'.encode() in page.data
|
||||
assert client.post("/tasks/state").status_code == 405
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["queued", "running"])
|
||||
def test_recovery_links_and_live_run_progress(live_app, status):
|
||||
run = start_live_run(live_app, status=status, recovery=True)
|
||||
client = live_app.test_client()
|
||||
display_status = "Recovery queued" if status == "queued" else "recovering"
|
||||
state = client.get(f"/runs/{run.id}/state").get_json()
|
||||
assert state["status"] == status
|
||||
assert state["display_status"] == display_status
|
||||
assert state["recovery_of_run_id"] == run.recovery_of_run_id
|
||||
assert state["progress"] == "2 of 5\nETA: 3 minutes"
|
||||
output = client.get(f"/runs/{run.id}/output?offset=0").get_json()
|
||||
assert output["progress"] == state["progress"]
|
||||
assert output["display_status"] == display_status
|
||||
detail = client.get(f"/runs/{run.id}")
|
||||
assert f'href="/runs/{run.recovery_of_run_id}"'.encode() in detail.data
|
||||
assert b'id="run-progress" class="run-progress">2 of 5\nETA: 3 minutes' in detail.data
|
||||
old_detail = client.get(f"/runs/{run.recovery_of_run_id}")
|
||||
assert b"Recovery run" in old_detail.data
|
||||
assert f'href="/runs/{run.id}"'.encode() in old_detail.data
|
||||
old_state = client.get(f"/runs/{run.recovery_of_run_id}/state").get_json()
|
||||
assert old_state["recovery_run"]["run_id"] == run.id
|
||||
assert b"recovery of" in client.get("/history/task/example/long.py").data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
|
||||
@pytest.mark.parametrize("target", ["/tasks/related/change.py/run", "/groups/related/run"])
|
||||
def test_conflicting_submissions_return_409_with_blocking_run(live_app, accept, target):
|
||||
run = start_live_run(live_app)
|
||||
response = live_app.test_client().post(target, headers={"Accept": accept})
|
||||
assert response.status_code == 409
|
||||
if accept == "application/json":
|
||||
payload = response.get_json()
|
||||
assert payload["blocked_by"]["run_id"] == run.id
|
||||
assert payload["blocked_by"]["detail_url"] == f"/runs/{run.id}"
|
||||
assert f"#{run.id}" in payload["error"]
|
||||
else:
|
||||
assert f'href="/runs/{run.id}"'.encode() in response.data
|
||||
assert live_app.extensions["run_store"].last_run("task", "related/change.py") is None
|
||||
unrelated = live_app.test_client().post("/tasks/independent/free.py/run", headers={"Accept": accept})
|
||||
assert unrelated.status_code == (202 if accept == "application/json" else 302)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
|
||||
@pytest.mark.parametrize("target", ["/tasks/example/long.py/run", "/groups/example/run"])
|
||||
def test_shutdown_is_reported_without_enqueuing(live_app, accept, target):
|
||||
live_app.extensions["run_manager"]._closing.set()
|
||||
response = live_app.test_client().post(target, headers={"Accept": accept})
|
||||
assert response.status_code == 503
|
||||
assert b"shutting down" in response.data
|
||||
|
||||
|
||||
def test_startup_precedes_cleanup_and_scheduler(settings, monkeypatch):
|
||||
calls = []
|
||||
monkeypatch.setattr(RunManager, "startup", lambda *_: calls.append("startup"))
|
||||
monkeypatch.setattr(RunManager, "cleanup_transient_files", lambda *_: calls.append("cleanup"))
|
||||
monkeypatch.setattr(BackgroundScheduler, "start", lambda *_: calls.append("scheduler"))
|
||||
create_app(settings)
|
||||
assert calls == ["startup", "cleanup", "scheduler"]
|
||||
|
||||
|
||||
def test_progress_is_plain_text_and_escaped(live_app):
|
||||
run = start_live_run(live_app)
|
||||
path = live_app.config["settings"].state_root / "runs" / f"{run.id}.progress.json"
|
||||
path.write_text(json.dumps({"text": "<script>not HTML</script>\nNext item"}))
|
||||
for url in ("/", f"/runs/{run.id}"):
|
||||
page = live_app.test_client().get(url)
|
||||
assert b"<script>not HTML</script>\nNext item" in page.data
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Optional Playwright checks against disposable, locally served task fixtures."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from .conftest import write_task
|
||||
from .test_live_status import live_app as live_app, start_live_run
|
||||
|
||||
playwright = pytest.importorskip("playwright.sync_api")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def browser():
|
||||
with playwright.sync_playwright() as driver:
|
||||
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE", driver.chromium.executable_path)
|
||||
if not Path(executable).is_file():
|
||||
pytest.skip("Install Playwright's Chromium or set PLAYWRIGHT_CHROMIUM_EXECUTABLE")
|
||||
browser = driver.chromium.launch(executable_path=executable)
|
||||
yield browser
|
||||
browser.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_server(live_app):
|
||||
server = make_server("127.0.0.1", 0, live_app, threaded=True)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield live_app, f"http://127.0.0.1:{server.server_port}"
|
||||
server.shutdown()
|
||||
thread.join(timeout=5)
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_page(browser, live_server):
|
||||
app, url = live_server
|
||||
context = browser.new_context(accept_downloads=True)
|
||||
context.add_init_script("""(() => {
|
||||
window.leaveListeners = [];
|
||||
const original = window.addEventListener;
|
||||
window.addEventListener = function(type, ...args) {
|
||||
if (type === 'beforeunload') window.leaveListeners.push(type);
|
||||
return original.call(this, type, ...args);
|
||||
};
|
||||
})();""")
|
||||
page = context.new_page()
|
||||
errors = []
|
||||
dialogs = []
|
||||
page.on("pageerror", lambda error: errors.append(str(error)))
|
||||
|
||||
def dismiss(dialog):
|
||||
dialogs.append(dialog.type)
|
||||
dialog.dismiss()
|
||||
|
||||
page.on("dialog", dismiss)
|
||||
yield app, url, page
|
||||
context.close()
|
||||
assert not errors
|
||||
assert not dialogs
|
||||
|
||||
|
||||
def task_form(page, task_id):
|
||||
return page.locator(f'form[data-task-id="{task_id}"]')
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [1280, 390, 320])
|
||||
@pytest.mark.parametrize("status", ["queued", "running", "recovering", "recovery queued"])
|
||||
def test_reload_navigation_and_multiline_progress(live_page, tmp_path, width, status):
|
||||
app, url, page = live_page
|
||||
run = start_live_run(
|
||||
app, status="queued" if status in {"queued", "recovery queued"} else "running",
|
||||
recovery=status in {"recovering", "recovery queued"},
|
||||
)
|
||||
page.set_viewport_size({"width": width, "height": 844})
|
||||
page.goto(url)
|
||||
own = task_form(page, "example/long.py")
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize())
|
||||
playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
assert own.locator('button[type="submit"]').is_disabled()
|
||||
for name in ("example/sibling.py", "related/change.py"):
|
||||
form = task_form(page, name)
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Blocked by")
|
||||
assert form.locator('button[type="submit"]').is_disabled()
|
||||
assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled()
|
||||
page.reload()
|
||||
playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
own.locator(f'.task-result a[href="/runs/{run.id}"]').click()
|
||||
playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
page.reload()
|
||||
playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes")
|
||||
playwright.expect(page.locator("#run-status")).to_have_text("Recovery queued" if status == "recovery queued" else status)
|
||||
assert page.locator("#run-progress").evaluate("el => getComputedStyle(el).whiteSpace") == "pre-wrap"
|
||||
page.screenshot(path=str(tmp_path / f"run-{status}-{width}.png"), full_page=True)
|
||||
assert page.evaluate("window.leaveListeners") == []
|
||||
page.locator('main a[href="/"]').first.click()
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize())
|
||||
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
|
||||
page.screenshot(path=str(tmp_path / f"live-{status}-{width}.png"), full_page=True)
|
||||
|
||||
|
||||
def test_first_render_blocks_conflicts_without_javascript(browser, live_server):
|
||||
app, url = live_server
|
||||
start_live_run(app)
|
||||
context = browser.new_context(java_script_enabled=False)
|
||||
try:
|
||||
page = context.new_page()
|
||||
page.goto(url)
|
||||
assert task_form(page, "example/long.py").locator('button[type="submit"]').is_disabled()
|
||||
assert task_form(page, "related/change.py").locator('button[type="submit"]').is_disabled()
|
||||
assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled()
|
||||
assert task_form(page, "example/long.py").locator(".task-progress").inner_text() == "2 of 5\nETA: 3 minutes"
|
||||
finally:
|
||||
context.close()
|
||||
|
||||
|
||||
def test_live_changes_and_racing_conflict_show_blocking_run(live_page):
|
||||
app, url, page = live_page
|
||||
page.goto(url)
|
||||
snapshot = {"value": app.test_client().get("/tasks/state").get_json()}
|
||||
page.route("**/tasks/state", lambda route: route.fulfill(json=snapshot["value"]))
|
||||
run = start_live_run(app, status="queued")
|
||||
related = task_form(page, "related/change.py")
|
||||
with page.expect_response(lambda response: response.url.endswith("/tasks/related/change.py/run") and response.status == 409):
|
||||
related.locator('button[type="submit"]').click()
|
||||
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
|
||||
playwright.expect(related.locator(".task-result")).to_contain_text(f"Blocked by run #{run.id}")
|
||||
assert related.locator('button[type="submit"]').is_disabled()
|
||||
own = task_form(page, "example/long.py")
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text("Queued")
|
||||
app.extensions["run_store"].claim_next("example")
|
||||
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text("Running")
|
||||
app.extensions["run_store"].finish(run.id, status="succeeded")
|
||||
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
|
||||
playwright.expect(own.locator(".task-result")).to_contain_text("Completed successfully.")
|
||||
playwright.expect(related.locator('button[type="submit"]')).to_be_enabled()
|
||||
|
||||
|
||||
def test_long_task_submission_redirects_and_restores_on_return(live_page):
|
||||
app, url, page = live_page
|
||||
page.goto(url)
|
||||
task_form(page, "example/long.py").locator('button[type="submit"]').click()
|
||||
page.wait_for_url("**/runs/*")
|
||||
playwright.expect(page.locator("#run-status")).to_have_text("queued")
|
||||
page.locator('main a[href="/"]').first.click()
|
||||
playwright.expect(task_form(page, "example/long.py").locator(".task-result")).to_contain_text("Queued")
|
||||
assert app.extensions["run_store"].last_run("task", "example/long.py").status == "queued"
|
||||
|
||||
|
||||
def test_csv_download_after_reload_does_not_repeat(live_page, settings):
|
||||
app, url, page = live_page
|
||||
task = write_task(settings.task_root, "exports", "review.py", 'print("done")\n')
|
||||
(task.parent / "task-inputs.json").write_text(json.dumps({
|
||||
"version": 1, "tasks": {task.name: {"wait_for_result": True, "download_artifacts": True}},
|
||||
}))
|
||||
page.goto(url)
|
||||
form = task_form(page, "exports/review.py")
|
||||
form.locator('button[type="submit"]').click()
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Queued")
|
||||
page.reload()
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Queued")
|
||||
assert page.evaluate("window.leaveListeners") == []
|
||||
run = app.extensions["run_store"].last_run("task", "exports/review.py")
|
||||
artifacts = settings.state_root / "runs" / f"{run.id}.artifacts"
|
||||
artifacts.mkdir(parents=True, exist_ok=True)
|
||||
(artifacts / "report.json").write_text('{"ok":true}')
|
||||
(artifacts / "review.csv").write_text("item,decision\n1,keep\n")
|
||||
downloads = []
|
||||
page.on("download", lambda download: downloads.append(download.suggested_filename))
|
||||
with page.expect_download() as downloaded:
|
||||
app.extensions["run_store"].finish(run.id, status="succeeded")
|
||||
download = downloaded.value
|
||||
assert download.suggested_filename == "review.csv"
|
||||
assert Path(download.path()).read_text() == "item,decision\n1,keep\n"
|
||||
playwright.expect(form.locator(".task-result")).to_contain_text("Completed successfully.")
|
||||
assert form.locator('button[type="submit"]').is_enabled()
|
||||
page.wait_for_timeout(2200)
|
||||
assert downloads == ["review.csv"]
|
||||
assert (artifacts / "report.json").is_file()
|
||||
assert not (artifacts / "review.csv").exists()
|
||||
|
||||
|
||||
def test_incremental_output_preserves_selection_and_stops_polling(live_page, settings):
|
||||
app, url, page = live_page
|
||||
run = start_live_run(app)
|
||||
log = settings.state_root / "runs" / f"{run.id}.log"
|
||||
log.write_text("Select this text\n")
|
||||
store = app.extensions["run_store"]
|
||||
store.set_log_path(run.id, str(log))
|
||||
polls = []
|
||||
artifacts = []
|
||||
|
||||
def record(request):
|
||||
if "/output?" in request.url:
|
||||
polls.append(request.url)
|
||||
if request.url.endswith("/artifacts"):
|
||||
artifacts.append(request.url)
|
||||
|
||||
page.on("request", record)
|
||||
page.goto(f"{url}/runs/{run.id}")
|
||||
page.evaluate("""() => {
|
||||
const node = document.querySelector('#run-output').firstChild;
|
||||
const range = document.createRange();
|
||||
range.setStart(node, 0); range.setEnd(node, 16);
|
||||
getSelection().removeAllRanges(); getSelection().addRange(range);
|
||||
}""")
|
||||
with log.open("a") as stream:
|
||||
stream.write("More output\n")
|
||||
progress = settings.state_root / "runs" / f"{run.id}.progress.json"
|
||||
temporary = progress.with_suffix(".tmp")
|
||||
temporary.write_text(json.dumps({"text": "3 of 5\nETA: 2 minutes"}))
|
||||
temporary.replace(progress)
|
||||
playwright.expect(page.locator("#run-output")).to_contain_text("More output")
|
||||
playwright.expect(page.locator("#run-progress")).to_have_text("3 of 5\nETA: 2 minutes")
|
||||
assert page.evaluate("getSelection().toString()") == "Select this text"
|
||||
assert not artifacts
|
||||
with log.open("a") as stream:
|
||||
stream.write("Finished\n")
|
||||
store.finish(run.id, status="succeeded")
|
||||
playwright.expect(page.locator("#run-output")).to_contain_text("Finished")
|
||||
playwright.expect(page.locator('form[action$="/cancel"]')).to_have_count(0)
|
||||
assert page.evaluate("getSelection().toString()") == "Select this text"
|
||||
finished_polls = len(polls)
|
||||
page.wait_for_timeout(2300)
|
||||
assert len(polls) == finished_polls
|
||||
assert len(artifacts) == 1
|
||||
assert all("offset=" in request for request in polls)
|
||||
|
||||
|
||||
def test_execute_control_appears_and_tracks_conflicts_after_completion(live_page, settings):
|
||||
app, url, page = live_page
|
||||
manifest = settings.task_root / "example" / "task-inputs.json"
|
||||
manifest.write_text(json.dumps({
|
||||
"version": 1, "tasks": {"long.py": {"execution": {
|
||||
"field": "mode", "dry_run_value": "review", "dry_run_label": "Review",
|
||||
"execute_value": "apply", "execute_label": "Execute",
|
||||
}}},
|
||||
}))
|
||||
manager = app.extensions["run_manager"]
|
||||
catalog = app.extensions["catalog"]
|
||||
run, _ = manager.run_task(catalog.task("example/long.py"), {"mode": "review"})
|
||||
manager.store.claim_next("example")
|
||||
page.goto(f"{url}/runs/{run.id}")
|
||||
manager.store.finish(run.id, status="succeeded")
|
||||
execute = task_form(page, "example/long.py")
|
||||
playwright.expect(execute.locator('button[type="submit"]')).to_be_visible()
|
||||
blocker, _ = manager.run_task(catalog.task("example/sibling.py"))
|
||||
playwright.expect(execute.locator(".task-result")).to_contain_text(f"Blocked by run #{blocker.id}")
|
||||
assert execute.locator('button[type="submit"]').is_disabled()
|
||||
manager.store.finish(blocker.id, status="succeeded")
|
||||
playwright.expect(execute.locator('button[type="submit"]')).to_be_enabled()
|
||||
execute.locator('button[type="submit"]').click()
|
||||
page.wait_for_url(lambda current: current.startswith(f"{url}/runs/") and current != f"{url}/runs/{run.id}")
|
||||
accepted = manager.store.last_run("task", "example/long.py")
|
||||
assert accepted.id != run.id
|
||||
assert json.loads(accepted.input_json) == {"mode": "apply"}
|
||||
@@ -0,0 +1,195 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.discovery import TaskCatalog
|
||||
from app.runner import RunManager, TaskConflictError
|
||||
from app.store import RunStore
|
||||
|
||||
from .conftest import write_task
|
||||
from .test_runner import wait_for, wait_for_output
|
||||
|
||||
|
||||
COUNTER = '''"""Reconcile a small external ledger before continuing."""
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
def main():
|
||||
data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text())
|
||||
ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt'
|
||||
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
||||
if not runtime.checkpoint_data():
|
||||
runtime.checkpoint({'next': 0}, force=True)
|
||||
for number in range(len(existing), data.get('count', 50)):
|
||||
runtime.check_stop()
|
||||
with ledger.open('a') as stream:
|
||||
stream.write(str(number) + '\\n')
|
||||
runtime.checkpoint({'next': number + 1})
|
||||
runtime.progress(f'Items: {number + 1}', force=True)
|
||||
runtime.sleep(.03)
|
||||
|
||||
def recover():
|
||||
print('Checking external ledger', flush=True)
|
||||
main()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager):
|
||||
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
|
||||
task = manager.catalog.task('count/task.py')
|
||||
run, _ = manager.run_task(task, {'count': 50})
|
||||
wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3')
|
||||
manager.shutdown(timeout=1)
|
||||
assert manager.store.get(run.id).status == 'interrupted'
|
||||
work = manager._work_root(run)
|
||||
saved = json.loads((work / 'checkpoint.json').read_text())
|
||||
assert saved['clean_shutdown'] is True
|
||||
assert (work / 'input.json').exists()
|
||||
manager.cleanup_transient_files(artifact_max_age_seconds=0)
|
||||
assert (work / 'input.json').exists()
|
||||
resumed = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
resumed.startup()
|
||||
recovery = manager.store.recovery_for(run.id)
|
||||
assert recovery is not None
|
||||
assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded'
|
||||
assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
|
||||
assert manager.store.get(run.id).status == 'interrupted'
|
||||
assert not (work / 'input.json').exists()
|
||||
assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text()
|
||||
resumed.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_forced_container_style_exit_recovers_from_older_checkpoint(settings):
|
||||
settings.task_root.mkdir(parents=True)
|
||||
write_task(settings.task_root, 'count', 'task.py', COUNTER)
|
||||
control = settings.state_root / 'control.json'
|
||||
program = '''
|
||||
import json,time
|
||||
from pathlib import Path
|
||||
from app.config import Settings
|
||||
from app.discovery import TaskCatalog
|
||||
from app.runner import RunManager
|
||||
from app.store import RunStore
|
||||
s=Settings.from_environment()
|
||||
c=TaskCatalog(s.task_root)
|
||||
db=RunStore(s.database_path)
|
||||
db.initialize()
|
||||
m=RunManager(s,c,db)
|
||||
r,_=m.run_task(c.task('count/task.py'),{'count':50})
|
||||
while r.id not in m._processes:
|
||||
time.sleep(.01)
|
||||
Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid}))
|
||||
while True:
|
||||
time.sleep(1)
|
||||
'''
|
||||
environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]),
|
||||
'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root),
|
||||
'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root),
|
||||
'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)}
|
||||
process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True)
|
||||
child_pid = None
|
||||
resumed = None
|
||||
try:
|
||||
wait_for_output(control, 'pid')
|
||||
data = json.loads(control.read_text())
|
||||
child_pid = data['pid']
|
||||
wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
||||
process.kill()
|
||||
process.wait(timeout=3)
|
||||
os.killpg(child_pid, signal.SIGKILL)
|
||||
child_pid = None
|
||||
saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text())
|
||||
assert saved['clean_shutdown'] is False
|
||||
assert saved['data']['next'] == 0
|
||||
store = RunStore(settings.database_path)
|
||||
resumed = RunManager(settings, TaskCatalog(settings.task_root), store)
|
||||
resumed.startup()
|
||||
recovery = store.recovery_for(data['run'])
|
||||
assert recovery is not None
|
||||
assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded'
|
||||
assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=3)
|
||||
if child_pid is not None:
|
||||
try:
|
||||
os.killpg(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
if resumed:
|
||||
resumed.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_cancelled_tasks_never_auto_recover(manager):
|
||||
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
|
||||
task = manager.catalog.task('count/task.py')
|
||||
run, _ = manager.run_task(task, {'count': 100})
|
||||
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
||||
manager.request_cancel(run.id)
|
||||
wait_for(manager, run.id, {'cancelled'})
|
||||
manager.shutdown(timeout=1)
|
||||
new = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
new.startup()
|
||||
assert manager.store.recovery_for(run.id) is None
|
||||
assert not (manager._work_root(run) / 'input.json').exists()
|
||||
new.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager):
|
||||
path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():'))
|
||||
run, _ = manager.run_task(manager.catalog.task('count/task.py'))
|
||||
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
||||
manager.shutdown(timeout=1)
|
||||
assert manager.store.get(run.id).status == 'interrupted'
|
||||
path.write_text(COUNTER)
|
||||
legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy')
|
||||
write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER)
|
||||
# The current task has opted out again; neither it nor a pre-runtime run may resume.
|
||||
path.write_text(COUNTER.replace('def recover():', 'def unused():'))
|
||||
new = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
new.startup()
|
||||
assert manager.store.recovery_for(run.id) is None
|
||||
assert manager.store.recovery_for(legacy.id) is None
|
||||
assert manager.store.active_runs() == []
|
||||
new.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_shared_resources_block_cross_group_submissions_and_recoveries(manager):
|
||||
for group in ('first','second'):
|
||||
write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER)
|
||||
first = manager.catalog.task('first/task.py')
|
||||
second = manager.catalog.task('second/task.py')
|
||||
run, _ = manager.run_task(first, {'count':50})
|
||||
with pytest.raises(TaskConflictError) as error:
|
||||
manager.run_task(second)
|
||||
assert error.value.run.id == run.id
|
||||
manager.shutdown(timeout=1)
|
||||
new = RunManager(manager.settings, manager.catalog, manager.store)
|
||||
new.startup()
|
||||
recovered = manager.store.recovery_for(run.id)
|
||||
assert recovered is not None
|
||||
with pytest.raises(TaskConflictError) as error:
|
||||
new.run_task(second)
|
||||
assert error.value.run.id == recovered.id
|
||||
new.shutdown(timeout=1)
|
||||
|
||||
|
||||
def test_shutdown_rejects_new_work(manager):
|
||||
write_task(manager.settings.task_root,'tasks','task.py','print("hello")')
|
||||
manager.shutdown(timeout=.1)
|
||||
with pytest.raises(RuntimeError,match='shutting down'):
|
||||
manager.run_task(manager.catalog.task('tasks/task.py'))
|
||||
@@ -58,7 +58,11 @@ def test_tasks_in_one_group_are_serialized_and_groups_overlap(manager: RunManage
|
||||
other_task = catalog.task("second_group/other.py")
|
||||
assert first_task is not None and second_task is not None and other_task is not None
|
||||
first, _ = manager.run_task(first_task)
|
||||
second, _ = manager.run_task(second_task)
|
||||
second, _ = manager.store.enqueue(
|
||||
target_kind="task", target_path=second_task.id, queue_group=second_task.group_id,
|
||||
resources=second_task.resources,
|
||||
)
|
||||
manager._wake_worker(second_task.group_id)
|
||||
other, _ = manager.run_task(other_task)
|
||||
|
||||
wait_for(manager, first.id, {"running"})
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
"""Recovery through the real Waitress entry point, without manager test doubles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import closing
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.test import EnvironBuilder
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration")
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n"
|
||||
SECRET = "disposable-process-test-secret"
|
||||
|
||||
|
||||
LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
RESOURCES = ("shared-ledger",)
|
||||
state = Path(os.environ["SERVER_MAINTENANCE_STATE"])
|
||||
|
||||
def values():
|
||||
data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||
assert data["secret"] == "disposable-process-test-secret"
|
||||
assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n"
|
||||
return data
|
||||
|
||||
def main():
|
||||
data = values()
|
||||
ledger = state / "ledger.txt"
|
||||
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
||||
if not runtime.checkpoint_data():
|
||||
runtime.checkpoint({"next": 0}, force=True)
|
||||
items = Path(data["items"]).read_text().splitlines()
|
||||
for number in range(len(existing), len(items)):
|
||||
runtime.check_stop()
|
||||
with ledger.open("a") as stream:
|
||||
stream.write(items[number] + "\\n")
|
||||
runtime.checkpoint({"next": number + 1})
|
||||
if number == 2:
|
||||
runtime.progress("Original waiting", force=True)
|
||||
while not (state / "finish").exists():
|
||||
runtime.sleep(.02)
|
||||
|
||||
def recover():
|
||||
values()
|
||||
(state / "observation.json").write_text(json.dumps({
|
||||
"checkpoint": runtime.checkpoint_data(),
|
||||
"clean_shutdown": runtime.was_clean_shutdown(),
|
||||
"ledger": (state / "ledger.txt").read_text().splitlines(),
|
||||
}))
|
||||
runtime.progress("Recovery waiting", force=True)
|
||||
while not (state / "finish").exists():
|
||||
runtime.sleep(.02)
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
runtime.checkpoint({"next": 0}, force=True)
|
||||
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
|
||||
runtime.checkpoint({"next": 1})
|
||||
Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text(
|
||||
json.dumps([os.getpid(), child.pid]))
|
||||
runtime.progress("Blocking", force=True)
|
||||
time.sleep(60)
|
||||
'''
|
||||
|
||||
|
||||
def wait_until(condition, description, timeout=8):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = condition()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(.02)
|
||||
pytest.fail(f"Timed out waiting for {description}")
|
||||
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def process_running(pid):
|
||||
try:
|
||||
status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1]
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return status[0] not in {"Z", "X"}
|
||||
|
||||
|
||||
class ProductionApp:
|
||||
def __init__(self, root):
|
||||
self.task_root = root / "tasks"
|
||||
self.state_root = root / "state"
|
||||
self.credentials_root = root / "credentials"
|
||||
for path in (self.task_root, self.state_root, self.credentials_root):
|
||||
path.mkdir()
|
||||
self.database = self.state_root / "server-maintenance.sqlite3"
|
||||
self.log_path = root / "server.log"
|
||||
self.process = None
|
||||
self.port = None
|
||||
self.grace = 1
|
||||
|
||||
def write_task(self, group, filename, source):
|
||||
path = self.task_root / group / filename
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(source, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def start(self):
|
||||
with socket.socket() as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
port = listener.getsockname()[1]
|
||||
self.launch(port)
|
||||
|
||||
def ready():
|
||||
assert self.process.poll() is None, self.log_path.read_text()
|
||||
try:
|
||||
status, _ = self.request("GET", "/tasks/state")
|
||||
except (ConnectionError, TimeoutError):
|
||||
return False
|
||||
return status == 200
|
||||
|
||||
wait_until(ready, f"Waitress startup; log: {self.log_path}")
|
||||
|
||||
def launch(self, port):
|
||||
assert self.process is None or self.process.poll() is not None
|
||||
self.port = port
|
||||
environment = {key: value for key, value in os.environ.items()
|
||||
if not key.startswith("SERVER_MAINTENANCE_")}
|
||||
environment.update({
|
||||
"PYTHONPATH": str(PROJECT_ROOT),
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"SERVER_MAINTENANCE_HOST": "127.0.0.1",
|
||||
"SERVER_MAINTENANCE_PORT": str(self.port),
|
||||
"SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root),
|
||||
"SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root),
|
||||
"SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root),
|
||||
"SERVER_MAINTENANCE_PYTHON": sys.executable,
|
||||
"SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace),
|
||||
})
|
||||
with self.log_path.open("ab") as log:
|
||||
self.process = subprocess.Popen(
|
||||
[sys.executable, "-m", "app"], cwd=PROJECT_ROOT,
|
||||
env=environment, stdin=subprocess.DEVNULL, stdout=log,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
)
|
||||
|
||||
def request(self, method, path, data=None):
|
||||
headers = {"Accept": "application/json"}
|
||||
body = None
|
||||
if data is not None:
|
||||
with closing(EnvironBuilder(method=method, data=data)) as builder:
|
||||
environ = builder.get_environ()
|
||||
body = environ["wsgi.input"].read()
|
||||
headers["Content-Type"] = environ["CONTENT_TYPE"]
|
||||
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2)
|
||||
try:
|
||||
connection.request(method, path, body=body, headers=headers)
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
assert response.getheader("Content-Type", "").startswith("application/json"), payload
|
||||
return response.status, json.loads(payload)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def rows(self):
|
||||
with closing(sqlite3.connect(self.database)) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")]
|
||||
|
||||
def run(self, task, data=None):
|
||||
status, response = self.request("POST", f"/tasks/{task}/run", data)
|
||||
assert status == 202, response
|
||||
return response["run_id"]
|
||||
|
||||
def wait_status(self, run_id, status):
|
||||
def matching():
|
||||
run = next(row for row in self.rows() if row["id"] == run_id)
|
||||
if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}:
|
||||
assert run["status"] == status, (run, self.log_path.read_text())
|
||||
return run if run["status"] == status else None
|
||||
return wait_until(matching, f"run {run_id} to become {status}")
|
||||
|
||||
def wait_progress(self, run_id, text):
|
||||
path = self.state_root / "runs" / f"{run_id}.progress.json"
|
||||
wait_until(lambda: (read_json(path) or {}).get("text") == text, text)
|
||||
|
||||
def pids(self):
|
||||
# Match only this disposable instance, including separately-sessioned workers.
|
||||
marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode()
|
||||
found = []
|
||||
for path in Path("/proc").iterdir():
|
||||
if not path.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
environment = (path / "environ").read_bytes().split(b"\0")
|
||||
except (FileNotFoundError, ProcessLookupError, PermissionError):
|
||||
continue
|
||||
if marker in environment:
|
||||
found.append(int(path.name))
|
||||
return found
|
||||
|
||||
def kill_remaining(self):
|
||||
def kill_snapshot():
|
||||
pids = self.pids()
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return not pids
|
||||
|
||||
kill_snapshot()
|
||||
if self.process is not None:
|
||||
self.process.wait(timeout=3)
|
||||
wait_until(kill_snapshot, "disposable instance teardown", timeout=3)
|
||||
|
||||
def stop(self):
|
||||
started = time.monotonic()
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
try:
|
||||
code = self.process.wait(timeout=self.grace + 3)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}")
|
||||
elapsed = time.monotonic() - started
|
||||
assert code == 0, self.log_path.read_text()
|
||||
assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text())
|
||||
return elapsed
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def production_app(tmp_path):
|
||||
app = ProductionApp(tmp_path)
|
||||
try:
|
||||
yield app
|
||||
finally:
|
||||
app.kill_remaining()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"])
|
||||
def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal):
|
||||
app = production_app
|
||||
ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK)
|
||||
(ledger_task.parent / "task-inputs.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"tasks": {"task.py": {"inputs": [
|
||||
{"name": "items", "label": "Items", "type": "file", "required": True,
|
||||
"accept": [".txt"], "maximum_bytes": 1024},
|
||||
{"name": "secret", "label": "Secret", "type": "text", "required": True,
|
||||
"sensitive": True},
|
||||
]}},
|
||||
}))
|
||||
app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n')
|
||||
app.write_task("maintenance", "task.py", 'print("done")\n')
|
||||
app.start()
|
||||
original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")})
|
||||
app.wait_progress(original_id, "Original waiting")
|
||||
runs = app.state_root / "runs"
|
||||
work = runs / f"{original_id}.work"
|
||||
saved_input = json.loads((work / "input.json").read_text())
|
||||
upload = Path(saved_input["items"])
|
||||
assert saved_input["secret"] == SECRET
|
||||
assert SECRET not in app.rows()[0]["input_json"]
|
||||
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
||||
old_pids = app.pids()
|
||||
if shutdown_signal == signal.SIGTERM:
|
||||
app.stop()
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
assert app.rows()[0]["exit_code"] == 75
|
||||
else:
|
||||
# SIGKILL the entire disposable instance, as container teardown does.
|
||||
app.kill_remaining()
|
||||
assert app.process.returncode == -signal.SIGKILL
|
||||
assert app.rows()[0]["status"] == "running"
|
||||
wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit")
|
||||
saved = read_json(work / "checkpoint.json")
|
||||
clean = shutdown_signal == signal.SIGTERM
|
||||
assert saved["clean_shutdown"] is clean
|
||||
assert saved["data"] == {"next": 3 if clean else 0}
|
||||
assert upload.read_bytes() == UPLOAD
|
||||
assert json.loads((work / "input.json").read_text()) == saved_input
|
||||
|
||||
stale = time.time() - 45 * 86400
|
||||
for path in (work, work / "input.json", upload.parent, upload):
|
||||
os.utime(path, (stale, stale))
|
||||
app.start()
|
||||
recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id)
|
||||
app.wait_progress(recovery["id"], "Recovery waiting")
|
||||
observation = read_json(app.state_root / "tasks/ledger/observation.json")
|
||||
assert observation == {
|
||||
"checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean,
|
||||
"ledger": ["0", "1", "2"],
|
||||
}
|
||||
status, states = app.request("GET", "/tasks/state")
|
||||
assert status == 200
|
||||
assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering"
|
||||
assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"]
|
||||
status, conflict = app.request("POST", "/tasks/contender/task.py/run")
|
||||
assert status == 409, conflict
|
||||
assert conflict["blocked_by"]["run_id"] == recovery["id"]
|
||||
assert not any(row["target_path"] == "contender/task.py" for row in app.rows())
|
||||
|
||||
# A real completion invokes cleanup while recovery still needs its ancestor's files.
|
||||
cleanup_probe = runs / "999999.input.json"
|
||||
cleanup_probe.write_text("{}\n")
|
||||
maintenance_id = app.run("maintenance/task.py")
|
||||
app.wait_status(maintenance_id, "succeeded")
|
||||
wait_until(lambda: not cleanup_probe.exists(), "completion cleanup")
|
||||
assert upload.read_bytes() == UPLOAD
|
||||
assert json.loads((work / "input.json").read_text()) == saved_input
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
|
||||
(app.state_root / "tasks/ledger/finish").touch()
|
||||
app.wait_status(recovery["id"], "succeeded")
|
||||
assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)]
|
||||
wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup")
|
||||
app.stop()
|
||||
assert not app.pids(), "A task escaped registration/shutdown"
|
||||
|
||||
|
||||
def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app):
|
||||
app = production_app
|
||||
app.write_task("blocking", "task.py", BLOCKING_TASK)
|
||||
app.write_task("watcher", "internal/task.py", BLOCKING_TASK)
|
||||
(app.task_root / "watcher/background-tasks.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}],
|
||||
}))
|
||||
app.start()
|
||||
run_id = app.run("blocking/task.py")
|
||||
app.wait_progress(run_id, "Blocking")
|
||||
background_work = app.state_root / "background/watcher/watcher.work"
|
||||
wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking",
|
||||
"background worker to start")
|
||||
work = app.state_root / "runs" / f"{run_id}.work"
|
||||
task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json")
|
||||
assert all(process_running(pid) for pid in task_pids)
|
||||
elapsed = app.stop()
|
||||
assert elapsed >= app.grace * .8, "The hard deadline was not exercised"
|
||||
wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit")
|
||||
assert not app.pids(), "A foreground/background task escaped shutdown registration"
|
||||
assert read_json(work / "checkpoint.json")["clean_shutdown"] is False
|
||||
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
||||
assert (work / "input.json").exists()
|
||||
|
||||
|
||||
def test_production_graceful_pause_does_not_leave_live_descendants(production_app):
|
||||
app = production_app
|
||||
source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n")
|
||||
app.write_task("cooperative", "task.py", source)
|
||||
app.start()
|
||||
run_id = app.run("cooperative/task.py")
|
||||
app.wait_progress(run_id, "Blocking")
|
||||
work = app.state_root / "runs" / f"{run_id}.work"
|
||||
worker_pid, child_pid = read_json(work / "pids.json")
|
||||
app.stop()
|
||||
assert read_json(work / "checkpoint.json")["clean_shutdown"] is True
|
||||
assert not process_running(worker_pid)
|
||||
wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5)
|
||||
|
||||
|
||||
def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app):
|
||||
app = production_app
|
||||
source = '''
|
||||
from app import task_runtime as runtime
|
||||
runtime.progress("Started", force=True)
|
||||
while True:
|
||||
runtime.sleep(.02)
|
||||
'''
|
||||
groups = [f"queue_{index}" for index in range(12)]
|
||||
for group in groups:
|
||||
app.write_task(group, "task.py", source)
|
||||
app.start()
|
||||
first_id = app.run(f"{groups[0]}/task.py")
|
||||
app.wait_progress(first_id, "Started")
|
||||
|
||||
def submit(group):
|
||||
try:
|
||||
return app.request("POST", f"/tasks/{group}/task.py/run")[0]
|
||||
except (ConnectionError, TimeoutError, http.client.RemoteDisconnected):
|
||||
# The listener may close before this in-flight request is accepted.
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as requests:
|
||||
pending = [requests.submit(submit, group) for group in groups[1:]]
|
||||
app.stop()
|
||||
statuses = [request.result(timeout=3) for request in pending]
|
||||
assert all(status in {202, 503, None} for status in statuses), statuses
|
||||
assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration"
|
||||
assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text()
|
||||
|
||||
|
||||
def test_production_bind_failure_stops_workers_started_by_recovery(production_app):
|
||||
app = production_app
|
||||
source = '''
|
||||
from app import task_runtime as runtime
|
||||
|
||||
def main():
|
||||
runtime.progress("Started", force=True)
|
||||
while True:
|
||||
runtime.sleep(.02)
|
||||
|
||||
def recover():
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
app.write_task("recoverable", "task.py", source)
|
||||
app.start()
|
||||
run_id = app.run("recoverable/task.py")
|
||||
app.wait_progress(run_id, "Started")
|
||||
app.stop()
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
|
||||
# Delay startup cleanup without patching app code, until recovery is running.
|
||||
cleanup_gate = app.state_root / "tasks/.orphaned-groups.json"
|
||||
os.mkfifo(cleanup_gate)
|
||||
with socket.socket() as occupied:
|
||||
occupied.bind(("127.0.0.1", 0))
|
||||
occupied.listen()
|
||||
app.launch(occupied.getsockname()[1])
|
||||
recovery = wait_until(
|
||||
lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None),
|
||||
"startup recovery registration",
|
||||
)
|
||||
app.wait_progress(recovery["id"], "Started")
|
||||
|
||||
def release_cleanup():
|
||||
try:
|
||||
descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK)
|
||||
except OSError as error:
|
||||
if error.errno == errno.ENXIO:
|
||||
return False
|
||||
raise
|
||||
with os.fdopen(descriptor, "w") as gate:
|
||||
json.dump({}, gate)
|
||||
return True
|
||||
|
||||
wait_until(release_cleanup, "startup cleanup gate", timeout=2)
|
||||
assert app.process.wait(timeout=4) != 0
|
||||
assert "Address already in use" in app.log_path.read_text()
|
||||
assert any(row["recovery_of_run_id"] == run_id for row in app.rows())
|
||||
wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)
|
||||
@@ -0,0 +1,81 @@
|
||||
import json
|
||||
import signal
|
||||
|
||||
import pytest
|
||||
|
||||
from app import task_runtime as runtime
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def work(tmp_path,monkeypatch):
|
||||
monkeypatch.setenv('SERVER_MAINTENANCE_WORK',str(tmp_path))
|
||||
old=signal.getsignal(signal.SIGUSR1)
|
||||
runtime.initialize()
|
||||
yield tmp_path
|
||||
signal.signal(signal.SIGUSR1,old)
|
||||
runtime._pause_requested=False
|
||||
runtime._defer=0
|
||||
|
||||
|
||||
def test_safe_point_saves_latest_memory_state_and_marks_clean(work):
|
||||
runtime.checkpoint({'next':1},force=True)
|
||||
runtime.checkpoint({'next':2})
|
||||
assert json.loads((work/'checkpoint.json').read_text())['data']=={'next':1}
|
||||
runtime.request_pause()
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
runtime.check_stop()
|
||||
saved=json.loads((work/'checkpoint.json').read_text())
|
||||
assert saved['data']=={'next':2}
|
||||
assert saved['clean_shutdown'] is True
|
||||
runtime.initialize()
|
||||
assert runtime.was_clean_shutdown() is True
|
||||
assert json.loads((work/'checkpoint.json').read_text())['clean_shutdown'] is False
|
||||
|
||||
|
||||
def test_small_critical_operation_finishes_before_pause(work):
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
with runtime.critical_section():
|
||||
runtime.request_pause()
|
||||
runtime.checkpoint({'committed':True},force=True)
|
||||
assert runtime.checkpoint_data()=={'committed':True}
|
||||
saved=json.loads((work/'checkpoint.json').read_text())
|
||||
assert saved['data']=={'committed':True}
|
||||
assert saved['clean_shutdown'] is True
|
||||
|
||||
|
||||
def test_sleep_is_interruptible(work):
|
||||
runtime.request_pause()
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
runtime.sleep(30)
|
||||
|
||||
|
||||
def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
|
||||
clock=[100.]
|
||||
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
|
||||
progress=runtime.Progress({'Removing files':10,'Linking files':20})
|
||||
progress.update('Removing files',0,force=True)
|
||||
clock[0]+=30
|
||||
progress.update('Removing files',5,force=True)
|
||||
text=json.loads((work/'progress.json').read_text())['text']
|
||||
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
|
||||
assert (work/'progress.json').stat().st_mode&0o777==0o600
|
||||
|
||||
|
||||
def test_runtime_does_not_write_or_pause_direct_scripts_without_work(monkeypatch):
|
||||
monkeypatch.delenv('SERVER_MAINTENANCE_WORK',raising=False)
|
||||
runtime.request_pause()
|
||||
runtime.checkpoint({'ignored':True})
|
||||
runtime.progress('hello',force=True)
|
||||
assert runtime.checkpoint_data()=={}
|
||||
runtime._pause_requested=False
|
||||
|
||||
|
||||
def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
|
||||
clock=[100.]
|
||||
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
|
||||
progress=runtime.Progress({'Items':50})
|
||||
progress.update('Items',39,force=True)
|
||||
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
|
||||
clock[0]+=60
|
||||
progress.update('Items',40,force=True)
|
||||
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
|
||||
@@ -141,3 +141,35 @@ def test_removing_file_input_from_task_cleans_up_unused_review_upload(settings):
|
||||
manifest.write_text(json.dumps(definition))
|
||||
app.extensions["run_manager"].cleanup_transient_files()
|
||||
assert not original.parent.exists()
|
||||
|
||||
|
||||
def test_conflicting_execute_is_linked_and_keeps_review_upload(settings, monkeypatch):
|
||||
app, client, review_id = start_review(settings)
|
||||
manager = app.extensions["run_manager"]
|
||||
original = Path(json.loads(manager.store.get(review_id).input_json)["csv"])
|
||||
write_task(settings.task_root, "imports", "other.py", 'print("done")\n')
|
||||
monkeypatch.setattr(manager, "_wake_worker", lambda *_: None)
|
||||
blocker, _ = manager.run_task(app.extensions["catalog"].task("imports/other.py"))
|
||||
page = client.get(f"/runs/{review_id}")
|
||||
assert b'fieldset class="task-controls" disabled' in page.data
|
||||
assert f'href="/runs/{blocker.id}"'.encode() in page.data
|
||||
response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
|
||||
assert response.status_code == 409
|
||||
assert response.get_json()["blocked_by"]["run_id"] == blocker.id
|
||||
assert original.is_file()
|
||||
manager.store.finish(blocker.id, status="cancelled")
|
||||
accepted = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
|
||||
assert accepted.status_code == 202
|
||||
assert accepted.get_json()["detail_url"].startswith("/runs/")
|
||||
assert accepted.get_json()["run_id"] != review_id
|
||||
|
||||
|
||||
def test_shutdown_during_execute_keeps_review_upload(settings):
|
||||
app, client, review_id = start_review(settings)
|
||||
manager = app.extensions["run_manager"]
|
||||
original = Path(json.loads(manager.store.get(review_id).input_json)["csv"])
|
||||
manager._closing.set()
|
||||
response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
|
||||
assert response.status_code == 503
|
||||
assert "shutting down" in response.get_json()["error"]
|
||||
assert original.is_file()
|
||||
@@ -0,0 +1,56 @@
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
import pytest
|
||||
|
||||
from app.worker import service_unavailable
|
||||
from app import worker
|
||||
|
||||
|
||||
@pytest.mark.parametrize('error', [
|
||||
ConnectionRefusedError(), TimeoutError(), URLError(OSError('connection refused')),
|
||||
HTTPError('http://example.invalid',503,'unavailable',{},None),
|
||||
])
|
||||
def test_service_failures_can_retry_recovery(error):
|
||||
wrapped=RuntimeError('Service request failed')
|
||||
wrapped.__cause__=error
|
||||
assert service_unavailable(wrapped)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('error', [
|
||||
RuntimeError('Unrecognized file'), FileNotFoundError(), PermissionError(),
|
||||
HTTPError('http://example.invalid',401,'unauthorized',{},None),
|
||||
HTTPError('http://example.invalid',400,'bad input',{},None),
|
||||
])
|
||||
def test_task_and_authentication_errors_are_not_automatically_repeated(error):
|
||||
assert not service_unavailable(error)
|
||||
|
||||
|
||||
def test_database_starting_up_is_transient_but_authentication_is_not():
|
||||
error=RuntimeError('Database unavailable')
|
||||
error.sqlstate='57P03'
|
||||
assert service_unavailable(error)
|
||||
error.sqlstate='28P01'
|
||||
assert not service_unavailable(error)
|
||||
|
||||
|
||||
def test_recovery_rechecks_live_state_after_lost_service_response(tmp_path, monkeypatch):
|
||||
ledger = []
|
||||
calls = []
|
||||
waits = []
|
||||
|
||||
def recover():
|
||||
calls.append(list(ledger))
|
||||
if not ledger:
|
||||
ledger.append('applied')
|
||||
raise URLError(ConnectionResetError('response lost'))
|
||||
|
||||
monkeypatch.setattr(worker.sys, 'argv', ['worker', str(tmp_path/'task.py')])
|
||||
monkeypatch.setenv('SERVER_MAINTENANCE_RECOVER','1')
|
||||
monkeypatch.setattr(worker.runtime,'initialize',lambda:None)
|
||||
monkeypatch.setattr(worker.runtime,'progress',lambda *a,**kw:None)
|
||||
monkeypatch.setattr(worker.runtime,'sleep',waits.append)
|
||||
monkeypatch.setattr(worker.runpy,'run_path',lambda *a,**kw:{'recover':recover})
|
||||
worker.main()
|
||||
assert ledger==['applied']
|
||||
assert calls==[[],['applied']]
|
||||
assert waits==[2]
|
||||
Reference in new issue
Block a user