Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-10-29 22:12:04 +00:00
1 parent 034fecbd51
commit 3bfd199b9c
28 files changed
+2356 -172

No files matched your search

+21
View File
@@ -9,6 +9,27 @@ from app.discovery import Group, TaskCatalog
from .conftest import write_task
def test_discovers_embedded_recovery_and_literal_resources_without_import(tmp_path):
path=write_task(tmp_path,'tasks','change.py', '''
"""A recoverable task."""
RESOURCES = ('shared-files', 'shared-files')
def recover():
raise RuntimeError('Must not run during discovery')
raise RuntimeError('Must not import during discovery')
''')
task=TaskCatalog(tmp_path).task('tasks/change.py')
assert task.recoverable is True
assert task.resources==('group:tasks','shared-files')
path.write_text('class Example:\n def recover(self): pass\n')
assert TaskCatalog(tmp_path).task('tasks/change.py').recoverable is False
def test_resources_do_not_execute_expressions(tmp_path):
write_task(tmp_path,'tasks','change.py',"RESOURCES = tuple(['shared-files'])\n")
with pytest.raises(ValueError):
TaskCatalog(tmp_path).groups()
def test_discovers_only_direct_non_hidden_python_tasks(tmp_path: Path) -> None:
root = tmp_path / "tasks"
write_task(root, "media_tasks", "check_one.py", '"""Checks one thing."""\n')
@@ -21,7 +21,11 @@ def output_run(settings):
def test_log_polling_returns_only_appended_bytes_and_status(output_run):
client, run_id, path, store = output_run
first = client.get(f"/runs/{run_id}/output?offset=0").get_json()
assert first == {"output": "first\n", "offset": 6, "reset": False, "has_more": False, "status": "running"}
assert first == {
"output": "first\n", "offset": 6, "reset": False, "has_more": False,
"status": "running", "display_status": "running", "progress": "",
"recovery_of_run_id": None,
}
with path.open("ab") as stream:
stream.write("next: \u20ac\n".encode())
store.finish(run_id, status="succeeded")
+142
View File
@@ -0,0 +1,142 @@
"""Web contracts for restoring live task state and rejecting resource conflicts."""
import json
import pytest
from app.runner import RunManager
from app.scheduler import BackgroundScheduler
from app.web import create_app
from .conftest import write_task
@pytest.fixture
def live_app(settings, monkeypatch):
for group, filename, source in (
("example", "long.py", 'RESOURCES = ("shared-data",)\n'),
("example", "sibling.py", ""),
("related", "change.py", 'RESOURCES = ("shared-data",)\n'),
("independent", "free.py", ""),
):
write_task(settings.task_root, group, filename, source + 'print("done")\n')
monkeypatch.setattr(RunManager, "_wake_worker", lambda *_: None)
app = create_app(settings)
yield app
app.extensions["background_scheduler"].stop()
def start_live_run(app, *, status="running", recovery=False):
manager = app.extensions["run_manager"]
task = app.extensions["catalog"].task("example/long.py")
run, _ = manager.run_task(task)
if recovery:
old = manager.store.finish(run.id, status="interrupted")
run, _ = manager.store.enqueue(
target_kind="task", target_path=task.id, queue_group=task.group_id,
resources=task.resources, recovery_of_run_id=old.id,
)
if status == "running":
run = manager.store.claim_next(task.group_id)
progress = manager.settings.state_root / "runs" / f"{run.id}.progress.json"
progress.parent.mkdir(parents=True, exist_ok=True)
progress.write_text(json.dumps({"text": "2 of 5\nETA: 3 minutes"}))
return run
@pytest.mark.parametrize("status", ["queued", "running"])
def test_task_states_restore_active_conflicts_and_progress(live_app, status):
run = start_live_run(live_app, status=status)
client = live_app.test_client()
response = client.get("/tasks/state")
assert response.status_code == 200
assert response.headers["Cache-Control"] == "no-store"
states = response.get_json()
own = states["tasks"]["example/long.py"]
assert own["active_run"]["run_id"] == run.id
assert own["active_run"]["display_status"] == status
assert own["progress"] == "2 of 5\nETA: 3 minutes"
assert own["blocked_by"] is None
for name in ("example/sibling.py", "related/change.py"):
state = states["tasks"][name]
assert state["active_run"] is None
assert state["blocked_by"]["detail_url"] == f"/runs/{run.id}"
assert states["tasks"]["independent/free.py"] == {
"active_run": None, "blocked_by": None, "progress": "",
}
assert states["groups"]["related"]["blocked_by"]["run_id"] == run.id
assert not {"input_json", "resources_json", "log_path"}.intersection(own["active_run"])
page = client.get("/")
assert b'fieldset class="task-controls" disabled' in page.data
assert b"2 of 5\nETA: 3 minutes" in page.data
assert f'Run #{run.id}'.encode() in page.data
assert client.post("/tasks/state").status_code == 405
@pytest.mark.parametrize("status", ["queued", "running"])
def test_recovery_links_and_live_run_progress(live_app, status):
run = start_live_run(live_app, status=status, recovery=True)
client = live_app.test_client()
display_status = "Recovery queued" if status == "queued" else "recovering"
state = client.get(f"/runs/{run.id}/state").get_json()
assert state["status"] == status
assert state["display_status"] == display_status
assert state["recovery_of_run_id"] == run.recovery_of_run_id
assert state["progress"] == "2 of 5\nETA: 3 minutes"
output = client.get(f"/runs/{run.id}/output?offset=0").get_json()
assert output["progress"] == state["progress"]
assert output["display_status"] == display_status
detail = client.get(f"/runs/{run.id}")
assert f'href="/runs/{run.recovery_of_run_id}"'.encode() in detail.data
assert b'id="run-progress" class="run-progress">2 of 5\nETA: 3 minutes' in detail.data
old_detail = client.get(f"/runs/{run.recovery_of_run_id}")
assert b"Recovery run" in old_detail.data
assert f'href="/runs/{run.id}"'.encode() in old_detail.data
old_state = client.get(f"/runs/{run.recovery_of_run_id}/state").get_json()
assert old_state["recovery_run"]["run_id"] == run.id
assert b"recovery of" in client.get("/history/task/example/long.py").data
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
@pytest.mark.parametrize("target", ["/tasks/related/change.py/run", "/groups/related/run"])
def test_conflicting_submissions_return_409_with_blocking_run(live_app, accept, target):
run = start_live_run(live_app)
response = live_app.test_client().post(target, headers={"Accept": accept})
assert response.status_code == 409
if accept == "application/json":
payload = response.get_json()
assert payload["blocked_by"]["run_id"] == run.id
assert payload["blocked_by"]["detail_url"] == f"/runs/{run.id}"
assert f"#{run.id}" in payload["error"]
else:
assert f'href="/runs/{run.id}"'.encode() in response.data
assert live_app.extensions["run_store"].last_run("task", "related/change.py") is None
unrelated = live_app.test_client().post("/tasks/independent/free.py/run", headers={"Accept": accept})
assert unrelated.status_code == (202 if accept == "application/json" else 302)
@pytest.mark.parametrize("accept", ["text/html", "application/json"])
@pytest.mark.parametrize("target", ["/tasks/example/long.py/run", "/groups/example/run"])
def test_shutdown_is_reported_without_enqueuing(live_app, accept, target):
live_app.extensions["run_manager"]._closing.set()
response = live_app.test_client().post(target, headers={"Accept": accept})
assert response.status_code == 503
assert b"shutting down" in response.data
def test_startup_precedes_cleanup_and_scheduler(settings, monkeypatch):
calls = []
monkeypatch.setattr(RunManager, "startup", lambda *_: calls.append("startup"))
monkeypatch.setattr(RunManager, "cleanup_transient_files", lambda *_: calls.append("cleanup"))
monkeypatch.setattr(BackgroundScheduler, "start", lambda *_: calls.append("scheduler"))
create_app(settings)
assert calls == ["startup", "cleanup", "scheduler"]
def test_progress_is_plain_text_and_escaped(live_app):
run = start_live_run(live_app)
path = live_app.config["settings"].state_root / "runs" / f"{run.id}.progress.json"
path.write_text(json.dumps({"text": "<script>not HTML</script>\nNext item"}))
for url in ("/", f"/runs/{run.id}"):
page = live_app.test_client().get(url)
assert b"&lt;script&gt;not HTML&lt;/script&gt;\nNext item" in page.data
@@ -0,0 +1,261 @@
"""Optional Playwright checks against disposable, locally served task fixtures."""
import json
import os
import threading
from pathlib import Path
import pytest
from werkzeug.serving import make_server
from .conftest import write_task
from .test_live_status import live_app as live_app, start_live_run
playwright = pytest.importorskip("playwright.sync_api")
@pytest.fixture(scope="module")
def browser():
with playwright.sync_playwright() as driver:
executable = os.environ.get("PLAYWRIGHT_CHROMIUM_EXECUTABLE", driver.chromium.executable_path)
if not Path(executable).is_file():
pytest.skip("Install Playwright's Chromium or set PLAYWRIGHT_CHROMIUM_EXECUTABLE")
browser = driver.chromium.launch(executable_path=executable)
yield browser
browser.close()
@pytest.fixture
def live_server(live_app):
server = make_server("127.0.0.1", 0, live_app, threaded=True)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
yield live_app, f"http://127.0.0.1:{server.server_port}"
server.shutdown()
thread.join(timeout=5)
server.server_close()
@pytest.fixture
def live_page(browser, live_server):
app, url = live_server
context = browser.new_context(accept_downloads=True)
context.add_init_script("""(() => {
window.leaveListeners = [];
const original = window.addEventListener;
window.addEventListener = function(type, ...args) {
if (type === 'beforeunload') window.leaveListeners.push(type);
return original.call(this, type, ...args);
};
})();""")
page = context.new_page()
errors = []
dialogs = []
page.on("pageerror", lambda error: errors.append(str(error)))
def dismiss(dialog):
dialogs.append(dialog.type)
dialog.dismiss()
page.on("dialog", dismiss)
yield app, url, page
context.close()
assert not errors
assert not dialogs
def task_form(page, task_id):
return page.locator(f'form[data-task-id="{task_id}"]')
@pytest.mark.parametrize("width", [1280, 390, 320])
@pytest.mark.parametrize("status", ["queued", "running", "recovering", "recovery queued"])
def test_reload_navigation_and_multiline_progress(live_page, tmp_path, width, status):
app, url, page = live_page
run = start_live_run(
app, status="queued" if status in {"queued", "recovery queued"} else "running",
recovery=status in {"recovering", "recovery queued"},
)
page.set_viewport_size({"width": width, "height": 844})
page.goto(url)
own = task_form(page, "example/long.py")
playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize())
playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes")
assert own.locator('button[type="submit"]').is_disabled()
for name in ("example/sibling.py", "related/change.py"):
form = task_form(page, name)
playwright.expect(form.locator(".task-result")).to_contain_text("Blocked by")
assert form.locator('button[type="submit"]').is_disabled()
assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled()
page.reload()
playwright.expect(own.locator(".task-progress")).to_have_text("2 of 5\nETA: 3 minutes")
own.locator(f'.task-result a[href="/runs/{run.id}"]').click()
playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes")
page.reload()
playwright.expect(page.locator("#run-progress")).to_have_text("2 of 5\nETA: 3 minutes")
playwright.expect(page.locator("#run-status")).to_have_text("Recovery queued" if status == "recovery queued" else status)
assert page.locator("#run-progress").evaluate("el => getComputedStyle(el).whiteSpace") == "pre-wrap"
page.screenshot(path=str(tmp_path / f"run-{status}-{width}.png"), full_page=True)
assert page.evaluate("window.leaveListeners") == []
page.locator('main a[href="/"]').first.click()
playwright.expect(own.locator(".task-result")).to_contain_text(status.capitalize())
assert page.evaluate("document.documentElement.scrollWidth <= innerWidth")
page.screenshot(path=str(tmp_path / f"live-{status}-{width}.png"), full_page=True)
def test_first_render_blocks_conflicts_without_javascript(browser, live_server):
app, url = live_server
start_live_run(app)
context = browser.new_context(java_script_enabled=False)
try:
page = context.new_page()
page.goto(url)
assert task_form(page, "example/long.py").locator('button[type="submit"]').is_disabled()
assert task_form(page, "related/change.py").locator('button[type="submit"]').is_disabled()
assert task_form(page, "independent/free.py").locator('button[type="submit"]').is_enabled()
assert task_form(page, "example/long.py").locator(".task-progress").inner_text() == "2 of 5\nETA: 3 minutes"
finally:
context.close()
def test_live_changes_and_racing_conflict_show_blocking_run(live_page):
app, url, page = live_page
page.goto(url)
snapshot = {"value": app.test_client().get("/tasks/state").get_json()}
page.route("**/tasks/state", lambda route: route.fulfill(json=snapshot["value"]))
run = start_live_run(app, status="queued")
related = task_form(page, "related/change.py")
with page.expect_response(lambda response: response.url.endswith("/tasks/related/change.py/run") and response.status == 409):
related.locator('button[type="submit"]').click()
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
playwright.expect(related.locator(".task-result")).to_contain_text(f"Blocked by run #{run.id}")
assert related.locator('button[type="submit"]').is_disabled()
own = task_form(page, "example/long.py")
playwright.expect(own.locator(".task-result")).to_contain_text("Queued")
app.extensions["run_store"].claim_next("example")
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
playwright.expect(own.locator(".task-result")).to_contain_text("Running")
app.extensions["run_store"].finish(run.id, status="succeeded")
snapshot["value"] = app.test_client().get("/tasks/state").get_json()
playwright.expect(own.locator(".task-result")).to_contain_text("Completed successfully.")
playwright.expect(related.locator('button[type="submit"]')).to_be_enabled()
def test_long_task_submission_redirects_and_restores_on_return(live_page):
app, url, page = live_page
page.goto(url)
task_form(page, "example/long.py").locator('button[type="submit"]').click()
page.wait_for_url("**/runs/*")
playwright.expect(page.locator("#run-status")).to_have_text("queued")
page.locator('main a[href="/"]').first.click()
playwright.expect(task_form(page, "example/long.py").locator(".task-result")).to_contain_text("Queued")
assert app.extensions["run_store"].last_run("task", "example/long.py").status == "queued"
def test_csv_download_after_reload_does_not_repeat(live_page, settings):
app, url, page = live_page
task = write_task(settings.task_root, "exports", "review.py", 'print("done")\n')
(task.parent / "task-inputs.json").write_text(json.dumps({
"version": 1, "tasks": {task.name: {"wait_for_result": True, "download_artifacts": True}},
}))
page.goto(url)
form = task_form(page, "exports/review.py")
form.locator('button[type="submit"]').click()
playwright.expect(form.locator(".task-result")).to_contain_text("Queued")
page.reload()
playwright.expect(form.locator(".task-result")).to_contain_text("Queued")
assert page.evaluate("window.leaveListeners") == []
run = app.extensions["run_store"].last_run("task", "exports/review.py")
artifacts = settings.state_root / "runs" / f"{run.id}.artifacts"
artifacts.mkdir(parents=True, exist_ok=True)
(artifacts / "report.json").write_text('{"ok":true}')
(artifacts / "review.csv").write_text("item,decision\n1,keep\n")
downloads = []
page.on("download", lambda download: downloads.append(download.suggested_filename))
with page.expect_download() as downloaded:
app.extensions["run_store"].finish(run.id, status="succeeded")
download = downloaded.value
assert download.suggested_filename == "review.csv"
assert Path(download.path()).read_text() == "item,decision\n1,keep\n"
playwright.expect(form.locator(".task-result")).to_contain_text("Completed successfully.")
assert form.locator('button[type="submit"]').is_enabled()
page.wait_for_timeout(2200)
assert downloads == ["review.csv"]
assert (artifacts / "report.json").is_file()
assert not (artifacts / "review.csv").exists()
def test_incremental_output_preserves_selection_and_stops_polling(live_page, settings):
app, url, page = live_page
run = start_live_run(app)
log = settings.state_root / "runs" / f"{run.id}.log"
log.write_text("Select this text\n")
store = app.extensions["run_store"]
store.set_log_path(run.id, str(log))
polls = []
artifacts = []
def record(request):
if "/output?" in request.url:
polls.append(request.url)
if request.url.endswith("/artifacts"):
artifacts.append(request.url)
page.on("request", record)
page.goto(f"{url}/runs/{run.id}")
page.evaluate("""() => {
const node = document.querySelector('#run-output').firstChild;
const range = document.createRange();
range.setStart(node, 0); range.setEnd(node, 16);
getSelection().removeAllRanges(); getSelection().addRange(range);
}""")
with log.open("a") as stream:
stream.write("More output\n")
progress = settings.state_root / "runs" / f"{run.id}.progress.json"
temporary = progress.with_suffix(".tmp")
temporary.write_text(json.dumps({"text": "3 of 5\nETA: 2 minutes"}))
temporary.replace(progress)
playwright.expect(page.locator("#run-output")).to_contain_text("More output")
playwright.expect(page.locator("#run-progress")).to_have_text("3 of 5\nETA: 2 minutes")
assert page.evaluate("getSelection().toString()") == "Select this text"
assert not artifacts
with log.open("a") as stream:
stream.write("Finished\n")
store.finish(run.id, status="succeeded")
playwright.expect(page.locator("#run-output")).to_contain_text("Finished")
playwright.expect(page.locator('form[action$="/cancel"]')).to_have_count(0)
assert page.evaluate("getSelection().toString()") == "Select this text"
finished_polls = len(polls)
page.wait_for_timeout(2300)
assert len(polls) == finished_polls
assert len(artifacts) == 1
assert all("offset=" in request for request in polls)
def test_execute_control_appears_and_tracks_conflicts_after_completion(live_page, settings):
app, url, page = live_page
manifest = settings.task_root / "example" / "task-inputs.json"
manifest.write_text(json.dumps({
"version": 1, "tasks": {"long.py": {"execution": {
"field": "mode", "dry_run_value": "review", "dry_run_label": "Review",
"execute_value": "apply", "execute_label": "Execute",
}}},
}))
manager = app.extensions["run_manager"]
catalog = app.extensions["catalog"]
run, _ = manager.run_task(catalog.task("example/long.py"), {"mode": "review"})
manager.store.claim_next("example")
page.goto(f"{url}/runs/{run.id}")
manager.store.finish(run.id, status="succeeded")
execute = task_form(page, "example/long.py")
playwright.expect(execute.locator('button[type="submit"]')).to_be_visible()
blocker, _ = manager.run_task(catalog.task("example/sibling.py"))
playwright.expect(execute.locator(".task-result")).to_contain_text(f"Blocked by run #{blocker.id}")
assert execute.locator('button[type="submit"]').is_disabled()
manager.store.finish(blocker.id, status="succeeded")
playwright.expect(execute.locator('button[type="submit"]')).to_be_enabled()
execute.locator('button[type="submit"]').click()
page.wait_for_url(lambda current: current.startswith(f"{url}/runs/") and current != f"{url}/runs/{run.id}")
accepted = manager.store.last_run("task", "example/long.py")
assert accepted.id != run.id
assert json.loads(accepted.input_json) == {"mode": "apply"}
+195
View File
@@ -0,0 +1,195 @@
from __future__ import annotations
import json
import os
import signal
import subprocess
import sys
import time
from dataclasses import replace
from pathlib import Path
import pytest
from app.discovery import TaskCatalog
from app.runner import RunManager, TaskConflictError
from app.store import RunStore
from .conftest import write_task
from .test_runner import wait_for, wait_for_output
COUNTER = '''"""Reconcile a small external ledger before continuing."""
import json, os
from pathlib import Path
from app import task_runtime as runtime
def main():
data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text())
ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt'
existing = ledger.read_text().splitlines() if ledger.exists() else []
if not runtime.checkpoint_data():
runtime.checkpoint({'next': 0}, force=True)
for number in range(len(existing), data.get('count', 50)):
runtime.check_stop()
with ledger.open('a') as stream:
stream.write(str(number) + '\\n')
runtime.checkpoint({'next': number + 1})
runtime.progress(f'Items: {number + 1}', force=True)
runtime.sleep(.03)
def recover():
print('Checking external ledger', flush=True)
main()
if __name__ == '__main__':
main()
'''
def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager):
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
task = manager.catalog.task('count/task.py')
run, _ = manager.run_task(task, {'count': 50})
wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3')
manager.shutdown(timeout=1)
assert manager.store.get(run.id).status == 'interrupted'
work = manager._work_root(run)
saved = json.loads((work / 'checkpoint.json').read_text())
assert saved['clean_shutdown'] is True
assert (work / 'input.json').exists()
manager.cleanup_transient_files(artifact_max_age_seconds=0)
assert (work / 'input.json').exists()
resumed = RunManager(manager.settings, manager.catalog, manager.store)
resumed.startup()
recovery = manager.store.recovery_for(run.id)
assert recovery is not None
assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded'
assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
assert manager.store.get(run.id).status == 'interrupted'
assert not (work / 'input.json').exists()
assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text()
resumed.shutdown(timeout=1)
def test_forced_container_style_exit_recovers_from_older_checkpoint(settings):
settings.task_root.mkdir(parents=True)
write_task(settings.task_root, 'count', 'task.py', COUNTER)
control = settings.state_root / 'control.json'
program = '''
import json,time
from pathlib import Path
from app.config import Settings
from app.discovery import TaskCatalog
from app.runner import RunManager
from app.store import RunStore
s=Settings.from_environment()
c=TaskCatalog(s.task_root)
db=RunStore(s.database_path)
db.initialize()
m=RunManager(s,c,db)
r,_=m.run_task(c.task('count/task.py'),{'count':50})
while r.id not in m._processes:
time.sleep(.01)
Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid}))
while True:
time.sleep(1)
'''
environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]),
'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root),
'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root),
'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)}
process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True)
child_pid = None
resumed = None
try:
wait_for_output(control, 'pid')
data = json.loads(control.read_text())
child_pid = data['pid']
wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n')
process.kill()
process.wait(timeout=3)
os.killpg(child_pid, signal.SIGKILL)
child_pid = None
saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text())
assert saved['clean_shutdown'] is False
assert saved['data']['next'] == 0
store = RunStore(settings.database_path)
resumed = RunManager(settings, TaskCatalog(settings.task_root), store)
resumed.startup()
recovery = store.recovery_for(data['run'])
assert recovery is not None
assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded'
assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=3)
if child_pid is not None:
try:
os.killpg(child_pid, signal.SIGKILL)
except ProcessLookupError:
pass
if resumed:
resumed.shutdown(timeout=1)
def test_cancelled_tasks_never_auto_recover(manager):
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
task = manager.catalog.task('count/task.py')
run, _ = manager.run_task(task, {'count': 100})
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
manager.request_cancel(run.id)
wait_for(manager, run.id, {'cancelled'})
manager.shutdown(timeout=1)
new = RunManager(manager.settings, manager.catalog, manager.store)
new.startup()
assert manager.store.recovery_for(run.id) is None
assert not (manager._work_root(run) / 'input.json').exists()
new.shutdown(timeout=1)
def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager):
path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():'))
run, _ = manager.run_task(manager.catalog.task('count/task.py'))
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
manager.shutdown(timeout=1)
assert manager.store.get(run.id).status == 'interrupted'
path.write_text(COUNTER)
legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy')
write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER)
# The current task has opted out again; neither it nor a pre-runtime run may resume.
path.write_text(COUNTER.replace('def recover():', 'def unused():'))
new = RunManager(manager.settings, manager.catalog, manager.store)
new.startup()
assert manager.store.recovery_for(run.id) is None
assert manager.store.recovery_for(legacy.id) is None
assert manager.store.active_runs() == []
new.shutdown(timeout=1)
def test_shared_resources_block_cross_group_submissions_and_recoveries(manager):
for group in ('first','second'):
write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER)
first = manager.catalog.task('first/task.py')
second = manager.catalog.task('second/task.py')
run, _ = manager.run_task(first, {'count':50})
with pytest.raises(TaskConflictError) as error:
manager.run_task(second)
assert error.value.run.id == run.id
manager.shutdown(timeout=1)
new = RunManager(manager.settings, manager.catalog, manager.store)
new.startup()
recovered = manager.store.recovery_for(run.id)
assert recovered is not None
with pytest.raises(TaskConflictError) as error:
new.run_task(second)
assert error.value.run.id == recovered.id
new.shutdown(timeout=1)
def test_shutdown_rejects_new_work(manager):
write_task(manager.settings.task_root,'tasks','task.py','print("hello")')
manager.shutdown(timeout=.1)
with pytest.raises(RuntimeError,match='shutting down'):
manager.run_task(manager.catalog.task('tasks/task.py'))
+5 -1
View File
@@ -58,7 +58,11 @@ def test_tasks_in_one_group_are_serialized_and_groups_overlap(manager: RunManage
other_task = catalog.task("second_group/other.py")
assert first_task is not None and second_task is not None and other_task is not None
first, _ = manager.run_task(first_task)
second, _ = manager.run_task(second_task)
second, _ = manager.store.enqueue(
target_kind="task", target_path=second_task.id, queue_group=second_task.group_id,
resources=second_task.resources,
)
manager._wake_worker(second_task.group_id)
other, _ = manager.run_task(other_task)
wait_for(manager, first.id, {"running"})
+478
View File
@@ -0,0 +1,478 @@
"""Recovery through the real Waitress entry point, without manager test doubles."""
from __future__ import annotations
import errno
import http.client
import json
import os
import signal
import socket
import sqlite3
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from contextlib import closing
from io import BytesIO
from pathlib import Path
import pytest
from werkzeug.test import EnvironBuilder
pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration")
PROJECT_ROOT = Path(__file__).resolve().parents[1]
UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n"
SECRET = "disposable-process-test-secret"
LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger."""
import json
import os
from pathlib import Path
from app import task_runtime as runtime
RESOURCES = ("shared-ledger",)
state = Path(os.environ["SERVER_MAINTENANCE_STATE"])
def values():
data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
assert data["secret"] == "disposable-process-test-secret"
assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n"
return data
def main():
data = values()
ledger = state / "ledger.txt"
existing = ledger.read_text().splitlines() if ledger.exists() else []
if not runtime.checkpoint_data():
runtime.checkpoint({"next": 0}, force=True)
items = Path(data["items"]).read_text().splitlines()
for number in range(len(existing), len(items)):
runtime.check_stop()
with ledger.open("a") as stream:
stream.write(items[number] + "\\n")
runtime.checkpoint({"next": number + 1})
if number == 2:
runtime.progress("Original waiting", force=True)
while not (state / "finish").exists():
runtime.sleep(.02)
def recover():
values()
(state / "observation.json").write_text(json.dumps({
"checkpoint": runtime.checkpoint_data(),
"clean_shutdown": runtime.was_clean_shutdown(),
"ledger": (state / "ledger.txt").read_text().splitlines(),
}))
runtime.progress("Recovery waiting", force=True)
while not (state / "finish").exists():
runtime.sleep(.02)
main()
if __name__ == "__main__":
main()
'''
BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from app import task_runtime as runtime
runtime.checkpoint({"next": 0}, force=True)
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
runtime.checkpoint({"next": 1})
Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text(
json.dumps([os.getpid(), child.pid]))
runtime.progress("Blocking", force=True)
time.sleep(60)
'''
def wait_until(condition, description, timeout=8):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = condition()
if result:
return result
time.sleep(.02)
pytest.fail(f"Timed out waiting for {description}")
def read_json(path):
try:
return json.loads(path.read_text())
except FileNotFoundError:
return None
def process_running(pid):
try:
status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1]
except FileNotFoundError:
return False
return status[0] not in {"Z", "X"}
class ProductionApp:
def __init__(self, root):
self.task_root = root / "tasks"
self.state_root = root / "state"
self.credentials_root = root / "credentials"
for path in (self.task_root, self.state_root, self.credentials_root):
path.mkdir()
self.database = self.state_root / "server-maintenance.sqlite3"
self.log_path = root / "server.log"
self.process = None
self.port = None
self.grace = 1
def write_task(self, group, filename, source):
path = self.task_root / group / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source, encoding="utf-8")
return path
def start(self):
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
port = listener.getsockname()[1]
self.launch(port)
def ready():
assert self.process.poll() is None, self.log_path.read_text()
try:
status, _ = self.request("GET", "/tasks/state")
except (ConnectionError, TimeoutError):
return False
return status == 200
wait_until(ready, f"Waitress startup; log: {self.log_path}")
def launch(self, port):
assert self.process is None or self.process.poll() is not None
self.port = port
environment = {key: value for key, value in os.environ.items()
if not key.startswith("SERVER_MAINTENANCE_")}
environment.update({
"PYTHONPATH": str(PROJECT_ROOT),
"PYTHONUNBUFFERED": "1",
"PYTHONDONTWRITEBYTECODE": "1",
"SERVER_MAINTENANCE_HOST": "127.0.0.1",
"SERVER_MAINTENANCE_PORT": str(self.port),
"SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root),
"SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root),
"SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root),
"SERVER_MAINTENANCE_PYTHON": sys.executable,
"SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace),
})
with self.log_path.open("ab") as log:
self.process = subprocess.Popen(
[sys.executable, "-m", "app"], cwd=PROJECT_ROOT,
env=environment, stdin=subprocess.DEVNULL, stdout=log,
stderr=subprocess.STDOUT, start_new_session=True,
)
def request(self, method, path, data=None):
headers = {"Accept": "application/json"}
body = None
if data is not None:
with closing(EnvironBuilder(method=method, data=data)) as builder:
environ = builder.get_environ()
body = environ["wsgi.input"].read()
headers["Content-Type"] = environ["CONTENT_TYPE"]
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2)
try:
connection.request(method, path, body=body, headers=headers)
response = connection.getresponse()
payload = response.read()
assert response.getheader("Content-Type", "").startswith("application/json"), payload
return response.status, json.loads(payload)
finally:
connection.close()
def rows(self):
with closing(sqlite3.connect(self.database)) as connection:
connection.row_factory = sqlite3.Row
return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")]
def run(self, task, data=None):
status, response = self.request("POST", f"/tasks/{task}/run", data)
assert status == 202, response
return response["run_id"]
def wait_status(self, run_id, status):
def matching():
run = next(row for row in self.rows() if row["id"] == run_id)
if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}:
assert run["status"] == status, (run, self.log_path.read_text())
return run if run["status"] == status else None
return wait_until(matching, f"run {run_id} to become {status}")
def wait_progress(self, run_id, text):
path = self.state_root / "runs" / f"{run_id}.progress.json"
wait_until(lambda: (read_json(path) or {}).get("text") == text, text)
def pids(self):
# Match only this disposable instance, including separately-sessioned workers.
marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode()
found = []
for path in Path("/proc").iterdir():
if not path.name.isdigit():
continue
try:
environment = (path / "environ").read_bytes().split(b"\0")
except (FileNotFoundError, ProcessLookupError, PermissionError):
continue
if marker in environment:
found.append(int(path.name))
return found
def kill_remaining(self):
def kill_snapshot():
pids = self.pids()
for pid in pids:
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
return not pids
kill_snapshot()
if self.process is not None:
self.process.wait(timeout=3)
wait_until(kill_snapshot, "disposable instance teardown", timeout=3)
def stop(self):
started = time.monotonic()
self.process.send_signal(signal.SIGTERM)
try:
code = self.process.wait(timeout=self.grace + 3)
except subprocess.TimeoutExpired:
pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}")
elapsed = time.monotonic() - started
assert code == 0, self.log_path.read_text()
assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text())
return elapsed
@pytest.fixture
def production_app(tmp_path):
app = ProductionApp(tmp_path)
try:
yield app
finally:
app.kill_remaining()
@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"])
def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal):
app = production_app
ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK)
(ledger_task.parent / "task-inputs.json").write_text(json.dumps({
"version": 1,
"tasks": {"task.py": {"inputs": [
{"name": "items", "label": "Items", "type": "file", "required": True,
"accept": [".txt"], "maximum_bytes": 1024},
{"name": "secret", "label": "Secret", "type": "text", "required": True,
"sensitive": True},
]}},
}))
app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n')
app.write_task("maintenance", "task.py", 'print("done")\n')
app.start()
original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")})
app.wait_progress(original_id, "Original waiting")
runs = app.state_root / "runs"
work = runs / f"{original_id}.work"
saved_input = json.loads((work / "input.json").read_text())
upload = Path(saved_input["items"])
assert saved_input["secret"] == SECRET
assert SECRET not in app.rows()[0]["input_json"]
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
old_pids = app.pids()
if shutdown_signal == signal.SIGTERM:
app.stop()
assert app.rows()[0]["status"] == "interrupted"
assert app.rows()[0]["exit_code"] == 75
else:
# SIGKILL the entire disposable instance, as container teardown does.
app.kill_remaining()
assert app.process.returncode == -signal.SIGKILL
assert app.rows()[0]["status"] == "running"
wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit")
saved = read_json(work / "checkpoint.json")
clean = shutdown_signal == signal.SIGTERM
assert saved["clean_shutdown"] is clean
assert saved["data"] == {"next": 3 if clean else 0}
assert upload.read_bytes() == UPLOAD
assert json.loads((work / "input.json").read_text()) == saved_input
stale = time.time() - 45 * 86400
for path in (work, work / "input.json", upload.parent, upload):
os.utime(path, (stale, stale))
app.start()
recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id)
app.wait_progress(recovery["id"], "Recovery waiting")
observation = read_json(app.state_root / "tasks/ledger/observation.json")
assert observation == {
"checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean,
"ledger": ["0", "1", "2"],
}
status, states = app.request("GET", "/tasks/state")
assert status == 200
assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering"
assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"]
status, conflict = app.request("POST", "/tasks/contender/task.py/run")
assert status == 409, conflict
assert conflict["blocked_by"]["run_id"] == recovery["id"]
assert not any(row["target_path"] == "contender/task.py" for row in app.rows())
# A real completion invokes cleanup while recovery still needs its ancestor's files.
cleanup_probe = runs / "999999.input.json"
cleanup_probe.write_text("{}\n")
maintenance_id = app.run("maintenance/task.py")
app.wait_status(maintenance_id, "succeeded")
wait_until(lambda: not cleanup_probe.exists(), "completion cleanup")
assert upload.read_bytes() == UPLOAD
assert json.loads((work / "input.json").read_text()) == saved_input
assert app.rows()[0]["status"] == "interrupted"
(app.state_root / "tasks/ledger/finish").touch()
app.wait_status(recovery["id"], "succeeded")
assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)]
wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup")
app.stop()
assert not app.pids(), "A task escaped registration/shutdown"
def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app):
app = production_app
app.write_task("blocking", "task.py", BLOCKING_TASK)
app.write_task("watcher", "internal/task.py", BLOCKING_TASK)
(app.task_root / "watcher/background-tasks.json").write_text(json.dumps({
"version": 1,
"tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}],
}))
app.start()
run_id = app.run("blocking/task.py")
app.wait_progress(run_id, "Blocking")
background_work = app.state_root / "background/watcher/watcher.work"
wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking",
"background worker to start")
work = app.state_root / "runs" / f"{run_id}.work"
task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json")
assert all(process_running(pid) for pid in task_pids)
elapsed = app.stop()
assert elapsed >= app.grace * .8, "The hard deadline was not exercised"
wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit")
assert not app.pids(), "A foreground/background task escaped shutdown registration"
assert read_json(work / "checkpoint.json")["clean_shutdown"] is False
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
assert (work / "input.json").exists()
def test_production_graceful_pause_does_not_leave_live_descendants(production_app):
app = production_app
source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n")
app.write_task("cooperative", "task.py", source)
app.start()
run_id = app.run("cooperative/task.py")
app.wait_progress(run_id, "Blocking")
work = app.state_root / "runs" / f"{run_id}.work"
worker_pid, child_pid = read_json(work / "pids.json")
app.stop()
assert read_json(work / "checkpoint.json")["clean_shutdown"] is True
assert not process_running(worker_pid)
wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5)
def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app):
app = production_app
source = '''
from app import task_runtime as runtime
runtime.progress("Started", force=True)
while True:
runtime.sleep(.02)
'''
groups = [f"queue_{index}" for index in range(12)]
for group in groups:
app.write_task(group, "task.py", source)
app.start()
first_id = app.run(f"{groups[0]}/task.py")
app.wait_progress(first_id, "Started")
def submit(group):
try:
return app.request("POST", f"/tasks/{group}/task.py/run")[0]
except (ConnectionError, TimeoutError, http.client.RemoteDisconnected):
# The listener may close before this in-flight request is accepted.
return None
with ThreadPoolExecutor(max_workers=6) as requests:
pending = [requests.submit(submit, group) for group in groups[1:]]
app.stop()
statuses = [request.result(timeout=3) for request in pending]
assert all(status in {202, 503, None} for status in statuses), statuses
assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration"
assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text()
def test_production_bind_failure_stops_workers_started_by_recovery(production_app):
app = production_app
source = '''
from app import task_runtime as runtime
def main():
runtime.progress("Started", force=True)
while True:
runtime.sleep(.02)
def recover():
main()
if __name__ == "__main__":
main()
'''
app.write_task("recoverable", "task.py", source)
app.start()
run_id = app.run("recoverable/task.py")
app.wait_progress(run_id, "Started")
app.stop()
assert app.rows()[0]["status"] == "interrupted"
# Delay startup cleanup without patching app code, until recovery is running.
cleanup_gate = app.state_root / "tasks/.orphaned-groups.json"
os.mkfifo(cleanup_gate)
with socket.socket() as occupied:
occupied.bind(("127.0.0.1", 0))
occupied.listen()
app.launch(occupied.getsockname()[1])
recovery = wait_until(
lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None),
"startup recovery registration",
)
app.wait_progress(recovery["id"], "Started")
def release_cleanup():
try:
descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK)
except OSError as error:
if error.errno == errno.ENXIO:
return False
raise
with os.fdopen(descriptor, "w") as gate:
json.dump({}, gate)
return True
wait_until(release_cleanup, "startup cleanup gate", timeout=2)
assert app.process.wait(timeout=4) != 0
assert "Address already in use" in app.log_path.read_text()
assert any(row["recovery_of_run_id"] == run_id for row in app.rows())
wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)
+81
View File
@@ -0,0 +1,81 @@
import json
import signal
import pytest
from app import task_runtime as runtime
@pytest.fixture
def work(tmp_path,monkeypatch):
monkeypatch.setenv('SERVER_MAINTENANCE_WORK',str(tmp_path))
old=signal.getsignal(signal.SIGUSR1)
runtime.initialize()
yield tmp_path
signal.signal(signal.SIGUSR1,old)
runtime._pause_requested=False
runtime._defer=0
def test_safe_point_saves_latest_memory_state_and_marks_clean(work):
runtime.checkpoint({'next':1},force=True)
runtime.checkpoint({'next':2})
assert json.loads((work/'checkpoint.json').read_text())['data']=={'next':1}
runtime.request_pause()
with pytest.raises(runtime.TaskPaused):
runtime.check_stop()
saved=json.loads((work/'checkpoint.json').read_text())
assert saved['data']=={'next':2}
assert saved['clean_shutdown'] is True
runtime.initialize()
assert runtime.was_clean_shutdown() is True
assert json.loads((work/'checkpoint.json').read_text())['clean_shutdown'] is False
def test_small_critical_operation_finishes_before_pause(work):
with pytest.raises(runtime.TaskPaused):
with runtime.critical_section():
runtime.request_pause()
runtime.checkpoint({'committed':True},force=True)
assert runtime.checkpoint_data()=={'committed':True}
saved=json.loads((work/'checkpoint.json').read_text())
assert saved['data']=={'committed':True}
assert saved['clean_shutdown'] is True
def test_sleep_is_interruptible(work):
runtime.request_pause()
with pytest.raises(runtime.TaskPaused):
runtime.sleep(30)
def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
clock=[100.]
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
progress=runtime.Progress({'Removing files':10,'Linking files':20})
progress.update('Removing files',0,force=True)
clock[0]+=30
progress.update('Removing files',5,force=True)
text=json.loads((work/'progress.json').read_text())['text']
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
assert (work/'progress.json').stat().st_mode&0o777==0o600
def test_runtime_does_not_write_or_pause_direct_scripts_without_work(monkeypatch):
monkeypatch.delenv('SERVER_MAINTENANCE_WORK',raising=False)
runtime.request_pause()
runtime.checkpoint({'ignored':True})
runtime.progress('hello',force=True)
assert runtime.checkpoint_data()=={}
runtime._pause_requested=False
def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
clock=[100.]
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
progress=runtime.Progress({'Items':50})
progress.update('Items',39,force=True)
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
clock[0]+=60
progress.update('Items',40,force=True)
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
+32
View File
@@ -141,3 +141,35 @@ def test_removing_file_input_from_task_cleans_up_unused_review_upload(settings):
manifest.write_text(json.dumps(definition))
app.extensions["run_manager"].cleanup_transient_files()
assert not original.parent.exists()
def test_conflicting_execute_is_linked_and_keeps_review_upload(settings, monkeypatch):
app, client, review_id = start_review(settings)
manager = app.extensions["run_manager"]
original = Path(json.loads(manager.store.get(review_id).input_json)["csv"])
write_task(settings.task_root, "imports", "other.py", 'print("done")\n')
monkeypatch.setattr(manager, "_wake_worker", lambda *_: None)
blocker, _ = manager.run_task(app.extensions["catalog"].task("imports/other.py"))
page = client.get(f"/runs/{review_id}")
assert b'fieldset class="task-controls" disabled' in page.data
assert f'href="/runs/{blocker.id}"'.encode() in page.data
response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
assert response.status_code == 409
assert response.get_json()["blocked_by"]["run_id"] == blocker.id
assert original.is_file()
manager.store.finish(blocker.id, status="cancelled")
accepted = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
assert accepted.status_code == 202
assert accepted.get_json()["detail_url"].startswith("/runs/")
assert accepted.get_json()["run_id"] != review_id
def test_shutdown_during_execute_keeps_review_upload(settings):
app, client, review_id = start_review(settings)
manager = app.extensions["run_manager"]
original = Path(json.loads(manager.store.get(review_id).input_json)["csv"])
manager._closing.set()
response = client.post(f"/runs/{review_id}/execute", headers={"Accept": "application/json"})
assert response.status_code == 503
assert "shutting down" in response.get_json()["error"]
assert original.is_file()
+56
View File
@@ -0,0 +1,56 @@
from urllib.error import HTTPError, URLError
import pytest
from app.worker import service_unavailable
from app import worker
@pytest.mark.parametrize('error', [
ConnectionRefusedError(), TimeoutError(), URLError(OSError('connection refused')),
HTTPError('http://example.invalid',503,'unavailable',{},None),
])
def test_service_failures_can_retry_recovery(error):
wrapped=RuntimeError('Service request failed')
wrapped.__cause__=error
assert service_unavailable(wrapped)
@pytest.mark.parametrize('error', [
RuntimeError('Unrecognized file'), FileNotFoundError(), PermissionError(),
HTTPError('http://example.invalid',401,'unauthorized',{},None),
HTTPError('http://example.invalid',400,'bad input',{},None),
])
def test_task_and_authentication_errors_are_not_automatically_repeated(error):
assert not service_unavailable(error)
def test_database_starting_up_is_transient_but_authentication_is_not():
error=RuntimeError('Database unavailable')
error.sqlstate='57P03'
assert service_unavailable(error)
error.sqlstate='28P01'
assert not service_unavailable(error)
def test_recovery_rechecks_live_state_after_lost_service_response(tmp_path, monkeypatch):
ledger = []
calls = []
waits = []
def recover():
calls.append(list(ledger))
if not ledger:
ledger.append('applied')
raise URLError(ConnectionResetError('response lost'))
monkeypatch.setattr(worker.sys, 'argv', ['worker', str(tmp_path/'task.py')])
monkeypatch.setenv('SERVER_MAINTENANCE_RECOVER','1')
monkeypatch.setattr(worker.runtime,'initialize',lambda:None)
monkeypatch.setattr(worker.runtime,'progress',lambda *a,**kw:None)
monkeypatch.setattr(worker.runtime,'sleep',waits.append)
monkeypatch.setattr(worker.runpy,'run_path',lambda *a,**kw:{'recover':recover})
worker.main()
assert ledger==['applied']
assert calls==[[],['applied']]
assert waits==[2]