Files
docker-images/python-tools/tests/test_shutdown_process.py
T
ajp_anton 3bfd199b9c
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Add cooperative task recovery and persistent live status
2026-10-29 22:12:04 +00:00

479 lines
19 KiB
Python

"""Recovery through the real Waitress entry point, without manager test doubles."""
from __future__ import annotations
import errno
import http.client
import json
import os
import signal
import socket
import sqlite3
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from contextlib import closing
from io import BytesIO
from pathlib import Path
import pytest
from werkzeug.test import EnvironBuilder
pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration")
PROJECT_ROOT = Path(__file__).resolve().parents[1]
UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n"
SECRET = "disposable-process-test-secret"
LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger."""
import json
import os
from pathlib import Path
from app import task_runtime as runtime
RESOURCES = ("shared-ledger",)
state = Path(os.environ["SERVER_MAINTENANCE_STATE"])
def values():
data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
assert data["secret"] == "disposable-process-test-secret"
assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n"
return data
def main():
data = values()
ledger = state / "ledger.txt"
existing = ledger.read_text().splitlines() if ledger.exists() else []
if not runtime.checkpoint_data():
runtime.checkpoint({"next": 0}, force=True)
items = Path(data["items"]).read_text().splitlines()
for number in range(len(existing), len(items)):
runtime.check_stop()
with ledger.open("a") as stream:
stream.write(items[number] + "\\n")
runtime.checkpoint({"next": number + 1})
if number == 2:
runtime.progress("Original waiting", force=True)
while not (state / "finish").exists():
runtime.sleep(.02)
def recover():
values()
(state / "observation.json").write_text(json.dumps({
"checkpoint": runtime.checkpoint_data(),
"clean_shutdown": runtime.was_clean_shutdown(),
"ledger": (state / "ledger.txt").read_text().splitlines(),
}))
runtime.progress("Recovery waiting", force=True)
while not (state / "finish").exists():
runtime.sleep(.02)
main()
if __name__ == "__main__":
main()
'''
BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from app import task_runtime as runtime
runtime.checkpoint({"next": 0}, force=True)
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
runtime.checkpoint({"next": 1})
Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text(
json.dumps([os.getpid(), child.pid]))
runtime.progress("Blocking", force=True)
time.sleep(60)
'''
def wait_until(condition, description, timeout=8):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = condition()
if result:
return result
time.sleep(.02)
pytest.fail(f"Timed out waiting for {description}")
def read_json(path):
try:
return json.loads(path.read_text())
except FileNotFoundError:
return None
def process_running(pid):
try:
status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1]
except FileNotFoundError:
return False
return status[0] not in {"Z", "X"}
class ProductionApp:
def __init__(self, root):
self.task_root = root / "tasks"
self.state_root = root / "state"
self.credentials_root = root / "credentials"
for path in (self.task_root, self.state_root, self.credentials_root):
path.mkdir()
self.database = self.state_root / "server-maintenance.sqlite3"
self.log_path = root / "server.log"
self.process = None
self.port = None
self.grace = 1
def write_task(self, group, filename, source):
path = self.task_root / group / filename
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source, encoding="utf-8")
return path
def start(self):
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
port = listener.getsockname()[1]
self.launch(port)
def ready():
assert self.process.poll() is None, self.log_path.read_text()
try:
status, _ = self.request("GET", "/tasks/state")
except (ConnectionError, TimeoutError):
return False
return status == 200
wait_until(ready, f"Waitress startup; log: {self.log_path}")
def launch(self, port):
assert self.process is None or self.process.poll() is not None
self.port = port
environment = {key: value for key, value in os.environ.items()
if not key.startswith("SERVER_MAINTENANCE_")}
environment.update({
"PYTHONPATH": str(PROJECT_ROOT),
"PYTHONUNBUFFERED": "1",
"PYTHONDONTWRITEBYTECODE": "1",
"SERVER_MAINTENANCE_HOST": "127.0.0.1",
"SERVER_MAINTENANCE_PORT": str(self.port),
"SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root),
"SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root),
"SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root),
"SERVER_MAINTENANCE_PYTHON": sys.executable,
"SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace),
})
with self.log_path.open("ab") as log:
self.process = subprocess.Popen(
[sys.executable, "-m", "app"], cwd=PROJECT_ROOT,
env=environment, stdin=subprocess.DEVNULL, stdout=log,
stderr=subprocess.STDOUT, start_new_session=True,
)
def request(self, method, path, data=None):
headers = {"Accept": "application/json"}
body = None
if data is not None:
with closing(EnvironBuilder(method=method, data=data)) as builder:
environ = builder.get_environ()
body = environ["wsgi.input"].read()
headers["Content-Type"] = environ["CONTENT_TYPE"]
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2)
try:
connection.request(method, path, body=body, headers=headers)
response = connection.getresponse()
payload = response.read()
assert response.getheader("Content-Type", "").startswith("application/json"), payload
return response.status, json.loads(payload)
finally:
connection.close()
def rows(self):
with closing(sqlite3.connect(self.database)) as connection:
connection.row_factory = sqlite3.Row
return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")]
def run(self, task, data=None):
status, response = self.request("POST", f"/tasks/{task}/run", data)
assert status == 202, response
return response["run_id"]
def wait_status(self, run_id, status):
def matching():
run = next(row for row in self.rows() if row["id"] == run_id)
if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}:
assert run["status"] == status, (run, self.log_path.read_text())
return run if run["status"] == status else None
return wait_until(matching, f"run {run_id} to become {status}")
def wait_progress(self, run_id, text):
path = self.state_root / "runs" / f"{run_id}.progress.json"
wait_until(lambda: (read_json(path) or {}).get("text") == text, text)
def pids(self):
# Match only this disposable instance, including separately-sessioned workers.
marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode()
found = []
for path in Path("/proc").iterdir():
if not path.name.isdigit():
continue
try:
environment = (path / "environ").read_bytes().split(b"\0")
except (FileNotFoundError, ProcessLookupError, PermissionError):
continue
if marker in environment:
found.append(int(path.name))
return found
def kill_remaining(self):
def kill_snapshot():
pids = self.pids()
for pid in pids:
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
pass
return not pids
kill_snapshot()
if self.process is not None:
self.process.wait(timeout=3)
wait_until(kill_snapshot, "disposable instance teardown", timeout=3)
def stop(self):
started = time.monotonic()
self.process.send_signal(signal.SIGTERM)
try:
code = self.process.wait(timeout=self.grace + 3)
except subprocess.TimeoutExpired:
pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}")
elapsed = time.monotonic() - started
assert code == 0, self.log_path.read_text()
assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text())
return elapsed
@pytest.fixture
def production_app(tmp_path):
app = ProductionApp(tmp_path)
try:
yield app
finally:
app.kill_remaining()
@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"])
def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal):
app = production_app
ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK)
(ledger_task.parent / "task-inputs.json").write_text(json.dumps({
"version": 1,
"tasks": {"task.py": {"inputs": [
{"name": "items", "label": "Items", "type": "file", "required": True,
"accept": [".txt"], "maximum_bytes": 1024},
{"name": "secret", "label": "Secret", "type": "text", "required": True,
"sensitive": True},
]}},
}))
app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n')
app.write_task("maintenance", "task.py", 'print("done")\n')
app.start()
original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")})
app.wait_progress(original_id, "Original waiting")
runs = app.state_root / "runs"
work = runs / f"{original_id}.work"
saved_input = json.loads((work / "input.json").read_text())
upload = Path(saved_input["items"])
assert saved_input["secret"] == SECRET
assert SECRET not in app.rows()[0]["input_json"]
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
old_pids = app.pids()
if shutdown_signal == signal.SIGTERM:
app.stop()
assert app.rows()[0]["status"] == "interrupted"
assert app.rows()[0]["exit_code"] == 75
else:
# SIGKILL the entire disposable instance, as container teardown does.
app.kill_remaining()
assert app.process.returncode == -signal.SIGKILL
assert app.rows()[0]["status"] == "running"
wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit")
saved = read_json(work / "checkpoint.json")
clean = shutdown_signal == signal.SIGTERM
assert saved["clean_shutdown"] is clean
assert saved["data"] == {"next": 3 if clean else 0}
assert upload.read_bytes() == UPLOAD
assert json.loads((work / "input.json").read_text()) == saved_input
stale = time.time() - 45 * 86400
for path in (work, work / "input.json", upload.parent, upload):
os.utime(path, (stale, stale))
app.start()
recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id)
app.wait_progress(recovery["id"], "Recovery waiting")
observation = read_json(app.state_root / "tasks/ledger/observation.json")
assert observation == {
"checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean,
"ledger": ["0", "1", "2"],
}
status, states = app.request("GET", "/tasks/state")
assert status == 200
assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering"
assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"]
status, conflict = app.request("POST", "/tasks/contender/task.py/run")
assert status == 409, conflict
assert conflict["blocked_by"]["run_id"] == recovery["id"]
assert not any(row["target_path"] == "contender/task.py" for row in app.rows())
# A real completion invokes cleanup while recovery still needs its ancestor's files.
cleanup_probe = runs / "999999.input.json"
cleanup_probe.write_text("{}\n")
maintenance_id = app.run("maintenance/task.py")
app.wait_status(maintenance_id, "succeeded")
wait_until(lambda: not cleanup_probe.exists(), "completion cleanup")
assert upload.read_bytes() == UPLOAD
assert json.loads((work / "input.json").read_text()) == saved_input
assert app.rows()[0]["status"] == "interrupted"
(app.state_root / "tasks/ledger/finish").touch()
app.wait_status(recovery["id"], "succeeded")
assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)]
wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup")
app.stop()
assert not app.pids(), "A task escaped registration/shutdown"
def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app):
app = production_app
app.write_task("blocking", "task.py", BLOCKING_TASK)
app.write_task("watcher", "internal/task.py", BLOCKING_TASK)
(app.task_root / "watcher/background-tasks.json").write_text(json.dumps({
"version": 1,
"tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}],
}))
app.start()
run_id = app.run("blocking/task.py")
app.wait_progress(run_id, "Blocking")
background_work = app.state_root / "background/watcher/watcher.work"
wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking",
"background worker to start")
work = app.state_root / "runs" / f"{run_id}.work"
task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json")
assert all(process_running(pid) for pid in task_pids)
elapsed = app.stop()
assert elapsed >= app.grace * .8, "The hard deadline was not exercised"
wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit")
assert not app.pids(), "A foreground/background task escaped shutdown registration"
assert read_json(work / "checkpoint.json")["clean_shutdown"] is False
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
assert (work / "input.json").exists()
def test_production_graceful_pause_does_not_leave_live_descendants(production_app):
app = production_app
source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n")
app.write_task("cooperative", "task.py", source)
app.start()
run_id = app.run("cooperative/task.py")
app.wait_progress(run_id, "Blocking")
work = app.state_root / "runs" / f"{run_id}.work"
worker_pid, child_pid = read_json(work / "pids.json")
app.stop()
assert read_json(work / "checkpoint.json")["clean_shutdown"] is True
assert not process_running(worker_pid)
wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5)
def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app):
app = production_app
source = '''
from app import task_runtime as runtime
runtime.progress("Started", force=True)
while True:
runtime.sleep(.02)
'''
groups = [f"queue_{index}" for index in range(12)]
for group in groups:
app.write_task(group, "task.py", source)
app.start()
first_id = app.run(f"{groups[0]}/task.py")
app.wait_progress(first_id, "Started")
def submit(group):
try:
return app.request("POST", f"/tasks/{group}/task.py/run")[0]
except (ConnectionError, TimeoutError, http.client.RemoteDisconnected):
# The listener may close before this in-flight request is accepted.
return None
with ThreadPoolExecutor(max_workers=6) as requests:
pending = [requests.submit(submit, group) for group in groups[1:]]
app.stop()
statuses = [request.result(timeout=3) for request in pending]
assert all(status in {202, 503, None} for status in statuses), statuses
assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration"
assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text()
def test_production_bind_failure_stops_workers_started_by_recovery(production_app):
app = production_app
source = '''
from app import task_runtime as runtime
def main():
runtime.progress("Started", force=True)
while True:
runtime.sleep(.02)
def recover():
main()
if __name__ == "__main__":
main()
'''
app.write_task("recoverable", "task.py", source)
app.start()
run_id = app.run("recoverable/task.py")
app.wait_progress(run_id, "Started")
app.stop()
assert app.rows()[0]["status"] == "interrupted"
# Delay startup cleanup without patching app code, until recovery is running.
cleanup_gate = app.state_root / "tasks/.orphaned-groups.json"
os.mkfifo(cleanup_gate)
with socket.socket() as occupied:
occupied.bind(("127.0.0.1", 0))
occupied.listen()
app.launch(occupied.getsockname()[1])
recovery = wait_until(
lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None),
"startup recovery registration",
)
app.wait_progress(recovery["id"], "Started")
def release_cleanup():
try:
descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK)
except OSError as error:
if error.errno == errno.ENXIO:
return False
raise
with os.fdopen(descriptor, "w") as gate:
json.dump({}, gate)
return True
wait_until(release_cleanup, "startup cleanup gate", timeout=2)
assert app.process.wait(timeout=4) != 0
assert "Address already in use" in app.log_path.read_text()
assert any(row["recovery_of_run_id"] == run_id for row in app.rows())
wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)