"""Recovery through the real Waitress entry point, without manager test doubles.""" from __future__ import annotations import errno import http.client import json import os import signal import socket import sqlite3 import subprocess import sys import time from concurrent.futures import ThreadPoolExecutor from contextlib import closing from io import BytesIO from pathlib import Path import pytest from werkzeug.test import EnvironBuilder pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration") PROJECT_ROOT = Path(__file__).resolve().parents[1] UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n" SECRET = "disposable-process-test-secret" LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger.""" import json import os from pathlib import Path from app import task_runtime as runtime RESOURCES = ("shared-ledger",) state = Path(os.environ["SERVER_MAINTENANCE_STATE"]) def values(): data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text()) assert data["secret"] == "disposable-process-test-secret" assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n" return data def main(): data = values() ledger = state / "ledger.txt" existing = ledger.read_text().splitlines() if ledger.exists() else [] if not runtime.checkpoint_data(): runtime.checkpoint({"next": 0}, force=True) items = Path(data["items"]).read_text().splitlines() for number in range(len(existing), len(items)): runtime.check_stop() with ledger.open("a") as stream: stream.write(items[number] + "\\n") runtime.checkpoint({"next": number + 1}) if number == 2: runtime.progress("Original waiting", force=True) while not (state / "finish").exists(): runtime.sleep(.02) def recover(): values() (state / "observation.json").write_text(json.dumps({ "checkpoint": runtime.checkpoint_data(), "clean_shutdown": runtime.was_clean_shutdown(), "ledger": (state / "ledger.txt").read_text().splitlines(), })) runtime.progress("Recovery waiting", force=True) while not (state / "finish").exists(): runtime.sleep(.02) main() if __name__ == "__main__": main() ''' BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline.""" import json import os import subprocess import sys import time from pathlib import Path from app import task_runtime as runtime runtime.checkpoint({"next": 0}, force=True) child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) runtime.checkpoint({"next": 1}) Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text( json.dumps([os.getpid(), child.pid])) runtime.progress("Blocking", force=True) time.sleep(60) ''' def wait_until(condition, description, timeout=8): deadline = time.monotonic() + timeout while time.monotonic() < deadline: result = condition() if result: return result time.sleep(.02) pytest.fail(f"Timed out waiting for {description}") def read_json(path): try: return json.loads(path.read_text()) except FileNotFoundError: return None def process_running(pid): try: status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1] except FileNotFoundError: return False return status[0] not in {"Z", "X"} class ProductionApp: def __init__(self, root): self.task_root = root / "tasks" self.state_root = root / "state" self.credentials_root = root / "credentials" for path in (self.task_root, self.state_root, self.credentials_root): path.mkdir() self.database = self.state_root / "server-maintenance.sqlite3" self.log_path = root / "server.log" self.process = None self.port = None self.grace = 1 def write_task(self, group, filename, source): path = self.task_root / group / filename path.parent.mkdir(parents=True, exist_ok=True) path.write_text(source, encoding="utf-8") return path def start(self): with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) port = listener.getsockname()[1] self.launch(port) def ready(): assert self.process.poll() is None, self.log_path.read_text() try: status, _ = self.request("GET", "/tasks/state") except (ConnectionError, TimeoutError): return False return status == 200 wait_until(ready, f"Waitress startup; log: {self.log_path}") def launch(self, port): assert self.process is None or self.process.poll() is not None self.port = port environment = {key: value for key, value in os.environ.items() if not key.startswith("SERVER_MAINTENANCE_")} environment.update({ "PYTHONPATH": str(PROJECT_ROOT), "PYTHONUNBUFFERED": "1", "PYTHONDONTWRITEBYTECODE": "1", "SERVER_MAINTENANCE_HOST": "127.0.0.1", "SERVER_MAINTENANCE_PORT": str(self.port), "SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root), "SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root), "SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root), "SERVER_MAINTENANCE_PYTHON": sys.executable, "SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace), }) with self.log_path.open("ab") as log: self.process = subprocess.Popen( [sys.executable, "-m", "app"], cwd=PROJECT_ROOT, env=environment, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, ) def request(self, method, path, data=None): headers = {"Accept": "application/json"} body = None if data is not None: with closing(EnvironBuilder(method=method, data=data)) as builder: environ = builder.get_environ() body = environ["wsgi.input"].read() headers["Content-Type"] = environ["CONTENT_TYPE"] connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2) try: connection.request(method, path, body=body, headers=headers) response = connection.getresponse() payload = response.read() assert response.getheader("Content-Type", "").startswith("application/json"), payload return response.status, json.loads(payload) finally: connection.close() def rows(self): with closing(sqlite3.connect(self.database)) as connection: connection.row_factory = sqlite3.Row return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")] def run(self, task, data=None): status, response = self.request("POST", f"/tasks/{task}/run", data) assert status == 202, response return response["run_id"] def wait_status(self, run_id, status): def matching(): run = next(row for row in self.rows() if row["id"] == run_id) if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}: assert run["status"] == status, (run, self.log_path.read_text()) return run if run["status"] == status else None return wait_until(matching, f"run {run_id} to become {status}") def wait_progress(self, run_id, text): path = self.state_root / "runs" / f"{run_id}.progress.json" wait_until(lambda: (read_json(path) or {}).get("text") == text, text) def pids(self): # Match only this disposable instance, including separately-sessioned workers. marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode() found = [] for path in Path("/proc").iterdir(): if not path.name.isdigit(): continue try: environment = (path / "environ").read_bytes().split(b"\0") except (FileNotFoundError, ProcessLookupError, PermissionError): continue if marker in environment: found.append(int(path.name)) return found def kill_remaining(self): def kill_snapshot(): pids = self.pids() for pid in pids: try: os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass return not pids kill_snapshot() if self.process is not None: self.process.wait(timeout=3) wait_until(kill_snapshot, "disposable instance teardown", timeout=3) def stop(self): started = time.monotonic() self.process.send_signal(signal.SIGTERM) try: code = self.process.wait(timeout=self.grace + 3) except subprocess.TimeoutExpired: pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}") elapsed = time.monotonic() - started assert code == 0, self.log_path.read_text() assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text()) return elapsed @pytest.fixture def production_app(tmp_path): app = ProductionApp(tmp_path) try: yield app finally: app.kill_remaining() @pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"]) def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal): app = production_app ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK) (ledger_task.parent / "task-inputs.json").write_text(json.dumps({ "version": 1, "tasks": {"task.py": {"inputs": [ {"name": "items", "label": "Items", "type": "file", "required": True, "accept": [".txt"], "maximum_bytes": 1024}, {"name": "secret", "label": "Secret", "type": "text", "required": True, "sensitive": True}, ]}}, })) app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n') app.write_task("maintenance", "task.py", 'print("done")\n') app.start() original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")}) app.wait_progress(original_id, "Original waiting") runs = app.state_root / "runs" work = runs / f"{original_id}.work" saved_input = json.loads((work / "input.json").read_text()) upload = Path(saved_input["items"]) assert saved_input["secret"] == SECRET assert SECRET not in app.rows()[0]["input_json"] assert read_json(work / "checkpoint.json")["data"] == {"next": 0} old_pids = app.pids() if shutdown_signal == signal.SIGTERM: app.stop() assert app.rows()[0]["status"] == "interrupted" assert app.rows()[0]["exit_code"] == 75 else: # SIGKILL the entire disposable instance, as container teardown does. app.kill_remaining() assert app.process.returncode == -signal.SIGKILL assert app.rows()[0]["status"] == "running" wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit") saved = read_json(work / "checkpoint.json") clean = shutdown_signal == signal.SIGTERM assert saved["clean_shutdown"] is clean assert saved["data"] == {"next": 3 if clean else 0} assert upload.read_bytes() == UPLOAD assert json.loads((work / "input.json").read_text()) == saved_input stale = time.time() - 45 * 86400 for path in (work, work / "input.json", upload.parent, upload): os.utime(path, (stale, stale)) app.start() recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id) app.wait_progress(recovery["id"], "Recovery waiting") observation = read_json(app.state_root / "tasks/ledger/observation.json") assert observation == { "checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean, "ledger": ["0", "1", "2"], } status, states = app.request("GET", "/tasks/state") assert status == 200 assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering" assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"] status, conflict = app.request("POST", "/tasks/contender/task.py/run") assert status == 409, conflict assert conflict["blocked_by"]["run_id"] == recovery["id"] assert not any(row["target_path"] == "contender/task.py" for row in app.rows()) # A real completion invokes cleanup while recovery still needs its ancestor's files. cleanup_probe = runs / "999999.input.json" cleanup_probe.write_text("{}\n") maintenance_id = app.run("maintenance/task.py") app.wait_status(maintenance_id, "succeeded") wait_until(lambda: not cleanup_probe.exists(), "completion cleanup") assert upload.read_bytes() == UPLOAD assert json.loads((work / "input.json").read_text()) == saved_input assert app.rows()[0]["status"] == "interrupted" (app.state_root / "tasks/ledger/finish").touch() app.wait_status(recovery["id"], "succeeded") assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)] wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup") app.stop() assert not app.pids(), "A task escaped registration/shutdown" def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app): app = production_app app.write_task("blocking", "task.py", BLOCKING_TASK) app.write_task("watcher", "internal/task.py", BLOCKING_TASK) (app.task_root / "watcher/background-tasks.json").write_text(json.dumps({ "version": 1, "tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}], })) app.start() run_id = app.run("blocking/task.py") app.wait_progress(run_id, "Blocking") background_work = app.state_root / "background/watcher/watcher.work" wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking", "background worker to start") work = app.state_root / "runs" / f"{run_id}.work" task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json") assert all(process_running(pid) for pid in task_pids) elapsed = app.stop() assert elapsed >= app.grace * .8, "The hard deadline was not exercised" wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit") assert not app.pids(), "A foreground/background task escaped shutdown registration" assert read_json(work / "checkpoint.json")["clean_shutdown"] is False assert read_json(work / "checkpoint.json")["data"] == {"next": 0} assert (work / "input.json").exists() def test_production_graceful_pause_does_not_leave_live_descendants(production_app): app = production_app source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n") app.write_task("cooperative", "task.py", source) app.start() run_id = app.run("cooperative/task.py") app.wait_progress(run_id, "Blocking") work = app.state_root / "runs" / f"{run_id}.work" worker_pid, child_pid = read_json(work / "pids.json") app.stop() assert read_json(work / "checkpoint.json")["clean_shutdown"] is True assert not process_running(worker_pid) wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5) def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app): app = production_app source = ''' from app import task_runtime as runtime runtime.progress("Started", force=True) while True: runtime.sleep(.02) ''' groups = [f"queue_{index}" for index in range(12)] for group in groups: app.write_task(group, "task.py", source) app.start() first_id = app.run(f"{groups[0]}/task.py") app.wait_progress(first_id, "Started") def submit(group): try: return app.request("POST", f"/tasks/{group}/task.py/run")[0] except (ConnectionError, TimeoutError, http.client.RemoteDisconnected): # The listener may close before this in-flight request is accepted. return None with ThreadPoolExecutor(max_workers=6) as requests: pending = [requests.submit(submit, group) for group in groups[1:]] app.stop() statuses = [request.result(timeout=3) for request in pending] assert all(status in {202, 503, None} for status in statuses), statuses assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration" assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text() def test_production_bind_failure_stops_workers_started_by_recovery(production_app): app = production_app source = ''' from app import task_runtime as runtime def main(): runtime.progress("Started", force=True) while True: runtime.sleep(.02) def recover(): main() if __name__ == "__main__": main() ''' app.write_task("recoverable", "task.py", source) app.start() run_id = app.run("recoverable/task.py") app.wait_progress(run_id, "Started") app.stop() assert app.rows()[0]["status"] == "interrupted" # Delay startup cleanup without patching app code, until recovery is running. cleanup_gate = app.state_root / "tasks/.orphaned-groups.json" os.mkfifo(cleanup_gate) with socket.socket() as occupied: occupied.bind(("127.0.0.1", 0)) occupied.listen() app.launch(occupied.getsockname()[1]) recovery = wait_until( lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None), "startup recovery registration", ) app.wait_progress(recovery["id"], "Started") def release_cleanup(): try: descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK) except OSError as error: if error.errno == errno.ENXIO: return False raise with os.fdopen(descriptor, "w") as gate: json.dump({}, gate) return True wait_until(release_cleanup, "startup cleanup gate", timeout=2) assert app.process.wait(timeout=4) != 0 assert "Address already in use" in app.log_path.read_text() assert any(row["recovery_of_run_id"] == run_id for row in app.rows()) wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)