Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
479 lines
19 KiB
Python
479 lines
19 KiB
Python
"""Recovery through the real Waitress entry point, without manager test doubles."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import errno
|
|
import http.client
|
|
import json
|
|
import os
|
|
import signal
|
|
import socket
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import closing
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from werkzeug.test import EnvironBuilder
|
|
|
|
|
|
pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration")
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n"
|
|
SECRET = "disposable-process-test-secret"
|
|
|
|
|
|
LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from app import task_runtime as runtime
|
|
|
|
RESOURCES = ("shared-ledger",)
|
|
state = Path(os.environ["SERVER_MAINTENANCE_STATE"])
|
|
|
|
def values():
|
|
data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
|
assert data["secret"] == "disposable-process-test-secret"
|
|
assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n"
|
|
return data
|
|
|
|
def main():
|
|
data = values()
|
|
ledger = state / "ledger.txt"
|
|
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
|
if not runtime.checkpoint_data():
|
|
runtime.checkpoint({"next": 0}, force=True)
|
|
items = Path(data["items"]).read_text().splitlines()
|
|
for number in range(len(existing), len(items)):
|
|
runtime.check_stop()
|
|
with ledger.open("a") as stream:
|
|
stream.write(items[number] + "\\n")
|
|
runtime.checkpoint({"next": number + 1})
|
|
if number == 2:
|
|
runtime.progress("Original waiting", force=True)
|
|
while not (state / "finish").exists():
|
|
runtime.sleep(.02)
|
|
|
|
def recover():
|
|
values()
|
|
(state / "observation.json").write_text(json.dumps({
|
|
"checkpoint": runtime.checkpoint_data(),
|
|
"clean_shutdown": runtime.was_clean_shutdown(),
|
|
"ledger": (state / "ledger.txt").read_text().splitlines(),
|
|
}))
|
|
runtime.progress("Recovery waiting", force=True)
|
|
while not (state / "finish").exists():
|
|
runtime.sleep(.02)
|
|
main()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
'''
|
|
|
|
|
|
BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from app import task_runtime as runtime
|
|
|
|
runtime.checkpoint({"next": 0}, force=True)
|
|
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
|
|
runtime.checkpoint({"next": 1})
|
|
Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text(
|
|
json.dumps([os.getpid(), child.pid]))
|
|
runtime.progress("Blocking", force=True)
|
|
time.sleep(60)
|
|
'''
|
|
|
|
|
|
def wait_until(condition, description, timeout=8):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
result = condition()
|
|
if result:
|
|
return result
|
|
time.sleep(.02)
|
|
pytest.fail(f"Timed out waiting for {description}")
|
|
|
|
|
|
def read_json(path):
|
|
try:
|
|
return json.loads(path.read_text())
|
|
except FileNotFoundError:
|
|
return None
|
|
|
|
|
|
def process_running(pid):
|
|
try:
|
|
status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1]
|
|
except FileNotFoundError:
|
|
return False
|
|
return status[0] not in {"Z", "X"}
|
|
|
|
|
|
class ProductionApp:
|
|
def __init__(self, root):
|
|
self.task_root = root / "tasks"
|
|
self.state_root = root / "state"
|
|
self.credentials_root = root / "credentials"
|
|
for path in (self.task_root, self.state_root, self.credentials_root):
|
|
path.mkdir()
|
|
self.database = self.state_root / "server-maintenance.sqlite3"
|
|
self.log_path = root / "server.log"
|
|
self.process = None
|
|
self.port = None
|
|
self.grace = 1
|
|
|
|
def write_task(self, group, filename, source):
|
|
path = self.task_root / group / filename
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(source, encoding="utf-8")
|
|
return path
|
|
|
|
def start(self):
|
|
with socket.socket() as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
port = listener.getsockname()[1]
|
|
self.launch(port)
|
|
|
|
def ready():
|
|
assert self.process.poll() is None, self.log_path.read_text()
|
|
try:
|
|
status, _ = self.request("GET", "/tasks/state")
|
|
except (ConnectionError, TimeoutError):
|
|
return False
|
|
return status == 200
|
|
|
|
wait_until(ready, f"Waitress startup; log: {self.log_path}")
|
|
|
|
def launch(self, port):
|
|
assert self.process is None or self.process.poll() is not None
|
|
self.port = port
|
|
environment = {key: value for key, value in os.environ.items()
|
|
if not key.startswith("SERVER_MAINTENANCE_")}
|
|
environment.update({
|
|
"PYTHONPATH": str(PROJECT_ROOT),
|
|
"PYTHONUNBUFFERED": "1",
|
|
"PYTHONDONTWRITEBYTECODE": "1",
|
|
"SERVER_MAINTENANCE_HOST": "127.0.0.1",
|
|
"SERVER_MAINTENANCE_PORT": str(self.port),
|
|
"SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root),
|
|
"SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root),
|
|
"SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root),
|
|
"SERVER_MAINTENANCE_PYTHON": sys.executable,
|
|
"SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace),
|
|
})
|
|
with self.log_path.open("ab") as log:
|
|
self.process = subprocess.Popen(
|
|
[sys.executable, "-m", "app"], cwd=PROJECT_ROOT,
|
|
env=environment, stdin=subprocess.DEVNULL, stdout=log,
|
|
stderr=subprocess.STDOUT, start_new_session=True,
|
|
)
|
|
|
|
def request(self, method, path, data=None):
|
|
headers = {"Accept": "application/json"}
|
|
body = None
|
|
if data is not None:
|
|
with closing(EnvironBuilder(method=method, data=data)) as builder:
|
|
environ = builder.get_environ()
|
|
body = environ["wsgi.input"].read()
|
|
headers["Content-Type"] = environ["CONTENT_TYPE"]
|
|
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2)
|
|
try:
|
|
connection.request(method, path, body=body, headers=headers)
|
|
response = connection.getresponse()
|
|
payload = response.read()
|
|
assert response.getheader("Content-Type", "").startswith("application/json"), payload
|
|
return response.status, json.loads(payload)
|
|
finally:
|
|
connection.close()
|
|
|
|
def rows(self):
|
|
with closing(sqlite3.connect(self.database)) as connection:
|
|
connection.row_factory = sqlite3.Row
|
|
return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")]
|
|
|
|
def run(self, task, data=None):
|
|
status, response = self.request("POST", f"/tasks/{task}/run", data)
|
|
assert status == 202, response
|
|
return response["run_id"]
|
|
|
|
def wait_status(self, run_id, status):
|
|
def matching():
|
|
run = next(row for row in self.rows() if row["id"] == run_id)
|
|
if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}:
|
|
assert run["status"] == status, (run, self.log_path.read_text())
|
|
return run if run["status"] == status else None
|
|
return wait_until(matching, f"run {run_id} to become {status}")
|
|
|
|
def wait_progress(self, run_id, text):
|
|
path = self.state_root / "runs" / f"{run_id}.progress.json"
|
|
wait_until(lambda: (read_json(path) or {}).get("text") == text, text)
|
|
|
|
def pids(self):
|
|
# Match only this disposable instance, including separately-sessioned workers.
|
|
marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode()
|
|
found = []
|
|
for path in Path("/proc").iterdir():
|
|
if not path.name.isdigit():
|
|
continue
|
|
try:
|
|
environment = (path / "environ").read_bytes().split(b"\0")
|
|
except (FileNotFoundError, ProcessLookupError, PermissionError):
|
|
continue
|
|
if marker in environment:
|
|
found.append(int(path.name))
|
|
return found
|
|
|
|
def kill_remaining(self):
|
|
def kill_snapshot():
|
|
pids = self.pids()
|
|
for pid in pids:
|
|
try:
|
|
os.kill(pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
return not pids
|
|
|
|
kill_snapshot()
|
|
if self.process is not None:
|
|
self.process.wait(timeout=3)
|
|
wait_until(kill_snapshot, "disposable instance teardown", timeout=3)
|
|
|
|
def stop(self):
|
|
started = time.monotonic()
|
|
self.process.send_signal(signal.SIGTERM)
|
|
try:
|
|
code = self.process.wait(timeout=self.grace + 3)
|
|
except subprocess.TimeoutExpired:
|
|
pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}")
|
|
elapsed = time.monotonic() - started
|
|
assert code == 0, self.log_path.read_text()
|
|
assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text())
|
|
return elapsed
|
|
|
|
|
|
@pytest.fixture
|
|
def production_app(tmp_path):
|
|
app = ProductionApp(tmp_path)
|
|
try:
|
|
yield app
|
|
finally:
|
|
app.kill_remaining()
|
|
|
|
|
|
@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"])
|
|
def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal):
|
|
app = production_app
|
|
ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK)
|
|
(ledger_task.parent / "task-inputs.json").write_text(json.dumps({
|
|
"version": 1,
|
|
"tasks": {"task.py": {"inputs": [
|
|
{"name": "items", "label": "Items", "type": "file", "required": True,
|
|
"accept": [".txt"], "maximum_bytes": 1024},
|
|
{"name": "secret", "label": "Secret", "type": "text", "required": True,
|
|
"sensitive": True},
|
|
]}},
|
|
}))
|
|
app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n')
|
|
app.write_task("maintenance", "task.py", 'print("done")\n')
|
|
app.start()
|
|
original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")})
|
|
app.wait_progress(original_id, "Original waiting")
|
|
runs = app.state_root / "runs"
|
|
work = runs / f"{original_id}.work"
|
|
saved_input = json.loads((work / "input.json").read_text())
|
|
upload = Path(saved_input["items"])
|
|
assert saved_input["secret"] == SECRET
|
|
assert SECRET not in app.rows()[0]["input_json"]
|
|
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
|
old_pids = app.pids()
|
|
if shutdown_signal == signal.SIGTERM:
|
|
app.stop()
|
|
assert app.rows()[0]["status"] == "interrupted"
|
|
assert app.rows()[0]["exit_code"] == 75
|
|
else:
|
|
# SIGKILL the entire disposable instance, as container teardown does.
|
|
app.kill_remaining()
|
|
assert app.process.returncode == -signal.SIGKILL
|
|
assert app.rows()[0]["status"] == "running"
|
|
wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit")
|
|
saved = read_json(work / "checkpoint.json")
|
|
clean = shutdown_signal == signal.SIGTERM
|
|
assert saved["clean_shutdown"] is clean
|
|
assert saved["data"] == {"next": 3 if clean else 0}
|
|
assert upload.read_bytes() == UPLOAD
|
|
assert json.loads((work / "input.json").read_text()) == saved_input
|
|
|
|
stale = time.time() - 45 * 86400
|
|
for path in (work, work / "input.json", upload.parent, upload):
|
|
os.utime(path, (stale, stale))
|
|
app.start()
|
|
recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id)
|
|
app.wait_progress(recovery["id"], "Recovery waiting")
|
|
observation = read_json(app.state_root / "tasks/ledger/observation.json")
|
|
assert observation == {
|
|
"checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean,
|
|
"ledger": ["0", "1", "2"],
|
|
}
|
|
status, states = app.request("GET", "/tasks/state")
|
|
assert status == 200
|
|
assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering"
|
|
assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"]
|
|
status, conflict = app.request("POST", "/tasks/contender/task.py/run")
|
|
assert status == 409, conflict
|
|
assert conflict["blocked_by"]["run_id"] == recovery["id"]
|
|
assert not any(row["target_path"] == "contender/task.py" for row in app.rows())
|
|
|
|
# A real completion invokes cleanup while recovery still needs its ancestor's files.
|
|
cleanup_probe = runs / "999999.input.json"
|
|
cleanup_probe.write_text("{}\n")
|
|
maintenance_id = app.run("maintenance/task.py")
|
|
app.wait_status(maintenance_id, "succeeded")
|
|
wait_until(lambda: not cleanup_probe.exists(), "completion cleanup")
|
|
assert upload.read_bytes() == UPLOAD
|
|
assert json.loads((work / "input.json").read_text()) == saved_input
|
|
assert app.rows()[0]["status"] == "interrupted"
|
|
|
|
(app.state_root / "tasks/ledger/finish").touch()
|
|
app.wait_status(recovery["id"], "succeeded")
|
|
assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)]
|
|
wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup")
|
|
app.stop()
|
|
assert not app.pids(), "A task escaped registration/shutdown"
|
|
|
|
|
|
def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app):
|
|
app = production_app
|
|
app.write_task("blocking", "task.py", BLOCKING_TASK)
|
|
app.write_task("watcher", "internal/task.py", BLOCKING_TASK)
|
|
(app.task_root / "watcher/background-tasks.json").write_text(json.dumps({
|
|
"version": 1,
|
|
"tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}],
|
|
}))
|
|
app.start()
|
|
run_id = app.run("blocking/task.py")
|
|
app.wait_progress(run_id, "Blocking")
|
|
background_work = app.state_root / "background/watcher/watcher.work"
|
|
wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking",
|
|
"background worker to start")
|
|
work = app.state_root / "runs" / f"{run_id}.work"
|
|
task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json")
|
|
assert all(process_running(pid) for pid in task_pids)
|
|
elapsed = app.stop()
|
|
assert elapsed >= app.grace * .8, "The hard deadline was not exercised"
|
|
wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit")
|
|
assert not app.pids(), "A foreground/background task escaped shutdown registration"
|
|
assert read_json(work / "checkpoint.json")["clean_shutdown"] is False
|
|
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
|
assert (work / "input.json").exists()
|
|
|
|
|
|
def test_production_graceful_pause_does_not_leave_live_descendants(production_app):
|
|
app = production_app
|
|
source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n")
|
|
app.write_task("cooperative", "task.py", source)
|
|
app.start()
|
|
run_id = app.run("cooperative/task.py")
|
|
app.wait_progress(run_id, "Blocking")
|
|
work = app.state_root / "runs" / f"{run_id}.work"
|
|
worker_pid, child_pid = read_json(work / "pids.json")
|
|
app.stop()
|
|
assert read_json(work / "checkpoint.json")["clean_shutdown"] is True
|
|
assert not process_running(worker_pid)
|
|
wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5)
|
|
|
|
|
|
def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app):
|
|
app = production_app
|
|
source = '''
|
|
from app import task_runtime as runtime
|
|
runtime.progress("Started", force=True)
|
|
while True:
|
|
runtime.sleep(.02)
|
|
'''
|
|
groups = [f"queue_{index}" for index in range(12)]
|
|
for group in groups:
|
|
app.write_task(group, "task.py", source)
|
|
app.start()
|
|
first_id = app.run(f"{groups[0]}/task.py")
|
|
app.wait_progress(first_id, "Started")
|
|
|
|
def submit(group):
|
|
try:
|
|
return app.request("POST", f"/tasks/{group}/task.py/run")[0]
|
|
except (ConnectionError, TimeoutError, http.client.RemoteDisconnected):
|
|
# The listener may close before this in-flight request is accepted.
|
|
return None
|
|
|
|
with ThreadPoolExecutor(max_workers=6) as requests:
|
|
pending = [requests.submit(submit, group) for group in groups[1:]]
|
|
app.stop()
|
|
statuses = [request.result(timeout=3) for request in pending]
|
|
assert all(status in {202, 503, None} for status in statuses), statuses
|
|
assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration"
|
|
assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text()
|
|
|
|
|
|
def test_production_bind_failure_stops_workers_started_by_recovery(production_app):
|
|
app = production_app
|
|
source = '''
|
|
from app import task_runtime as runtime
|
|
|
|
def main():
|
|
runtime.progress("Started", force=True)
|
|
while True:
|
|
runtime.sleep(.02)
|
|
|
|
def recover():
|
|
main()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
'''
|
|
app.write_task("recoverable", "task.py", source)
|
|
app.start()
|
|
run_id = app.run("recoverable/task.py")
|
|
app.wait_progress(run_id, "Started")
|
|
app.stop()
|
|
assert app.rows()[0]["status"] == "interrupted"
|
|
|
|
# Delay startup cleanup without patching app code, until recovery is running.
|
|
cleanup_gate = app.state_root / "tasks/.orphaned-groups.json"
|
|
os.mkfifo(cleanup_gate)
|
|
with socket.socket() as occupied:
|
|
occupied.bind(("127.0.0.1", 0))
|
|
occupied.listen()
|
|
app.launch(occupied.getsockname()[1])
|
|
recovery = wait_until(
|
|
lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None),
|
|
"startup recovery registration",
|
|
)
|
|
app.wait_progress(recovery["id"], "Started")
|
|
|
|
def release_cleanup():
|
|
try:
|
|
descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK)
|
|
except OSError as error:
|
|
if error.errno == errno.ENXIO:
|
|
return False
|
|
raise
|
|
with os.fdopen(descriptor, "w") as gate:
|
|
json.dump({}, gate)
|
|
return True
|
|
|
|
wait_until(release_cleanup, "startup cleanup gate", timeout=2)
|
|
assert app.process.wait(timeout=4) != 0
|
|
assert "Address already in use" in app.log_path.read_text()
|
|
assert any(row["recovery_of_run_id"] == run_id for row in app.rows())
|
|
wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)
|