Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -0,0 +1,478 @@
|
||||
"""Recovery through the real Waitress entry point, without manager test doubles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import closing
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.test import EnvironBuilder
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process/signal integration")
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
UPLOAD = b"0\n1\n2\n3\n4\n5\n6\n7\n"
|
||||
SECRET = "disposable-process-test-secret"
|
||||
|
||||
|
||||
LEDGER_TASK = '''"""Reconcile an uploaded list against a durable local ledger."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
RESOURCES = ("shared-ledger",)
|
||||
state = Path(os.environ["SERVER_MAINTENANCE_STATE"])
|
||||
|
||||
def values():
|
||||
data = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||
assert data["secret"] == "disposable-process-test-secret"
|
||||
assert Path(data["items"]).read_text() == "0\\n1\\n2\\n3\\n4\\n5\\n6\\n7\\n"
|
||||
return data
|
||||
|
||||
def main():
|
||||
data = values()
|
||||
ledger = state / "ledger.txt"
|
||||
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
||||
if not runtime.checkpoint_data():
|
||||
runtime.checkpoint({"next": 0}, force=True)
|
||||
items = Path(data["items"]).read_text().splitlines()
|
||||
for number in range(len(existing), len(items)):
|
||||
runtime.check_stop()
|
||||
with ledger.open("a") as stream:
|
||||
stream.write(items[number] + "\\n")
|
||||
runtime.checkpoint({"next": number + 1})
|
||||
if number == 2:
|
||||
runtime.progress("Original waiting", force=True)
|
||||
while not (state / "finish").exists():
|
||||
runtime.sleep(.02)
|
||||
|
||||
def recover():
|
||||
values()
|
||||
(state / "observation.json").write_text(json.dumps({
|
||||
"checkpoint": runtime.checkpoint_data(),
|
||||
"clean_shutdown": runtime.was_clean_shutdown(),
|
||||
"ledger": (state / "ledger.txt").read_text().splitlines(),
|
||||
}))
|
||||
runtime.progress("Recovery waiting", force=True)
|
||||
while not (state / "finish").exists():
|
||||
runtime.sleep(.02)
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
|
||||
|
||||
BLOCKING_TASK = '''"""Ignore cooperative stop checks to exercise the hard deadline."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from app import task_runtime as runtime
|
||||
|
||||
runtime.checkpoint({"next": 0}, force=True)
|
||||
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
|
||||
runtime.checkpoint({"next": 1})
|
||||
Path(os.environ["SERVER_MAINTENANCE_WORK"], "pids.json").write_text(
|
||||
json.dumps([os.getpid(), child.pid]))
|
||||
runtime.progress("Blocking", force=True)
|
||||
time.sleep(60)
|
||||
'''
|
||||
|
||||
|
||||
def wait_until(condition, description, timeout=8):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = condition()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(.02)
|
||||
pytest.fail(f"Timed out waiting for {description}")
|
||||
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def process_running(pid):
|
||||
try:
|
||||
status = Path(f"/proc/{pid}/stat").read_text().rsplit(") ", 1)[1]
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return status[0] not in {"Z", "X"}
|
||||
|
||||
|
||||
class ProductionApp:
|
||||
def __init__(self, root):
|
||||
self.task_root = root / "tasks"
|
||||
self.state_root = root / "state"
|
||||
self.credentials_root = root / "credentials"
|
||||
for path in (self.task_root, self.state_root, self.credentials_root):
|
||||
path.mkdir()
|
||||
self.database = self.state_root / "server-maintenance.sqlite3"
|
||||
self.log_path = root / "server.log"
|
||||
self.process = None
|
||||
self.port = None
|
||||
self.grace = 1
|
||||
|
||||
def write_task(self, group, filename, source):
|
||||
path = self.task_root / group / filename
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(source, encoding="utf-8")
|
||||
return path
|
||||
|
||||
def start(self):
|
||||
with socket.socket() as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
port = listener.getsockname()[1]
|
||||
self.launch(port)
|
||||
|
||||
def ready():
|
||||
assert self.process.poll() is None, self.log_path.read_text()
|
||||
try:
|
||||
status, _ = self.request("GET", "/tasks/state")
|
||||
except (ConnectionError, TimeoutError):
|
||||
return False
|
||||
return status == 200
|
||||
|
||||
wait_until(ready, f"Waitress startup; log: {self.log_path}")
|
||||
|
||||
def launch(self, port):
|
||||
assert self.process is None or self.process.poll() is not None
|
||||
self.port = port
|
||||
environment = {key: value for key, value in os.environ.items()
|
||||
if not key.startswith("SERVER_MAINTENANCE_")}
|
||||
environment.update({
|
||||
"PYTHONPATH": str(PROJECT_ROOT),
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"SERVER_MAINTENANCE_HOST": "127.0.0.1",
|
||||
"SERVER_MAINTENANCE_PORT": str(self.port),
|
||||
"SERVER_MAINTENANCE_TASK_ROOT": str(self.task_root),
|
||||
"SERVER_MAINTENANCE_STATE_ROOT": str(self.state_root),
|
||||
"SERVER_MAINTENANCE_CREDENTIALS_ROOT": str(self.credentials_root),
|
||||
"SERVER_MAINTENANCE_PYTHON": sys.executable,
|
||||
"SERVER_MAINTENANCE_SHUTDOWN_GRACE_SECONDS": str(self.grace),
|
||||
})
|
||||
with self.log_path.open("ab") as log:
|
||||
self.process = subprocess.Popen(
|
||||
[sys.executable, "-m", "app"], cwd=PROJECT_ROOT,
|
||||
env=environment, stdin=subprocess.DEVNULL, stdout=log,
|
||||
stderr=subprocess.STDOUT, start_new_session=True,
|
||||
)
|
||||
|
||||
def request(self, method, path, data=None):
|
||||
headers = {"Accept": "application/json"}
|
||||
body = None
|
||||
if data is not None:
|
||||
with closing(EnvironBuilder(method=method, data=data)) as builder:
|
||||
environ = builder.get_environ()
|
||||
body = environ["wsgi.input"].read()
|
||||
headers["Content-Type"] = environ["CONTENT_TYPE"]
|
||||
connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=2)
|
||||
try:
|
||||
connection.request(method, path, body=body, headers=headers)
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
assert response.getheader("Content-Type", "").startswith("application/json"), payload
|
||||
return response.status, json.loads(payload)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
def rows(self):
|
||||
with closing(sqlite3.connect(self.database)) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
return [dict(row) for row in connection.execute("SELECT * FROM runs ORDER BY id")]
|
||||
|
||||
def run(self, task, data=None):
|
||||
status, response = self.request("POST", f"/tasks/{task}/run", data)
|
||||
assert status == 202, response
|
||||
return response["run_id"]
|
||||
|
||||
def wait_status(self, run_id, status):
|
||||
def matching():
|
||||
run = next(row for row in self.rows() if row["id"] == run_id)
|
||||
if run["status"] in {"succeeded", "failed", "cancelled", "interrupted"}:
|
||||
assert run["status"] == status, (run, self.log_path.read_text())
|
||||
return run if run["status"] == status else None
|
||||
return wait_until(matching, f"run {run_id} to become {status}")
|
||||
|
||||
def wait_progress(self, run_id, text):
|
||||
path = self.state_root / "runs" / f"{run_id}.progress.json"
|
||||
wait_until(lambda: (read_json(path) or {}).get("text") == text, text)
|
||||
|
||||
def pids(self):
|
||||
# Match only this disposable instance, including separately-sessioned workers.
|
||||
marker = f"SERVER_MAINTENANCE_STATE_ROOT={self.state_root}".encode()
|
||||
found = []
|
||||
for path in Path("/proc").iterdir():
|
||||
if not path.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
environment = (path / "environ").read_bytes().split(b"\0")
|
||||
except (FileNotFoundError, ProcessLookupError, PermissionError):
|
||||
continue
|
||||
if marker in environment:
|
||||
found.append(int(path.name))
|
||||
return found
|
||||
|
||||
def kill_remaining(self):
|
||||
def kill_snapshot():
|
||||
pids = self.pids()
|
||||
for pid in pids:
|
||||
try:
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
return not pids
|
||||
|
||||
kill_snapshot()
|
||||
if self.process is not None:
|
||||
self.process.wait(timeout=3)
|
||||
wait_until(kill_snapshot, "disposable instance teardown", timeout=3)
|
||||
|
||||
def stop(self):
|
||||
started = time.monotonic()
|
||||
self.process.send_signal(signal.SIGTERM)
|
||||
try:
|
||||
code = self.process.wait(timeout=self.grace + 3)
|
||||
except subprocess.TimeoutExpired:
|
||||
pytest.fail(f"Production shutdown exceeded deadline:\n{self.log_path.read_text()}")
|
||||
elapsed = time.monotonic() - started
|
||||
assert code == 0, self.log_path.read_text()
|
||||
assert elapsed < self.grace + 2, (elapsed, self.log_path.read_text())
|
||||
return elapsed
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def production_app(tmp_path):
|
||||
app = ProductionApp(tmp_path)
|
||||
try:
|
||||
yield app
|
||||
finally:
|
||||
app.kill_remaining()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shutdown_signal", [signal.SIGTERM, signal.SIGKILL], ids=["sigterm", "sigkill"])
|
||||
def test_production_restart_preserves_recovery_inputs_and_resource_locks(production_app, shutdown_signal):
|
||||
app = production_app
|
||||
ledger_task = app.write_task("ledger", "task.py", LEDGER_TASK)
|
||||
(ledger_task.parent / "task-inputs.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"tasks": {"task.py": {"inputs": [
|
||||
{"name": "items", "label": "Items", "type": "file", "required": True,
|
||||
"accept": [".txt"], "maximum_bytes": 1024},
|
||||
{"name": "secret", "label": "Secret", "type": "text", "required": True,
|
||||
"sensitive": True},
|
||||
]}},
|
||||
}))
|
||||
app.write_task("contender", "task.py", 'RESOURCES = ("shared-ledger",)\nprint("done")\n')
|
||||
app.write_task("maintenance", "task.py", 'print("done")\n')
|
||||
app.start()
|
||||
original_id = app.run("ledger/task.py", {"secret": SECRET, "items": (BytesIO(UPLOAD), "items.txt")})
|
||||
app.wait_progress(original_id, "Original waiting")
|
||||
runs = app.state_root / "runs"
|
||||
work = runs / f"{original_id}.work"
|
||||
saved_input = json.loads((work / "input.json").read_text())
|
||||
upload = Path(saved_input["items"])
|
||||
assert saved_input["secret"] == SECRET
|
||||
assert SECRET not in app.rows()[0]["input_json"]
|
||||
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
||||
old_pids = app.pids()
|
||||
if shutdown_signal == signal.SIGTERM:
|
||||
app.stop()
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
assert app.rows()[0]["exit_code"] == 75
|
||||
else:
|
||||
# SIGKILL the entire disposable instance, as container teardown does.
|
||||
app.kill_remaining()
|
||||
assert app.process.returncode == -signal.SIGKILL
|
||||
assert app.rows()[0]["status"] == "running"
|
||||
wait_until(lambda: not any(process_running(pid) for pid in old_pids), "old instance to exit")
|
||||
saved = read_json(work / "checkpoint.json")
|
||||
clean = shutdown_signal == signal.SIGTERM
|
||||
assert saved["clean_shutdown"] is clean
|
||||
assert saved["data"] == {"next": 3 if clean else 0}
|
||||
assert upload.read_bytes() == UPLOAD
|
||||
assert json.loads((work / "input.json").read_text()) == saved_input
|
||||
|
||||
stale = time.time() - 45 * 86400
|
||||
for path in (work, work / "input.json", upload.parent, upload):
|
||||
os.utime(path, (stale, stale))
|
||||
app.start()
|
||||
recovery = next(row for row in app.rows() if row["recovery_of_run_id"] == original_id)
|
||||
app.wait_progress(recovery["id"], "Recovery waiting")
|
||||
observation = read_json(app.state_root / "tasks/ledger/observation.json")
|
||||
assert observation == {
|
||||
"checkpoint": {"next": 3 if clean else 0}, "clean_shutdown": clean,
|
||||
"ledger": ["0", "1", "2"],
|
||||
}
|
||||
status, states = app.request("GET", "/tasks/state")
|
||||
assert status == 200
|
||||
assert states["tasks"]["ledger/task.py"]["active_run"]["display_status"] == "recovering"
|
||||
assert states["tasks"]["contender/task.py"]["blocked_by"]["run_id"] == recovery["id"]
|
||||
status, conflict = app.request("POST", "/tasks/contender/task.py/run")
|
||||
assert status == 409, conflict
|
||||
assert conflict["blocked_by"]["run_id"] == recovery["id"]
|
||||
assert not any(row["target_path"] == "contender/task.py" for row in app.rows())
|
||||
|
||||
# A real completion invokes cleanup while recovery still needs its ancestor's files.
|
||||
cleanup_probe = runs / "999999.input.json"
|
||||
cleanup_probe.write_text("{}\n")
|
||||
maintenance_id = app.run("maintenance/task.py")
|
||||
app.wait_status(maintenance_id, "succeeded")
|
||||
wait_until(lambda: not cleanup_probe.exists(), "completion cleanup")
|
||||
assert upload.read_bytes() == UPLOAD
|
||||
assert json.loads((work / "input.json").read_text()) == saved_input
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
|
||||
(app.state_root / "tasks/ledger/finish").touch()
|
||||
app.wait_status(recovery["id"], "succeeded")
|
||||
assert (app.state_root / "tasks/ledger/ledger.txt").read_text().splitlines() == [str(i) for i in range(8)]
|
||||
wait_until(lambda: not (work / "input.json").exists(), "completed recovery input cleanup")
|
||||
app.stop()
|
||||
assert not app.pids(), "A task escaped registration/shutdown"
|
||||
|
||||
|
||||
def test_production_shutdown_deadline_kills_foreground_background_and_descendants(production_app):
|
||||
app = production_app
|
||||
app.write_task("blocking", "task.py", BLOCKING_TASK)
|
||||
app.write_task("watcher", "internal/task.py", BLOCKING_TASK)
|
||||
(app.task_root / "watcher/background-tasks.json").write_text(json.dumps({
|
||||
"version": 1,
|
||||
"tasks": [{"id": "watcher", "script": "internal/task.py", "interval_seconds": 3600}],
|
||||
}))
|
||||
app.start()
|
||||
run_id = app.run("blocking/task.py")
|
||||
app.wait_progress(run_id, "Blocking")
|
||||
background_work = app.state_root / "background/watcher/watcher.work"
|
||||
wait_until(lambda: (read_json(background_work / "progress.json") or {}).get("text") == "Blocking",
|
||||
"background worker to start")
|
||||
work = app.state_root / "runs" / f"{run_id}.work"
|
||||
task_pids = read_json(work / "pids.json") + read_json(background_work / "pids.json")
|
||||
assert all(process_running(pid) for pid in task_pids)
|
||||
elapsed = app.stop()
|
||||
assert elapsed >= app.grace * .8, "The hard deadline was not exercised"
|
||||
wait_until(lambda: not any(process_running(pid) for pid in task_pids), "worker process groups to exit")
|
||||
assert not app.pids(), "A foreground/background task escaped shutdown registration"
|
||||
assert read_json(work / "checkpoint.json")["clean_shutdown"] is False
|
||||
assert read_json(work / "checkpoint.json")["data"] == {"next": 0}
|
||||
assert (work / "input.json").exists()
|
||||
|
||||
|
||||
def test_production_graceful_pause_does_not_leave_live_descendants(production_app):
|
||||
app = production_app
|
||||
source = BLOCKING_TASK.replace("time.sleep(60)\n", "while True:\n runtime.sleep(.02)\n")
|
||||
app.write_task("cooperative", "task.py", source)
|
||||
app.start()
|
||||
run_id = app.run("cooperative/task.py")
|
||||
app.wait_progress(run_id, "Blocking")
|
||||
work = app.state_root / "runs" / f"{run_id}.work"
|
||||
worker_pid, child_pid = read_json(work / "pids.json")
|
||||
app.stop()
|
||||
assert read_json(work / "checkpoint.json")["clean_shutdown"] is True
|
||||
assert not process_running(worker_pid)
|
||||
wait_until(lambda: not process_running(child_pid), "paused worker's subprocess to exit", timeout=.5)
|
||||
|
||||
|
||||
def test_production_sigterm_during_concurrent_submissions_leaves_no_workers(production_app):
|
||||
app = production_app
|
||||
source = '''
|
||||
from app import task_runtime as runtime
|
||||
runtime.progress("Started", force=True)
|
||||
while True:
|
||||
runtime.sleep(.02)
|
||||
'''
|
||||
groups = [f"queue_{index}" for index in range(12)]
|
||||
for group in groups:
|
||||
app.write_task(group, "task.py", source)
|
||||
app.start()
|
||||
first_id = app.run(f"{groups[0]}/task.py")
|
||||
app.wait_progress(first_id, "Started")
|
||||
|
||||
def submit(group):
|
||||
try:
|
||||
return app.request("POST", f"/tasks/{group}/task.py/run")[0]
|
||||
except (ConnectionError, TimeoutError, http.client.RemoteDisconnected):
|
||||
# The listener may close before this in-flight request is accepted.
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as requests:
|
||||
pending = [requests.submit(submit, group) for group in groups[1:]]
|
||||
app.stop()
|
||||
statuses = [request.result(timeout=3) for request in pending]
|
||||
assert all(status in {202, 503, None} for status in statuses), statuses
|
||||
assert not app.pids(), "A worker launched concurrently with SIGTERM escaped registration"
|
||||
assert "Traceback" not in app.log_path.read_text(), app.log_path.read_text()
|
||||
|
||||
|
||||
def test_production_bind_failure_stops_workers_started_by_recovery(production_app):
|
||||
app = production_app
|
||||
source = '''
|
||||
from app import task_runtime as runtime
|
||||
|
||||
def main():
|
||||
runtime.progress("Started", force=True)
|
||||
while True:
|
||||
runtime.sleep(.02)
|
||||
|
||||
def recover():
|
||||
main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
'''
|
||||
app.write_task("recoverable", "task.py", source)
|
||||
app.start()
|
||||
run_id = app.run("recoverable/task.py")
|
||||
app.wait_progress(run_id, "Started")
|
||||
app.stop()
|
||||
assert app.rows()[0]["status"] == "interrupted"
|
||||
|
||||
# Delay startup cleanup without patching app code, until recovery is running.
|
||||
cleanup_gate = app.state_root / "tasks/.orphaned-groups.json"
|
||||
os.mkfifo(cleanup_gate)
|
||||
with socket.socket() as occupied:
|
||||
occupied.bind(("127.0.0.1", 0))
|
||||
occupied.listen()
|
||||
app.launch(occupied.getsockname()[1])
|
||||
recovery = wait_until(
|
||||
lambda: next((row for row in app.rows() if row["recovery_of_run_id"] == run_id), None),
|
||||
"startup recovery registration",
|
||||
)
|
||||
app.wait_progress(recovery["id"], "Started")
|
||||
|
||||
def release_cleanup():
|
||||
try:
|
||||
descriptor = os.open(cleanup_gate, os.O_WRONLY | os.O_NONBLOCK)
|
||||
except OSError as error:
|
||||
if error.errno == errno.ENXIO:
|
||||
return False
|
||||
raise
|
||||
with os.fdopen(descriptor, "w") as gate:
|
||||
json.dump({}, gate)
|
||||
return True
|
||||
|
||||
wait_until(release_cleanup, "startup cleanup gate", timeout=2)
|
||||
assert app.process.wait(timeout=4) != 0
|
||||
assert "Address already in use" in app.log_path.read_text()
|
||||
assert any(row["recovery_of_run_id"] == run_id for row in app.rows())
|
||||
wait_until(lambda: not app.pids(), "recovery workers to stop after Waitress bind failure", timeout=.5)
|
||||
Reference in new issue
Block a user