Files
docker-images/python-tools/tests/test_recovery.py
T
ajp_anton 3bfd199b9c
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Add cooperative task recovery and persistent live status
2026-10-29 22:12:04 +00:00

196 lines
8.0 KiB
Python

from __future__ import annotations
import json
import os
import signal
import subprocess
import sys
import time
from dataclasses import replace
from pathlib import Path
import pytest
from app.discovery import TaskCatalog
from app.runner import RunManager, TaskConflictError
from app.store import RunStore
from .conftest import write_task
from .test_runner import wait_for, wait_for_output
COUNTER = '''"""Reconcile a small external ledger before continuing."""
import json, os
from pathlib import Path
from app import task_runtime as runtime
def main():
data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text())
ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt'
existing = ledger.read_text().splitlines() if ledger.exists() else []
if not runtime.checkpoint_data():
runtime.checkpoint({'next': 0}, force=True)
for number in range(len(existing), data.get('count', 50)):
runtime.check_stop()
with ledger.open('a') as stream:
stream.write(str(number) + '\\n')
runtime.checkpoint({'next': number + 1})
runtime.progress(f'Items: {number + 1}', force=True)
runtime.sleep(.03)
def recover():
print('Checking external ledger', flush=True)
main()
if __name__ == '__main__':
main()
'''
def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager):
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
task = manager.catalog.task('count/task.py')
run, _ = manager.run_task(task, {'count': 50})
wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3')
manager.shutdown(timeout=1)
assert manager.store.get(run.id).status == 'interrupted'
work = manager._work_root(run)
saved = json.loads((work / 'checkpoint.json').read_text())
assert saved['clean_shutdown'] is True
assert (work / 'input.json').exists()
manager.cleanup_transient_files(artifact_max_age_seconds=0)
assert (work / 'input.json').exists()
resumed = RunManager(manager.settings, manager.catalog, manager.store)
resumed.startup()
recovery = manager.store.recovery_for(run.id)
assert recovery is not None
assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded'
assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
assert manager.store.get(run.id).status == 'interrupted'
assert not (work / 'input.json').exists()
assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text()
resumed.shutdown(timeout=1)
def test_forced_container_style_exit_recovers_from_older_checkpoint(settings):
settings.task_root.mkdir(parents=True)
write_task(settings.task_root, 'count', 'task.py', COUNTER)
control = settings.state_root / 'control.json'
program = '''
import json,time
from pathlib import Path
from app.config import Settings
from app.discovery import TaskCatalog
from app.runner import RunManager
from app.store import RunStore
s=Settings.from_environment()
c=TaskCatalog(s.task_root)
db=RunStore(s.database_path)
db.initialize()
m=RunManager(s,c,db)
r,_=m.run_task(c.task('count/task.py'),{'count':50})
while r.id not in m._processes:
time.sleep(.01)
Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid}))
while True:
time.sleep(1)
'''
environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]),
'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root),
'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root),
'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)}
process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True)
child_pid = None
resumed = None
try:
wait_for_output(control, 'pid')
data = json.loads(control.read_text())
child_pid = data['pid']
wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n')
process.kill()
process.wait(timeout=3)
os.killpg(child_pid, signal.SIGKILL)
child_pid = None
saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text())
assert saved['clean_shutdown'] is False
assert saved['data']['next'] == 0
store = RunStore(settings.database_path)
resumed = RunManager(settings, TaskCatalog(settings.task_root), store)
resumed.startup()
recovery = store.recovery_for(data['run'])
assert recovery is not None
assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded'
assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=3)
if child_pid is not None:
try:
os.killpg(child_pid, signal.SIGKILL)
except ProcessLookupError:
pass
if resumed:
resumed.shutdown(timeout=1)
def test_cancelled_tasks_never_auto_recover(manager):
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
task = manager.catalog.task('count/task.py')
run, _ = manager.run_task(task, {'count': 100})
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
manager.request_cancel(run.id)
wait_for(manager, run.id, {'cancelled'})
manager.shutdown(timeout=1)
new = RunManager(manager.settings, manager.catalog, manager.store)
new.startup()
assert manager.store.recovery_for(run.id) is None
assert not (manager._work_root(run) / 'input.json').exists()
new.shutdown(timeout=1)
def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager):
path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():'))
run, _ = manager.run_task(manager.catalog.task('count/task.py'))
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
manager.shutdown(timeout=1)
assert manager.store.get(run.id).status == 'interrupted'
path.write_text(COUNTER)
legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy')
write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER)
# The current task has opted out again; neither it nor a pre-runtime run may resume.
path.write_text(COUNTER.replace('def recover():', 'def unused():'))
new = RunManager(manager.settings, manager.catalog, manager.store)
new.startup()
assert manager.store.recovery_for(run.id) is None
assert manager.store.recovery_for(legacy.id) is None
assert manager.store.active_runs() == []
new.shutdown(timeout=1)
def test_shared_resources_block_cross_group_submissions_and_recoveries(manager):
for group in ('first','second'):
write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER)
first = manager.catalog.task('first/task.py')
second = manager.catalog.task('second/task.py')
run, _ = manager.run_task(first, {'count':50})
with pytest.raises(TaskConflictError) as error:
manager.run_task(second)
assert error.value.run.id == run.id
manager.shutdown(timeout=1)
new = RunManager(manager.settings, manager.catalog, manager.store)
new.startup()
recovered = manager.store.recovery_for(run.id)
assert recovered is not None
with pytest.raises(TaskConflictError) as error:
new.run_task(second)
assert error.value.run.id == recovered.id
new.shutdown(timeout=1)
def test_shutdown_rejects_new_work(manager):
write_task(manager.settings.task_root,'tasks','task.py','print("hello")')
manager.shutdown(timeout=.1)
with pytest.raises(RuntimeError,match='shutting down'):
manager.run_task(manager.catalog.task('tasks/task.py'))