from __future__ import annotations import json import os import signal import subprocess import sys import time from dataclasses import replace from pathlib import Path import pytest from app.discovery import TaskCatalog from app.runner import RunManager, TaskConflictError from app.store import RunStore from .conftest import write_task from .test_runner import wait_for, wait_for_output COUNTER = '''"""Reconcile a small external ledger before continuing.""" import json, os from pathlib import Path from app import task_runtime as runtime def main(): data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text()) ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt' existing = ledger.read_text().splitlines() if ledger.exists() else [] if not runtime.checkpoint_data(): runtime.checkpoint({'next': 0}, force=True) for number in range(len(existing), data.get('count', 50)): runtime.check_stop() with ledger.open('a') as stream: stream.write(str(number) + '\\n') runtime.checkpoint({'next': number + 1}) runtime.progress(f'Items: {number + 1}', force=True) runtime.sleep(.03) def recover(): print('Checking external ledger', flush=True) main() if __name__ == '__main__': main() ''' def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager): write_task(manager.settings.task_root, 'count', 'task.py', COUNTER) task = manager.catalog.task('count/task.py') run, _ = manager.run_task(task, {'count': 50}) wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3') manager.shutdown(timeout=1) assert manager.store.get(run.id).status == 'interrupted' work = manager._work_root(run) saved = json.loads((work / 'checkpoint.json').read_text()) assert saved['clean_shutdown'] is True assert (work / 'input.json').exists() manager.cleanup_transient_files(artifact_max_age_seconds=0) assert (work / 'input.json').exists() resumed = RunManager(manager.settings, manager.catalog, manager.store) resumed.startup() recovery = manager.store.recovery_for(run.id) assert recovery is not None assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded' assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)] assert manager.store.get(run.id).status == 'interrupted' assert not (work / 'input.json').exists() assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text() resumed.shutdown(timeout=1) def test_forced_container_style_exit_recovers_from_older_checkpoint(settings): settings.task_root.mkdir(parents=True) write_task(settings.task_root, 'count', 'task.py', COUNTER) control = settings.state_root / 'control.json' program = ''' import json,time from pathlib import Path from app.config import Settings from app.discovery import TaskCatalog from app.runner import RunManager from app.store import RunStore s=Settings.from_environment() c=TaskCatalog(s.task_root) db=RunStore(s.database_path) db.initialize() m=RunManager(s,c,db) r,_=m.run_task(c.task('count/task.py'),{'count':50}) while r.id not in m._processes: time.sleep(.01) Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid})) while True: time.sleep(1) ''' environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]), 'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root), 'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root), 'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)} process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True) child_pid = None resumed = None try: wait_for_output(control, 'pid') data = json.loads(control.read_text()) child_pid = data['pid'] wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n') process.kill() process.wait(timeout=3) os.killpg(child_pid, signal.SIGKILL) child_pid = None saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text()) assert saved['clean_shutdown'] is False assert saved['data']['next'] == 0 store = RunStore(settings.database_path) resumed = RunManager(settings, TaskCatalog(settings.task_root), store) resumed.startup() recovery = store.recovery_for(data['run']) assert recovery is not None assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded' assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)] finally: if process.poll() is None: process.kill() process.wait(timeout=3) if child_pid is not None: try: os.killpg(child_pid, signal.SIGKILL) except ProcessLookupError: pass if resumed: resumed.shutdown(timeout=1) def test_cancelled_tasks_never_auto_recover(manager): write_task(manager.settings.task_root, 'count', 'task.py', COUNTER) task = manager.catalog.task('count/task.py') run, _ = manager.run_task(task, {'count': 100}) wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n') manager.request_cancel(run.id) wait_for(manager, run.id, {'cancelled'}) manager.shutdown(timeout=1) new = RunManager(manager.settings, manager.catalog, manager.store) new.startup() assert manager.store.recovery_for(run.id) is None assert not (manager._work_root(run) / 'input.json').exists() new.shutdown(timeout=1) def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager): path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():')) run, _ = manager.run_task(manager.catalog.task('count/task.py')) wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n') manager.shutdown(timeout=1) assert manager.store.get(run.id).status == 'interrupted' path.write_text(COUNTER) legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy') write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER) # The current task has opted out again; neither it nor a pre-runtime run may resume. path.write_text(COUNTER.replace('def recover():', 'def unused():')) new = RunManager(manager.settings, manager.catalog, manager.store) new.startup() assert manager.store.recovery_for(run.id) is None assert manager.store.recovery_for(legacy.id) is None assert manager.store.active_runs() == [] new.shutdown(timeout=1) def test_shared_resources_block_cross_group_submissions_and_recoveries(manager): for group in ('first','second'): write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER) first = manager.catalog.task('first/task.py') second = manager.catalog.task('second/task.py') run, _ = manager.run_task(first, {'count':50}) with pytest.raises(TaskConflictError) as error: manager.run_task(second) assert error.value.run.id == run.id manager.shutdown(timeout=1) new = RunManager(manager.settings, manager.catalog, manager.store) new.startup() recovered = manager.store.recovery_for(run.id) assert recovered is not None with pytest.raises(TaskConflictError) as error: new.run_task(second) assert error.value.run.id == recovered.id new.shutdown(timeout=1) def test_shutdown_rejects_new_work(manager): write_task(manager.settings.task_root,'tasks','task.py','print("hello")') manager.shutdown(timeout=.1) with pytest.raises(RuntimeError,match='shutting down'): manager.run_task(manager.catalog.task('tasks/task.py'))