Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
196 lines
8.0 KiB
Python
196 lines
8.0 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.discovery import TaskCatalog
|
|
from app.runner import RunManager, TaskConflictError
|
|
from app.store import RunStore
|
|
|
|
from .conftest import write_task
|
|
from .test_runner import wait_for, wait_for_output
|
|
|
|
|
|
COUNTER = '''"""Reconcile a small external ledger before continuing."""
|
|
import json, os
|
|
from pathlib import Path
|
|
from app import task_runtime as runtime
|
|
|
|
def main():
|
|
data = json.loads(Path(os.environ['SERVER_MAINTENANCE_INPUT']).read_text())
|
|
ledger = Path(os.environ['SERVER_MAINTENANCE_STATE']) / 'ledger.txt'
|
|
existing = ledger.read_text().splitlines() if ledger.exists() else []
|
|
if not runtime.checkpoint_data():
|
|
runtime.checkpoint({'next': 0}, force=True)
|
|
for number in range(len(existing), data.get('count', 50)):
|
|
runtime.check_stop()
|
|
with ledger.open('a') as stream:
|
|
stream.write(str(number) + '\\n')
|
|
runtime.checkpoint({'next': number + 1})
|
|
runtime.progress(f'Items: {number + 1}', force=True)
|
|
runtime.sleep(.03)
|
|
|
|
def recover():
|
|
print('Checking external ledger', flush=True)
|
|
main()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
'''
|
|
|
|
|
|
def test_graceful_pause_creates_linked_recovery_and_reconciles_live_state(manager):
|
|
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
|
|
task = manager.catalog.task('count/task.py')
|
|
run, _ = manager.run_task(task, {'count': 50})
|
|
wait_for_output(manager.settings.state_root / 'runs' / f'{run.id}.progress.json', 'Items: 3')
|
|
manager.shutdown(timeout=1)
|
|
assert manager.store.get(run.id).status == 'interrupted'
|
|
work = manager._work_root(run)
|
|
saved = json.loads((work / 'checkpoint.json').read_text())
|
|
assert saved['clean_shutdown'] is True
|
|
assert (work / 'input.json').exists()
|
|
manager.cleanup_transient_files(artifact_max_age_seconds=0)
|
|
assert (work / 'input.json').exists()
|
|
resumed = RunManager(manager.settings, manager.catalog, manager.store)
|
|
resumed.startup()
|
|
recovery = manager.store.recovery_for(run.id)
|
|
assert recovery is not None
|
|
assert wait_for(resumed, recovery.id, {'succeeded'}) == 'succeeded'
|
|
assert (manager.settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
|
|
assert manager.store.get(run.id).status == 'interrupted'
|
|
assert not (work / 'input.json').exists()
|
|
assert 'Checking external ledger' in Path(manager.store.get(recovery.id).log_path).read_text()
|
|
resumed.shutdown(timeout=1)
|
|
|
|
|
|
def test_forced_container_style_exit_recovers_from_older_checkpoint(settings):
|
|
settings.task_root.mkdir(parents=True)
|
|
write_task(settings.task_root, 'count', 'task.py', COUNTER)
|
|
control = settings.state_root / 'control.json'
|
|
program = '''
|
|
import json,time
|
|
from pathlib import Path
|
|
from app.config import Settings
|
|
from app.discovery import TaskCatalog
|
|
from app.runner import RunManager
|
|
from app.store import RunStore
|
|
s=Settings.from_environment()
|
|
c=TaskCatalog(s.task_root)
|
|
db=RunStore(s.database_path)
|
|
db.initialize()
|
|
m=RunManager(s,c,db)
|
|
r,_=m.run_task(c.task('count/task.py'),{'count':50})
|
|
while r.id not in m._processes:
|
|
time.sleep(.01)
|
|
Path(s.state_root/'control.json').write_text(json.dumps({'run':r.id,'pid':m._processes[r.id].pid}))
|
|
while True:
|
|
time.sleep(1)
|
|
'''
|
|
environment = {**os.environ, 'PYTHONPATH': str(Path(__file__).parents[1]),
|
|
'SERVER_MAINTENANCE_TASK_ROOT': str(settings.task_root),
|
|
'SERVER_MAINTENANCE_STATE_ROOT': str(settings.state_root),
|
|
'SERVER_MAINTENANCE_CREDENTIALS_ROOT': str(settings.credentials_root)}
|
|
process = subprocess.Popen([sys.executable, '-c', program], env=environment, start_new_session=True)
|
|
child_pid = None
|
|
resumed = None
|
|
try:
|
|
wait_for_output(control, 'pid')
|
|
data = json.loads(control.read_text())
|
|
child_pid = data['pid']
|
|
wait_for_output(settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
|
process.kill()
|
|
process.wait(timeout=3)
|
|
os.killpg(child_pid, signal.SIGKILL)
|
|
child_pid = None
|
|
saved = json.loads((settings.state_root / 'runs' / f"{data['run']}.work/checkpoint.json").read_text())
|
|
assert saved['clean_shutdown'] is False
|
|
assert saved['data']['next'] == 0
|
|
store = RunStore(settings.database_path)
|
|
resumed = RunManager(settings, TaskCatalog(settings.task_root), store)
|
|
resumed.startup()
|
|
recovery = store.recovery_for(data['run'])
|
|
assert recovery is not None
|
|
assert wait_for(resumed, recovery.id, {'succeeded'}, timeout=5) == 'succeeded'
|
|
assert (settings.state_root / 'tasks/count/ledger.txt').read_text().splitlines() == [str(i) for i in range(50)]
|
|
finally:
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait(timeout=3)
|
|
if child_pid is not None:
|
|
try:
|
|
os.killpg(child_pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
if resumed:
|
|
resumed.shutdown(timeout=1)
|
|
|
|
|
|
def test_cancelled_tasks_never_auto_recover(manager):
|
|
write_task(manager.settings.task_root, 'count', 'task.py', COUNTER)
|
|
task = manager.catalog.task('count/task.py')
|
|
run, _ = manager.run_task(task, {'count': 100})
|
|
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
|
manager.request_cancel(run.id)
|
|
wait_for(manager, run.id, {'cancelled'})
|
|
manager.shutdown(timeout=1)
|
|
new = RunManager(manager.settings, manager.catalog, manager.store)
|
|
new.startup()
|
|
assert manager.store.recovery_for(run.id) is None
|
|
assert not (manager._work_root(run) / 'input.json').exists()
|
|
new.shutdown(timeout=1)
|
|
|
|
|
|
def test_task_without_hook_is_not_restarted_and_historical_runs_are_not_replayed(manager):
|
|
path = write_task(manager.settings.task_root, 'count', 'task.py', COUNTER.replace('def recover():', 'def unused():'))
|
|
run, _ = manager.run_task(manager.catalog.task('count/task.py'))
|
|
wait_for_output(manager.settings.state_root / 'tasks/count/ledger.txt', '2\n')
|
|
manager.shutdown(timeout=1)
|
|
assert manager.store.get(run.id).status == 'interrupted'
|
|
path.write_text(COUNTER)
|
|
legacy, _ = manager.store.enqueue(target_kind='task',target_path='legacy/task.py',queue_group='legacy')
|
|
write_task(manager.settings.task_root, 'legacy', 'task.py', COUNTER)
|
|
# The current task has opted out again; neither it nor a pre-runtime run may resume.
|
|
path.write_text(COUNTER.replace('def recover():', 'def unused():'))
|
|
new = RunManager(manager.settings, manager.catalog, manager.store)
|
|
new.startup()
|
|
assert manager.store.recovery_for(run.id) is None
|
|
assert manager.store.recovery_for(legacy.id) is None
|
|
assert manager.store.active_runs() == []
|
|
new.shutdown(timeout=1)
|
|
|
|
|
|
def test_shared_resources_block_cross_group_submissions_and_recoveries(manager):
|
|
for group in ('first','second'):
|
|
write_task(manager.settings.task_root, group, 'task.py', "RESOURCES = ('shared-storage',)\n" + COUNTER)
|
|
first = manager.catalog.task('first/task.py')
|
|
second = manager.catalog.task('second/task.py')
|
|
run, _ = manager.run_task(first, {'count':50})
|
|
with pytest.raises(TaskConflictError) as error:
|
|
manager.run_task(second)
|
|
assert error.value.run.id == run.id
|
|
manager.shutdown(timeout=1)
|
|
new = RunManager(manager.settings, manager.catalog, manager.store)
|
|
new.startup()
|
|
recovered = manager.store.recovery_for(run.id)
|
|
assert recovered is not None
|
|
with pytest.raises(TaskConflictError) as error:
|
|
new.run_task(second)
|
|
assert error.value.run.id == recovered.id
|
|
new.shutdown(timeout=1)
|
|
|
|
|
|
def test_shutdown_rejects_new_work(manager):
|
|
write_task(manager.settings.task_root,'tasks','task.py','print("hello")')
|
|
manager.shutdown(timeout=.1)
|
|
with pytest.raises(RuntimeError,match='shutting down'):
|
|
manager.run_task(manager.catalog.task('tasks/task.py'))
|