Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -0,0 +1,81 @@
|
||||
import json
|
||||
import signal
|
||||
|
||||
import pytest
|
||||
|
||||
from app import task_runtime as runtime
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def work(tmp_path,monkeypatch):
|
||||
monkeypatch.setenv('SERVER_MAINTENANCE_WORK',str(tmp_path))
|
||||
old=signal.getsignal(signal.SIGUSR1)
|
||||
runtime.initialize()
|
||||
yield tmp_path
|
||||
signal.signal(signal.SIGUSR1,old)
|
||||
runtime._pause_requested=False
|
||||
runtime._defer=0
|
||||
|
||||
|
||||
def test_safe_point_saves_latest_memory_state_and_marks_clean(work):
|
||||
runtime.checkpoint({'next':1},force=True)
|
||||
runtime.checkpoint({'next':2})
|
||||
assert json.loads((work/'checkpoint.json').read_text())['data']=={'next':1}
|
||||
runtime.request_pause()
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
runtime.check_stop()
|
||||
saved=json.loads((work/'checkpoint.json').read_text())
|
||||
assert saved['data']=={'next':2}
|
||||
assert saved['clean_shutdown'] is True
|
||||
runtime.initialize()
|
||||
assert runtime.was_clean_shutdown() is True
|
||||
assert json.loads((work/'checkpoint.json').read_text())['clean_shutdown'] is False
|
||||
|
||||
|
||||
def test_small_critical_operation_finishes_before_pause(work):
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
with runtime.critical_section():
|
||||
runtime.request_pause()
|
||||
runtime.checkpoint({'committed':True},force=True)
|
||||
assert runtime.checkpoint_data()=={'committed':True}
|
||||
saved=json.loads((work/'checkpoint.json').read_text())
|
||||
assert saved['data']=={'committed':True}
|
||||
assert saved['clean_shutdown'] is True
|
||||
|
||||
|
||||
def test_sleep_is_interruptible(work):
|
||||
runtime.request_pause()
|
||||
with pytest.raises(runtime.TaskPaused):
|
||||
runtime.sleep(30)
|
||||
|
||||
|
||||
def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
|
||||
clock=[100.]
|
||||
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
|
||||
progress=runtime.Progress({'Removing files':10,'Linking files':20})
|
||||
progress.update('Removing files',0,force=True)
|
||||
clock[0]+=30
|
||||
progress.update('Removing files',5,force=True)
|
||||
text=json.loads((work/'progress.json').read_text())['text']
|
||||
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
|
||||
assert (work/'progress.json').stat().st_mode&0o777==0o600
|
||||
|
||||
|
||||
def test_runtime_does_not_write_or_pause_direct_scripts_without_work(monkeypatch):
|
||||
monkeypatch.delenv('SERVER_MAINTENANCE_WORK',raising=False)
|
||||
runtime.request_pause()
|
||||
runtime.checkpoint({'ignored':True})
|
||||
runtime.progress('hello',force=True)
|
||||
assert runtime.checkpoint_data()=={}
|
||||
runtime._pause_requested=False
|
||||
|
||||
|
||||
def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
|
||||
clock=[100.]
|
||||
monkeypatch.setattr(runtime.time,'monotonic',lambda:clock[0])
|
||||
progress=runtime.Progress({'Items':50})
|
||||
progress.update('Items',39,force=True)
|
||||
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
|
||||
clock[0]+=60
|
||||
progress.update('Items',40,force=True)
|
||||
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
|
||||
Reference in new issue
Block a user