Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-10-29 22:12:04 +00:00
1 parent 034fecbd51
commit 3bfd199b9c
28 files changed
+2356 -172

No files matched your search

+161
View File
@@ -0,0 +1,161 @@
"""Cooperative task checkpoints and optional plain-text progress."""
from __future__ import annotations
import json
import os
import signal
import time
from contextlib import contextmanager
from pathlib import Path
PAUSED_EXIT_CODE = 75
CHECKPOINT_SECONDS = 60
_pause_requested = False
_defer = 0
_state: dict = {}
_last_save = 0.0
_last_progress = 0.0
_previous_clean = False
class TaskPaused(BaseException):
"""Unwind at a safe boundary without treating a pause as an ordinary failure."""
def work_directory() -> Path | None:
value = os.environ.get("SERVER_MAINTENANCE_WORK")
return Path(value) if value else None
def atomic_json(path: Path, value: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
temporary = path.with_suffix(path.suffix + ".tmp")
with temporary.open("w", encoding="utf-8") as stream:
temporary.chmod(0o600)
json.dump(value, stream, ensure_ascii=False)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
temporary.replace(path)
descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def initialize() -> None:
global _state, _previous_clean, _pause_requested, _last_save
_pause_requested = False
root = work_directory()
if root is not None:
try:
saved = json.loads((root / "checkpoint.json").read_text())
except FileNotFoundError:
saved = {}
_state = saved.get("data", {})
_previous_clean = bool(saved.get("clean_shutdown"))
_write_checkpoint(False)
_last_save = time.monotonic()
signal.signal(signal.SIGUSR1, request_pause)
def request_pause(_signal=None, _frame=None) -> None:
global _pause_requested
_pause_requested = True
def pause_requested() -> bool:
return _pause_requested and work_directory() is not None
def checkpoint_data() -> dict:
return _state if work_directory() is not None else {}
def was_clean_shutdown() -> bool:
return _previous_clean
def _write_checkpoint(clean: bool) -> None:
root = work_directory()
if root is not None:
atomic_json(root / "checkpoint.json", {
"version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(),
})
def checkpoint(data: dict | None = None, *, force: bool = False) -> None:
"""Keep the latest state in memory; persist ordinary repeatable work once a minute."""
global _state, _last_save
if data is not None:
_state = data
if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS:
_write_checkpoint(False)
_last_save = time.monotonic()
check_stop()
def check_stop() -> None:
if _pause_requested and work_directory() is not None and not _defer:
_write_checkpoint(True)
raise TaskPaused()
@contextmanager
def critical_section():
"""Finish a small consistency-sensitive operation before observing a pause."""
global _defer
check_stop()
_defer += 1
try:
yield
finally:
_defer -= 1
check_stop()
def sleep(seconds: float) -> None:
deadline = time.monotonic() + seconds
while (remaining := deadline - time.monotonic()) > 0:
check_stop()
time.sleep(min(remaining, 0.25))
check_stop()
def progress(text: str, *, force: bool = False) -> None:
global _last_progress
root = work_directory()
if root is not None and (force or time.monotonic() - _last_progress >= 1):
path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json")))
atomic_json(path, {"text": text, "updated_at": time.time()})
_last_progress = time.monotonic()
class Progress:
"""Item-count estimates, independently usable for several named steps."""
def __init__(self, totals: dict[str, int]):
self.totals = totals
self.done = dict.fromkeys(totals, 0)
self.started: dict[str, float] = {}
self.baseline: dict[str, int] = {}
def update(self, label: str, done: int, *, force: bool = False) -> None:
self.started.setdefault(label, time.monotonic())
self.baseline.setdefault(label, done)
self.done[label] = done
lines = []
for name, total in self.totals.items():
count = self.done[name]
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
line = f"{name}: {percent}%"
processed = count - self.baseline.get(name, 0)
if processed > 0 and count < total and name in self.started:
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
line += f", ETA {estimate}"
lines.append(line)
progress("\n".join(lines), force=force)