"""Cooperative task checkpoints and optional plain-text progress.""" from __future__ import annotations import json import os import signal import time from contextlib import contextmanager from pathlib import Path PAUSED_EXIT_CODE = 75 CHECKPOINT_SECONDS = 60 _pause_requested = False _defer = 0 _state: dict = {} _last_save = 0.0 _last_progress = 0.0 _previous_clean = False class TaskPaused(BaseException): """Unwind at a safe boundary without treating a pause as an ordinary failure.""" def work_directory() -> Path | None: value = os.environ.get("SERVER_MAINTENANCE_WORK") return Path(value) if value else None def atomic_json(path: Path, value: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) temporary = path.with_suffix(path.suffix + ".tmp") with temporary.open("w", encoding="utf-8") as stream: temporary.chmod(0o600) json.dump(value, stream, ensure_ascii=False) stream.write("\n") stream.flush() os.fsync(stream.fileno()) temporary.replace(path) descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(descriptor) finally: os.close(descriptor) def initialize() -> None: global _state, _previous_clean, _pause_requested, _last_save _pause_requested = False root = work_directory() if root is not None: try: saved = json.loads((root / "checkpoint.json").read_text()) except FileNotFoundError: saved = {} _state = saved.get("data", {}) _previous_clean = bool(saved.get("clean_shutdown")) _write_checkpoint(False) _last_save = time.monotonic() signal.signal(signal.SIGUSR1, request_pause) def request_pause(_signal=None, _frame=None) -> None: global _pause_requested _pause_requested = True def pause_requested() -> bool: return _pause_requested and work_directory() is not None def checkpoint_data() -> dict: return _state if work_directory() is not None else {} def was_clean_shutdown() -> bool: return _previous_clean def _write_checkpoint(clean: bool) -> None: root = work_directory() if root is not None: atomic_json(root / "checkpoint.json", { "version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(), }) def checkpoint(data: dict | None = None, *, force: bool = False) -> None: """Keep the latest state in memory; persist ordinary repeatable work once a minute.""" global _state, _last_save if data is not None: _state = data if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS: _write_checkpoint(False) _last_save = time.monotonic() check_stop() def check_stop() -> None: if _pause_requested and work_directory() is not None and not _defer: _write_checkpoint(True) raise TaskPaused() @contextmanager def critical_section(): """Finish a small consistency-sensitive operation before observing a pause.""" global _defer check_stop() _defer += 1 try: yield finally: _defer -= 1 check_stop() def sleep(seconds: float) -> None: deadline = time.monotonic() + seconds while (remaining := deadline - time.monotonic()) > 0: check_stop() time.sleep(min(remaining, 0.25)) check_stop() def progress(text: str, *, force: bool = False) -> None: global _last_progress root = work_directory() if root is not None and (force or time.monotonic() - _last_progress >= 1): path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json"))) atomic_json(path, {"text": text, "updated_at": time.time()}) _last_progress = time.monotonic() class Progress: """Item-count estimates, independently usable for several named steps.""" def __init__(self, totals: dict[str, int]): self.totals = totals self.done = dict.fromkeys(totals, 0) self.started: dict[str, float] = {} self.baseline: dict[str, int] = {} def update(self, label: str, done: int, *, force: bool = False) -> None: force = force or label not in self.started or done >= self.totals[label] self.started.setdefault(label, time.monotonic()) self.baseline.setdefault(label, done) self.done[label] = done lines = [] for name, total in self.totals.items(): count = self.done[name] if total: line = f"{name}: {count}/{total} ({min(100, int(count * 100 / total))}%)" else: line = f"{name}: {'no items' if name in self.started else 'waiting'}" processed = count - self.baseline.get(name, 0) if processed > 0 and count < total and name in self.started: seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count))) value, unit = (seconds, "second") if seconds < 60 else (round(seconds / 60), "minute") line += f", ETA {value} {unit}{'' if value == 1 else 's'}" lines.append(line) progress("\n".join(lines), force=force)