Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -0,0 +1,161 @@
|
||||
"""Cooperative task checkpoints and optional plain-text progress."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
PAUSED_EXIT_CODE = 75
|
||||
CHECKPOINT_SECONDS = 60
|
||||
_pause_requested = False
|
||||
_defer = 0
|
||||
_state: dict = {}
|
||||
_last_save = 0.0
|
||||
_last_progress = 0.0
|
||||
_previous_clean = False
|
||||
|
||||
|
||||
class TaskPaused(BaseException):
|
||||
"""Unwind at a safe boundary without treating a pause as an ordinary failure."""
|
||||
|
||||
|
||||
def work_directory() -> Path | None:
|
||||
value = os.environ.get("SERVER_MAINTENANCE_WORK")
|
||||
return Path(value) if value else None
|
||||
|
||||
|
||||
def atomic_json(path: Path, value: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
with temporary.open("w", encoding="utf-8") as stream:
|
||||
temporary.chmod(0o600)
|
||||
json.dump(value, stream, ensure_ascii=False)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
temporary.replace(path)
|
||||
descriptor = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def initialize() -> None:
|
||||
global _state, _previous_clean, _pause_requested, _last_save
|
||||
_pause_requested = False
|
||||
root = work_directory()
|
||||
if root is not None:
|
||||
try:
|
||||
saved = json.loads((root / "checkpoint.json").read_text())
|
||||
except FileNotFoundError:
|
||||
saved = {}
|
||||
_state = saved.get("data", {})
|
||||
_previous_clean = bool(saved.get("clean_shutdown"))
|
||||
_write_checkpoint(False)
|
||||
_last_save = time.monotonic()
|
||||
signal.signal(signal.SIGUSR1, request_pause)
|
||||
|
||||
|
||||
def request_pause(_signal=None, _frame=None) -> None:
|
||||
global _pause_requested
|
||||
_pause_requested = True
|
||||
|
||||
|
||||
def pause_requested() -> bool:
|
||||
return _pause_requested and work_directory() is not None
|
||||
|
||||
|
||||
def checkpoint_data() -> dict:
|
||||
return _state if work_directory() is not None else {}
|
||||
|
||||
|
||||
def was_clean_shutdown() -> bool:
|
||||
return _previous_clean
|
||||
|
||||
|
||||
def _write_checkpoint(clean: bool) -> None:
|
||||
root = work_directory()
|
||||
if root is not None:
|
||||
atomic_json(root / "checkpoint.json", {
|
||||
"version": 1, "data": _state, "clean_shutdown": clean, "saved_at": time.time(),
|
||||
})
|
||||
|
||||
|
||||
def checkpoint(data: dict | None = None, *, force: bool = False) -> None:
|
||||
"""Keep the latest state in memory; persist ordinary repeatable work once a minute."""
|
||||
global _state, _last_save
|
||||
if data is not None:
|
||||
_state = data
|
||||
if force or time.monotonic() - _last_save >= CHECKPOINT_SECONDS:
|
||||
_write_checkpoint(False)
|
||||
_last_save = time.monotonic()
|
||||
check_stop()
|
||||
|
||||
|
||||
def check_stop() -> None:
|
||||
if _pause_requested and work_directory() is not None and not _defer:
|
||||
_write_checkpoint(True)
|
||||
raise TaskPaused()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def critical_section():
|
||||
"""Finish a small consistency-sensitive operation before observing a pause."""
|
||||
global _defer
|
||||
check_stop()
|
||||
_defer += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_defer -= 1
|
||||
check_stop()
|
||||
|
||||
|
||||
def sleep(seconds: float) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while (remaining := deadline - time.monotonic()) > 0:
|
||||
check_stop()
|
||||
time.sleep(min(remaining, 0.25))
|
||||
check_stop()
|
||||
|
||||
|
||||
def progress(text: str, *, force: bool = False) -> None:
|
||||
global _last_progress
|
||||
root = work_directory()
|
||||
if root is not None and (force or time.monotonic() - _last_progress >= 1):
|
||||
path = Path(os.environ.get("SERVER_MAINTENANCE_PROGRESS", str(root / "progress.json")))
|
||||
atomic_json(path, {"text": text, "updated_at": time.time()})
|
||||
_last_progress = time.monotonic()
|
||||
|
||||
|
||||
class Progress:
|
||||
"""Item-count estimates, independently usable for several named steps."""
|
||||
|
||||
def __init__(self, totals: dict[str, int]):
|
||||
self.totals = totals
|
||||
self.done = dict.fromkeys(totals, 0)
|
||||
self.started: dict[str, float] = {}
|
||||
self.baseline: dict[str, int] = {}
|
||||
|
||||
def update(self, label: str, done: int, *, force: bool = False) -> None:
|
||||
self.started.setdefault(label, time.monotonic())
|
||||
self.baseline.setdefault(label, done)
|
||||
self.done[label] = done
|
||||
lines = []
|
||||
for name, total in self.totals.items():
|
||||
count = self.done[name]
|
||||
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
|
||||
line = f"{name}: {percent}%"
|
||||
processed = count - self.baseline.get(name, 0)
|
||||
if processed > 0 and count < total and name in self.started:
|
||||
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
|
||||
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
|
||||
line += f", ETA {estimate}"
|
||||
lines.append(line)
|
||||
progress("\n".join(lines), force=force)
|
||||
Reference in new issue
Block a user