Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
@@ -0,0 +1,65 @@
|
||||
"""Execute task code in isolation, optionally entering its embedded recovery hook."""
|
||||
|
||||
import os
|
||||
import runpy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError, URLError
|
||||
|
||||
from . import task_runtime as runtime
|
||||
|
||||
|
||||
def service_unavailable(error: Exception) -> bool:
|
||||
"""Retry connection failures, not authentication, invalid data or task bugs."""
|
||||
while error is not None:
|
||||
if isinstance(error, HTTPError):
|
||||
return error.code in {408, 502, 503, 504}
|
||||
if isinstance(error, (ConnectionError, TimeoutError)):
|
||||
return True
|
||||
if isinstance(error, URLError):
|
||||
return isinstance(error.reason, (OSError, TimeoutError))
|
||||
sqlstate = getattr(error, "sqlstate", None)
|
||||
if sqlstate and (sqlstate.startswith("08") or sqlstate in {"57P01", "57P03"}):
|
||||
return True
|
||||
if type(error).__module__.startswith("psycopg") and type(error).__name__ == "OperationalError":
|
||||
text = str(error).casefold()
|
||||
return any(reason in text for reason in ("connection refused", "timeout expired", "could not translate host name", "server closed the connection"))
|
||||
error = error.__cause__
|
||||
return False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
path = Path(sys.argv[1]).resolve()
|
||||
sys.path.insert(0, str(path.parent))
|
||||
sys.argv = [str(path)]
|
||||
runtime.initialize()
|
||||
try:
|
||||
if os.environ.get("SERVER_MAINTENANCE_RECOVER") == "1":
|
||||
namespace = runpy.run_path(str(path), run_name="__task_recovery__")
|
||||
recover = namespace.get("recover")
|
||||
if not callable(recover):
|
||||
raise RuntimeError("This task no longer provides its recovery function")
|
||||
print("Recovering interrupted task; verifying current state.", flush=True)
|
||||
attempts = 0
|
||||
while True:
|
||||
try:
|
||||
recover()
|
||||
break
|
||||
except Exception as error:
|
||||
if not service_unavailable(error):
|
||||
raise
|
||||
attempts += 1
|
||||
delay = min(30, 2 ** min(attempts, 5))
|
||||
message = f"Recovery waiting for an unavailable service; retry in {delay} seconds."
|
||||
runtime.progress(message, force=True)
|
||||
print(message, flush=True)
|
||||
runtime.sleep(delay)
|
||||
else:
|
||||
runpy.run_path(str(path), run_name="__main__")
|
||||
except runtime.TaskPaused:
|
||||
print("Paused at a safe checkpoint for recovery after restart.", flush=True)
|
||||
raise SystemExit(runtime.PAUSED_EXIT_CODE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user