"""Execute task code in isolation, optionally entering its embedded recovery hook.""" import os import runpy import sys from pathlib import Path from urllib.error import HTTPError, URLError from . import task_runtime as runtime def service_unavailable(error: Exception) -> bool: """Retry connection failures, not authentication, invalid data or task bugs.""" while error is not None: if isinstance(error, HTTPError): return error.code in {408, 502, 503, 504} if isinstance(error, (ConnectionError, TimeoutError)): return True if isinstance(error, URLError): return isinstance(error.reason, (OSError, TimeoutError)) sqlstate = getattr(error, "sqlstate", None) if sqlstate and (sqlstate.startswith("08") or sqlstate in {"57P01", "57P03"}): return True if type(error).__module__.startswith("psycopg") and type(error).__name__ == "OperationalError": text = str(error).casefold() return any(reason in text for reason in ("connection refused", "timeout expired", "could not translate host name", "server closed the connection")) error = error.__cause__ return False def main() -> None: path = Path(sys.argv[1]).resolve() sys.path.insert(0, str(path.parent)) sys.argv = [str(path)] runtime.initialize() try: if os.environ.get("SERVER_MAINTENANCE_RECOVER") == "1": namespace = runpy.run_path(str(path), run_name="__task_recovery__") recover = namespace.get("recover") if not callable(recover): raise RuntimeError("This task no longer provides its recovery function") print("Recovering interrupted task; verifying current state.", flush=True) attempts = 0 while True: try: recover() break except Exception as error: if not service_unavailable(error): raise attempts += 1 delay = min(30, 2 ** min(attempts, 5)) message = f"Recovery waiting for an unavailable service; retry in {delay} seconds." runtime.progress(message, force=True) print(message, flush=True) runtime.sleep(delay) else: runpy.run_path(str(path), run_name="__main__") except runtime.TaskPaused: print("Paused at a safe checkpoint for recovery after restart.", flush=True) raise SystemExit(runtime.PAUSED_EXIT_CODE) if __name__ == "__main__": main()