Add cooperative task recovery and persistent live status
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 40s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
034fecbd51
commit
3bfd199b9c
28 files changed
+2356
-172
No files matched your search
+105
-15
@@ -14,7 +14,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u
|
||||
from .config import Settings
|
||||
from .discovery import TaskCatalog
|
||||
from .inputs import PendingUpload, validate_inputs, validate_uploads
|
||||
from .runner import RunManager
|
||||
from .runner import RunManager, TaskConflictError
|
||||
from .scheduler import BackgroundScheduler
|
||||
from .store import RunStore
|
||||
|
||||
@@ -24,11 +24,8 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
catalog = TaskCatalog(settings.task_root)
|
||||
store = RunStore(settings.database_path)
|
||||
store.initialize()
|
||||
store.mark_interrupted()
|
||||
manager = RunManager(settings, catalog, store)
|
||||
manager.cleanup_transient_files()
|
||||
scheduler = BackgroundScheduler(catalog, manager)
|
||||
scheduler.start()
|
||||
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
app = Flask(
|
||||
@@ -62,6 +59,66 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
return "—"
|
||||
return datetime.fromisoformat(value).astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
|
||||
|
||||
def run_links(run):
|
||||
if run is None:
|
||||
return None
|
||||
display_status = run.status
|
||||
if run.recovery_of_run_id is not None and run.status in {"queued", "running"}:
|
||||
display_status = "Recovery queued" if run.status == "queued" else "recovering"
|
||||
return {
|
||||
"run_id": run.id,
|
||||
"status": run.status,
|
||||
"display_status": display_status,
|
||||
"recovery_of_run_id": run.recovery_of_run_id,
|
||||
"artifacts_url": url_for("run_artifacts", run_id=run.id),
|
||||
"detail_url": url_for("run_detail", run_id=run.id),
|
||||
"output_url": url_for("run_output", run_id=run.id),
|
||||
"state_url": url_for("run_state", run_id=run.id),
|
||||
}
|
||||
|
||||
def task_state(task, active_runs=None):
|
||||
state = manager.task_status(task, active_runs=active_runs)
|
||||
return {
|
||||
"active_run": run_links(state["active_run"]),
|
||||
"blocked_by": run_links(state["blocked_by"]),
|
||||
"progress": state["progress"],
|
||||
}
|
||||
|
||||
def live_states(groups, latest):
|
||||
active = store.active_runs()
|
||||
tasks = {task.id: task_state(task, active) for group in groups for task in group.tasks}
|
||||
group_states = {}
|
||||
for group in groups:
|
||||
run = latest[("group", group.id)]
|
||||
active = run if run and run.status in {"queued", "running"} else None
|
||||
blocker = next((
|
||||
tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
|
||||
for task in group.tasks
|
||||
if tasks[task.id]["active_run"] or tasks[task.id]["blocked_by"]
|
||||
), None)
|
||||
group_states[group.id] = {
|
||||
"active_run": run_links(active),
|
||||
"blocked_by": None if active else blocker,
|
||||
"progress": manager.progress(active) if active else "",
|
||||
}
|
||||
return {"tasks": tasks, "groups": group_states}
|
||||
|
||||
def unavailable(error):
|
||||
if isinstance(error, TaskConflictError):
|
||||
return task_conflict(error)
|
||||
message = str(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {"error": message}, 503
|
||||
return render_template("unavailable.html", message=message, blocking_run=None), 503
|
||||
|
||||
@app.errorhandler(TaskConflictError)
|
||||
def task_conflict(error):
|
||||
blocking_run = run_links(error.run)
|
||||
message = f"Task blocked by run #{error.run.id} ({blocking_run['display_status']})."
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {"error": message, "blocked_by": blocking_run}, 409
|
||||
return render_template("unavailable.html", message=message, blocking_run=blocking_run), 409
|
||||
|
||||
def render_index(
|
||||
*,
|
||||
form_values: dict[str, dict[str, list[str]]] | None = None,
|
||||
@@ -80,6 +137,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
"index.html",
|
||||
groups=groups,
|
||||
latest=latest,
|
||||
live_states=live_states(groups, latest),
|
||||
form_values=form_values or {},
|
||||
input_errors=input_errors or {},
|
||||
),
|
||||
@@ -90,6 +148,14 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
def index():
|
||||
return render_index()
|
||||
|
||||
@app.get("/tasks/state")
|
||||
def tasks_state():
|
||||
groups = catalog.groups()
|
||||
latest = {("group", group.id): store.last_run("group", group.id) for group in groups}
|
||||
response = app.json.response(live_states(groups, latest))
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
return response
|
||||
|
||||
@app.post("/tasks/<path:task_id>/run")
|
||||
def run_task(task_id: str):
|
||||
task = catalog.task(task_id)
|
||||
@@ -114,15 +180,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
input_errors={task.id: errors},
|
||||
status=400,
|
||||
)
|
||||
run, _ = manager.run_task(task, input_data, uploads)
|
||||
try:
|
||||
run, _ = manager.run_task(task, input_data, uploads)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return {
|
||||
"artifacts_url": url_for("run_artifacts", run_id=run.id),
|
||||
"detail_url": url_for("run_detail", run_id=run.id),
|
||||
"output_url": url_for("run_output", run_id=run.id),
|
||||
"run_id": run.id,
|
||||
"state_url": url_for("run_state", run_id=run.id),
|
||||
}, 202
|
||||
return run_links(run), 202
|
||||
return redirect(url_for("run_detail", run_id=run.id))
|
||||
|
||||
@app.post("/groups/<path:group_id>/run")
|
||||
@@ -130,7 +193,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
group = catalog.group(group_id)
|
||||
if group is None or not group.can_run_as_group:
|
||||
abort(404)
|
||||
run, _ = manager.run_group(group)
|
||||
try:
|
||||
run, _ = manager.run_group(group)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return run_links(run), 202
|
||||
return redirect(url_for("run_detail", run_id=run.id))
|
||||
|
||||
@app.get("/runs/<int:run_id>")
|
||||
@@ -147,6 +215,10 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
output=output,
|
||||
output_offset=offset,
|
||||
execution_task=execution_task,
|
||||
recovery_run=run_links(store.recovery_for(run.id)),
|
||||
progress=manager.progress(run),
|
||||
display_status=run_links(run)["display_status"],
|
||||
live_states={"tasks": {execution_task.id: task_state(execution_task)} if execution_task else {}, "groups": {}},
|
||||
)
|
||||
|
||||
def execution_replay_task(run):
|
||||
@@ -190,9 +262,14 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
except OSError:
|
||||
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
|
||||
uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream)
|
||||
executed, created = manager.run_task(task, validated, uploads)
|
||||
try:
|
||||
executed, created = manager.run_task(task, validated, uploads)
|
||||
except RuntimeError as error:
|
||||
return unavailable(error)
|
||||
if created and executed.status == "queued":
|
||||
manager.discard_review_uploads(run.id)
|
||||
if request.accept_mimetypes.best == "application/json":
|
||||
return run_links(executed), 202
|
||||
return redirect(url_for("run_detail", run_id=executed.id))
|
||||
|
||||
@app.post("/runs/<int:run_id>/cancel")
|
||||
@@ -216,6 +293,9 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
return {
|
||||
"output": output, "offset": offset, "reset": reset, "has_more": has_more,
|
||||
"status": run.status,
|
||||
"display_status": run_links(run)["display_status"],
|
||||
"recovery_of_run_id": run.recovery_of_run_id,
|
||||
"progress": manager.progress(run),
|
||||
}
|
||||
|
||||
def read_run_output(run, offset: int = 0, maximum_bytes: int | None = None) -> tuple[str, int, bool, bool]:
|
||||
@@ -299,8 +379,10 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
if run is None:
|
||||
abort(404)
|
||||
return {
|
||||
"status": run.status,
|
||||
**run_links(run),
|
||||
"progress": manager.progress(run),
|
||||
"finished_at": run.finished_at,
|
||||
"recovery_run": run_links(store.recovery_for(run.id)),
|
||||
"exit_code": run.exit_code,
|
||||
"cancel_requested_at": run.cancel_requested_at,
|
||||
}
|
||||
@@ -316,4 +398,12 @@ def create_app(settings: Settings | None = None) -> Flask:
|
||||
runs=store.history(target_kind, target_path),
|
||||
)
|
||||
|
||||
try:
|
||||
manager.startup()
|
||||
manager.cleanup_transient_files()
|
||||
scheduler.start()
|
||||
except BaseException:
|
||||
scheduler.stop()
|
||||
manager.shutdown(timeout=5)
|
||||
raise
|
||||
return app
|
||||
Reference in new issue
Block a user