Allow uploaded review files to be reused for execution
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m15s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 25s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 33s

This commit is contained in:
ajp_anton committed 2026-10-27 04:10:10 +00:00
1 parent 55bbbe2fd7
commit 034fecbd51
4 files changed
+212 -15

No files matched your search

+6 -2
View File
@@ -111,6 +111,10 @@ execution mode from the run-detail page. The task must still revalidate the
live filesystem before changing anything.
Execution creates a separate run; the original review and its output remain
accessible through Run history on either run's detail page.
For tasks with file inputs, successful reviews keep the uploaded files privately
for up to 24 hours. Execute copies those files into the new run, then removes
the review's copies. Missing or expired uploads require a new review. Tasks
with sensitive inputs cannot use this replay feature.
```json
{
@@ -129,8 +133,8 @@ execution file instead. Tasks read the applicable file path from
`SERVER_MAINTENANCE_INPUT`; no user value is appended to the command line. A
task without an entry in the manifest gets an empty JSON object. Uploaded files
are stored in a private per-run directory; their absolute paths are provided in
the input JSON and the files are removed when the task finishes, fails, or is
cancelled.
the input JSON. Files are removed when the task finishes, fails, or is cancelled,
except for successful review uploads retained temporarily for execution as above.
A group may also include a private `task-config.json` for stable local details
such as mounted filesystem roots. This is not part of discovery or the web
+42 -1
View File
@@ -21,6 +21,7 @@ from .store import Run, RunStore
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
class RunManager:
@@ -117,6 +118,41 @@ class RunManager:
except OSError:
pass
def review_upload_paths(self, run: Run, task: Task) -> dict[str, Path] | None:
"""Return reusable uploads, or None when a run cannot be executed again."""
if run.status != "succeeded" or run.target_kind != "task" or task.execution is None:
return None
if any(field.sensitive for field in task.inputs):
return None
try:
values = json.loads(run.input_json)
except json.JSONDecodeError:
return None
if not isinstance(values, dict) or values.get(task.execution.field.name) != task.execution.dry_run_value:
return None
root = self._upload_root(run.id)
paths = {}
for field in task.inputs:
if field.type != "file":
continue
value = values.get(field.name)
if not value and not field.required:
continue
if not isinstance(value, str) or not run.finished_at:
return None
if time.time() - datetime.fromisoformat(run.finished_at).timestamp() >= REVIEW_UPLOAD_RETENTION_SECONDS:
return None
path = Path(value)
if root.is_symlink() or path.parent != root or path.is_symlink() or not path.is_file():
return None
if path.name != field.name + path.suffix or (field.accept and path.suffix.lower() not in field.accept):
return None
paths[field.name] = path
return paths
def discard_review_uploads(self, run_id: int) -> None:
self._delete_uploads(run_id)
def cleanup_transient_files(
self,
*,
@@ -144,6 +180,9 @@ class RunManager:
if kind in {"input.json", "execution-input.json"} and path.is_file():
path.unlink(missing_ok=True)
elif kind == "uploads" and path.is_dir():
run = self.store.get(run_id)
task = self.catalog.task(run.target_path) if run and run.target_kind == "task" else None
if task is None or not self.review_upload_paths(run, task):
shutil.rmtree(path, ignore_errors=True)
elif kind == "artifacts" and path.is_dir():
if status in {"failed", "interrupted", "cancelled"}:
@@ -370,13 +409,13 @@ class RunManager:
finally:
with self._lock:
self._processes.pop(run.id, None)
self._delete_uploads(run.id)
self._delete_execution_input(run.id)
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
if self.store.is_cancel_requested(run.id) or (
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
):
self._delete_uploads(run.id)
self._keep_diagnostic_reports(artifact_root)
return self.store.finish(
run.id,
@@ -390,6 +429,8 @@ class RunManager:
exit_code=exit_code,
log_path=str(log_path),
)
if not self.review_upload_paths(completed, task):
self._delete_uploads(run.id)
if completed.status != "succeeded":
self._keep_diagnostic_reports(artifact_root)
else:
+21 -12
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import codecs
import ipaddress
import json
from contextlib import ExitStack
from datetime import datetime
from pathlib import Path
@@ -12,7 +13,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u
from .config import Settings
from .discovery import TaskCatalog
from .inputs import validate_inputs, validate_uploads
from .inputs import PendingUpload, validate_inputs, validate_uploads
from .runner import RunManager
from .scheduler import BackgroundScheduler
from .store import RunStore
@@ -154,15 +155,7 @@ def create_app(settings: Settings | None = None) -> Flask:
task = catalog.task(run.target_path)
if task is None or task.execution is None:
return None
if any(field.type == "file" or field.sensitive for field in task.inputs):
return None
try:
input_data = json.loads(run.input_json)
except json.JSONDecodeError:
return None
if not isinstance(input_data, dict):
return None
if input_data.get(task.execution.field.name) != task.execution.dry_run_value:
if manager.review_upload_paths(run, task) is None:
return None
return task
@@ -177,13 +170,29 @@ def create_app(settings: Settings | None = None) -> Flask:
input_data = json.loads(run.input_json)
input_data[task.execution.field.name] = task.execution.execute_value
raw_values = {
name: [str(value) for value in value] if isinstance(value, list) else [str(value)]
name: ([str(item) for item in value] or [""]) if isinstance(value, list) else [str(value)]
for name, value in input_data.items()
}
validated, errors = validate_inputs(task.inputs, raw_values)
if errors:
abort(409)
executed, _ = manager.run_task(task, validated)
paths = manager.review_upload_paths(run, task)
if paths is None:
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
with ExitStack() as stack:
uploads = {}
for field in task.inputs:
if field.name not in paths:
continue
path = paths[field.name]
try:
stream = stack.enter_context(path.open("rb"))
except OSError:
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream)
executed, created = manager.run_task(task, validated, uploads)
if created and executed.status == "queued":
manager.discard_review_uploads(run.id)
return redirect(url_for("run_detail", run_id=executed.id))
@app.post("/runs/<int:run_id>/cancel")
+143
View File
@@ -0,0 +1,143 @@
from __future__ import annotations
import json
import time
from io import BytesIO
from pathlib import Path
import pytest
from app.web import create_app
from .conftest import write_task
from .test_runner import wait_for
def start_review(settings, *, sensitive=False, fail=False):
task = write_task(settings.task_root, "imports", "apply.py", '''
import json
import os
from pathlib import Path
values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
print(json.dumps({"mode": values["mode"], "csv": Path(values["csv"]).read_text()}))
''' + ("raise SystemExit(1)\n" if fail else ""))
(task.parent / "task-inputs.json").write_text(json.dumps({
"version": 1,
"tasks": {"apply.py": {
"inputs": [
{"name": "csv", "label": "CSV", "type": "file", "required": True,
"accept": [".csv"], "maximum_bytes": 1024, "sensitive": sensitive},
{"name": "choices", "label": "Choices", "type": "multi_choice",
"options": ["one"], "default": ["one"]},
],
"execution": {"field": "mode", "dry_run_value": "review", "dry_run_label": "Review",
"execute_value": "apply", "execute_label": "Execute"},
}},
}))
app = create_app(settings)
client = app.test_client()
response = client.post("/tasks/imports/apply.py/run", data={
"csv": (BytesIO(b"answer\nsame\n"), "decisions.csv"), "choices": "",
})
run_id = int(response.location.rsplit("/", 1)[1])
manager = app.extensions["run_manager"]
assert wait_for(manager, run_id, {"failed" if fail else "succeeded"}) == ("failed" if fail else "succeeded")
manager.cleanup_transient_files()
return app, client, run_id
def test_uploaded_review_can_execute_with_the_same_csv_and_keeps_its_output(settings):
app, client, review_id = start_review(settings)
store = app.extensions["run_store"]
original = Path(json.loads(store.get(review_id).input_json)["csv"])
assert original.is_file()
assert original.stat().st_mode & 0o777 == 0o600
assert f'action="/runs/{review_id}/execute"'.encode() in client.get(f"/runs/{review_id}").data
response = client.post(f"/runs/{review_id}/execute")
assert response.status_code == 302
executed_id = int(response.location.rsplit("/", 1)[1])
assert executed_id != review_id
assert wait_for(app.extensions["run_manager"], executed_id, {"succeeded"}) == "succeeded"
app.extensions["run_manager"].cleanup_transient_files()
review_output = json.loads(client.get(f"/runs/{review_id}/output").get_json()["output"])
execute_output = json.loads(client.get(f"/runs/{executed_id}/output").get_json()["output"])
assert review_output == {"mode": "review", "csv": "answer\nsame\n"}
assert execute_output == {"mode": "apply", "csv": "answer\nsame\n"}
assert json.loads(store.get(executed_id).input_json)["choices"] == []
assert not original.parent.exists()
assert not Path(json.loads(store.get(executed_id).input_json)["csv"]).parent.exists()
assert client.post(f"/runs/{review_id}/execute").status_code == 404
def test_review_upload_survives_application_restart(settings):
app, _, run_id = start_review(settings)
original = Path(json.loads(app.extensions["run_store"].get(run_id).input_json)["csv"])
restarted = create_app(settings)
assert original.is_file()
assert f'action="/runs/{run_id}/execute"'.encode() in restarted.test_client().get(f"/runs/{run_id}").data
def test_unused_review_upload_expires_after_one_day(settings, monkeypatch):
app, client, run_id = start_review(settings)
original = Path(json.loads(app.extensions["run_store"].get(run_id).input_json)["csv"])
future = time.time() + 86401
monkeypatch.setattr("app.runner.time.time", lambda: future)
assert client.post(f"/runs/{run_id}/execute").status_code == 404
app.extensions["run_manager"].cleanup_transient_files()
assert not original.parent.exists()
@pytest.mark.parametrize("sensitive,fail", [(True, False), (False, True)])
def test_sensitive_or_failed_review_does_not_retain_uploads(settings, sensitive, fail):
_, client, run_id = start_review(settings, sensitive=sensitive, fail=fail)
assert not (settings.state_root / "runs" / f"{run_id}.uploads").exists()
assert client.post(f"/runs/{run_id}/execute").status_code == 404
@pytest.mark.parametrize("invalid_upload", ["missing", "symlink", "outside"])
def test_missing_or_unsafe_review_upload_cannot_execute(settings, invalid_upload):
app, client, run_id = start_review(settings)
store = app.extensions["run_store"]
values = json.loads(store.get(run_id).input_json)
original = Path(values["csv"])
original.unlink()
if invalid_upload != "missing":
outside = settings.state_root / "other.csv"
outside.write_text("not the reviewed upload")
if invalid_upload == "symlink":
original.symlink_to(outside)
else:
values["csv"] = str(outside)
store.set_input(run_id, values)
assert client.post(f"/runs/{run_id}/execute").status_code == 404
assert f'action="/runs/{run_id}/execute"'.encode() not in client.get(f"/runs/{run_id}").data
def test_failed_execution_enqueue_keeps_original_review_upload(settings, monkeypatch):
app, client, run_id = start_review(settings)
manager = app.extensions["run_manager"]
original = Path(json.loads(manager.store.get(run_id).input_json)["csv"])
def cannot_save(*_):
raise OSError("test: disk full")
monkeypatch.setattr(manager, "_save_uploads", cannot_save)
response = client.post(f"/runs/{run_id}/execute")
failed_id = int(response.location.rsplit("/", 1)[1])
assert manager.store.get(failed_id).status == "failed"
assert original.is_file()
assert f'action="/runs/{run_id}/execute"'.encode() in client.get(f"/runs/{run_id}").data
def test_removing_file_input_from_task_cleans_up_unused_review_upload(settings):
app, _, run_id = start_review(settings)
original = Path(json.loads(app.extensions["run_store"].get(run_id).input_json)["csv"])
manifest = settings.task_root / "imports" / "task-inputs.json"
definition = json.loads(manifest.read_text())
definition["tasks"]["apply.py"]["inputs"] = []
manifest.write_text(json.dumps(definition))
app.extensions["run_manager"].cleanup_transient_files()
assert not original.parent.exists()