Allow uploaded review files to be reused for execution
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m15s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 25s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 33s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 38s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 49s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m15s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 25s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 33s
This commit is contained in:
1 parent
55bbbe2fd7
commit
034fecbd51
4 files changed
+212
-15
No files matched your search
@@ -111,6 +111,10 @@ execution mode from the run-detail page. The task must still revalidate the
|
|||||||
live filesystem before changing anything.
|
live filesystem before changing anything.
|
||||||
Execution creates a separate run; the original review and its output remain
|
Execution creates a separate run; the original review and its output remain
|
||||||
accessible through Run history on either run's detail page.
|
accessible through Run history on either run's detail page.
|
||||||
|
For tasks with file inputs, successful reviews keep the uploaded files privately
|
||||||
|
for up to 24 hours. Execute copies those files into the new run, then removes
|
||||||
|
the review's copies. Missing or expired uploads require a new review. Tasks
|
||||||
|
with sensitive inputs cannot use this replay feature.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -129,8 +133,8 @@ execution file instead. Tasks read the applicable file path from
|
|||||||
`SERVER_MAINTENANCE_INPUT`; no user value is appended to the command line. A
|
`SERVER_MAINTENANCE_INPUT`; no user value is appended to the command line. A
|
||||||
task without an entry in the manifest gets an empty JSON object. Uploaded files
|
task without an entry in the manifest gets an empty JSON object. Uploaded files
|
||||||
are stored in a private per-run directory; their absolute paths are provided in
|
are stored in a private per-run directory; their absolute paths are provided in
|
||||||
the input JSON and the files are removed when the task finishes, fails, or is
|
the input JSON. Files are removed when the task finishes, fails, or is cancelled,
|
||||||
cancelled.
|
except for successful review uploads retained temporarily for execution as above.
|
||||||
|
|
||||||
A group may also include a private `task-config.json` for stable local details
|
A group may also include a private `task-config.json` for stable local details
|
||||||
such as mounted filesystem roots. This is not part of discovery or the web
|
such as mounted filesystem roots. This is not part of discovery or the web
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from .store import Run, RunStore
|
|||||||
|
|
||||||
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
|
ORPHANED_TASK_STATE_FILENAME = ".orphaned-groups.json"
|
||||||
ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
|
ORPHANED_TASK_STATE_RETENTION_SECONDS = 14 * 86400
|
||||||
|
REVIEW_UPLOAD_RETENTION_SECONDS = 86400
|
||||||
|
|
||||||
|
|
||||||
class RunManager:
|
class RunManager:
|
||||||
@@ -117,6 +118,41 @@ class RunManager:
|
|||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def review_upload_paths(self, run: Run, task: Task) -> dict[str, Path] | None:
|
||||||
|
"""Return reusable uploads, or None when a run cannot be executed again."""
|
||||||
|
if run.status != "succeeded" or run.target_kind != "task" or task.execution is None:
|
||||||
|
return None
|
||||||
|
if any(field.sensitive for field in task.inputs):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
values = json.loads(run.input_json)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return None
|
||||||
|
if not isinstance(values, dict) or values.get(task.execution.field.name) != task.execution.dry_run_value:
|
||||||
|
return None
|
||||||
|
root = self._upload_root(run.id)
|
||||||
|
paths = {}
|
||||||
|
for field in task.inputs:
|
||||||
|
if field.type != "file":
|
||||||
|
continue
|
||||||
|
value = values.get(field.name)
|
||||||
|
if not value and not field.required:
|
||||||
|
continue
|
||||||
|
if not isinstance(value, str) or not run.finished_at:
|
||||||
|
return None
|
||||||
|
if time.time() - datetime.fromisoformat(run.finished_at).timestamp() >= REVIEW_UPLOAD_RETENTION_SECONDS:
|
||||||
|
return None
|
||||||
|
path = Path(value)
|
||||||
|
if root.is_symlink() or path.parent != root or path.is_symlink() or not path.is_file():
|
||||||
|
return None
|
||||||
|
if path.name != field.name + path.suffix or (field.accept and path.suffix.lower() not in field.accept):
|
||||||
|
return None
|
||||||
|
paths[field.name] = path
|
||||||
|
return paths
|
||||||
|
|
||||||
|
def discard_review_uploads(self, run_id: int) -> None:
|
||||||
|
self._delete_uploads(run_id)
|
||||||
|
|
||||||
def cleanup_transient_files(
|
def cleanup_transient_files(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -144,6 +180,9 @@ class RunManager:
|
|||||||
if kind in {"input.json", "execution-input.json"} and path.is_file():
|
if kind in {"input.json", "execution-input.json"} and path.is_file():
|
||||||
path.unlink(missing_ok=True)
|
path.unlink(missing_ok=True)
|
||||||
elif kind == "uploads" and path.is_dir():
|
elif kind == "uploads" and path.is_dir():
|
||||||
|
run = self.store.get(run_id)
|
||||||
|
task = self.catalog.task(run.target_path) if run and run.target_kind == "task" else None
|
||||||
|
if task is None or not self.review_upload_paths(run, task):
|
||||||
shutil.rmtree(path, ignore_errors=True)
|
shutil.rmtree(path, ignore_errors=True)
|
||||||
elif kind == "artifacts" and path.is_dir():
|
elif kind == "artifacts" and path.is_dir():
|
||||||
if status in {"failed", "interrupted", "cancelled"}:
|
if status in {"failed", "interrupted", "cancelled"}:
|
||||||
@@ -370,13 +409,13 @@ class RunManager:
|
|||||||
finally:
|
finally:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._processes.pop(run.id, None)
|
self._processes.pop(run.id, None)
|
||||||
self._delete_uploads(run.id)
|
|
||||||
self._delete_execution_input(run.id)
|
self._delete_execution_input(run.id)
|
||||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
||||||
|
|
||||||
if self.store.is_cancel_requested(run.id) or (
|
if self.store.is_cancel_requested(run.id) or (
|
||||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||||
):
|
):
|
||||||
|
self._delete_uploads(run.id)
|
||||||
self._keep_diagnostic_reports(artifact_root)
|
self._keep_diagnostic_reports(artifact_root)
|
||||||
return self.store.finish(
|
return self.store.finish(
|
||||||
run.id,
|
run.id,
|
||||||
@@ -390,6 +429,8 @@ class RunManager:
|
|||||||
exit_code=exit_code,
|
exit_code=exit_code,
|
||||||
log_path=str(log_path),
|
log_path=str(log_path),
|
||||||
)
|
)
|
||||||
|
if not self.review_upload_paths(completed, task):
|
||||||
|
self._delete_uploads(run.id)
|
||||||
if completed.status != "succeeded":
|
if completed.status != "succeeded":
|
||||||
self._keep_diagnostic_reports(artifact_root)
|
self._keep_diagnostic_reports(artifact_root)
|
||||||
else:
|
else:
|
||||||
|
|||||||
+21
-12
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
import codecs
|
import codecs
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
|
from contextlib import ExitStack
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -12,7 +13,7 @@ from flask import Flask, abort, redirect, render_template, request, send_file, u
|
|||||||
|
|
||||||
from .config import Settings
|
from .config import Settings
|
||||||
from .discovery import TaskCatalog
|
from .discovery import TaskCatalog
|
||||||
from .inputs import validate_inputs, validate_uploads
|
from .inputs import PendingUpload, validate_inputs, validate_uploads
|
||||||
from .runner import RunManager
|
from .runner import RunManager
|
||||||
from .scheduler import BackgroundScheduler
|
from .scheduler import BackgroundScheduler
|
||||||
from .store import RunStore
|
from .store import RunStore
|
||||||
@@ -154,15 +155,7 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
task = catalog.task(run.target_path)
|
task = catalog.task(run.target_path)
|
||||||
if task is None or task.execution is None:
|
if task is None or task.execution is None:
|
||||||
return None
|
return None
|
||||||
if any(field.type == "file" or field.sensitive for field in task.inputs):
|
if manager.review_upload_paths(run, task) is None:
|
||||||
return None
|
|
||||||
try:
|
|
||||||
input_data = json.loads(run.input_json)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
return None
|
|
||||||
if not isinstance(input_data, dict):
|
|
||||||
return None
|
|
||||||
if input_data.get(task.execution.field.name) != task.execution.dry_run_value:
|
|
||||||
return None
|
return None
|
||||||
return task
|
return task
|
||||||
|
|
||||||
@@ -177,13 +170,29 @@ def create_app(settings: Settings | None = None) -> Flask:
|
|||||||
input_data = json.loads(run.input_json)
|
input_data = json.loads(run.input_json)
|
||||||
input_data[task.execution.field.name] = task.execution.execute_value
|
input_data[task.execution.field.name] = task.execution.execute_value
|
||||||
raw_values = {
|
raw_values = {
|
||||||
name: [str(value) for value in value] if isinstance(value, list) else [str(value)]
|
name: ([str(item) for item in value] or [""]) if isinstance(value, list) else [str(value)]
|
||||||
for name, value in input_data.items()
|
for name, value in input_data.items()
|
||||||
}
|
}
|
||||||
validated, errors = validate_inputs(task.inputs, raw_values)
|
validated, errors = validate_inputs(task.inputs, raw_values)
|
||||||
if errors:
|
if errors:
|
||||||
abort(409)
|
abort(409)
|
||||||
executed, _ = manager.run_task(task, validated)
|
paths = manager.review_upload_paths(run, task)
|
||||||
|
if paths is None:
|
||||||
|
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
|
||||||
|
with ExitStack() as stack:
|
||||||
|
uploads = {}
|
||||||
|
for field in task.inputs:
|
||||||
|
if field.name not in paths:
|
||||||
|
continue
|
||||||
|
path = paths[field.name]
|
||||||
|
try:
|
||||||
|
stream = stack.enter_context(path.open("rb"))
|
||||||
|
except OSError:
|
||||||
|
abort(409, description="The reviewed upload is no longer available. Upload the file and review again.")
|
||||||
|
uploads[field.name] = PendingUpload(field.name, path.suffix, field.maximum_bytes, stream)
|
||||||
|
executed, created = manager.run_task(task, validated, uploads)
|
||||||
|
if created and executed.status == "queued":
|
||||||
|
manager.discard_review_uploads(run.id)
|
||||||
return redirect(url_for("run_detail", run_id=executed.id))
|
return redirect(url_for("run_detail", run_id=executed.id))
|
||||||
|
|
||||||
@app.post("/runs/<int:run_id>/cancel")
|
@app.post("/runs/<int:run_id>/cancel")
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.web import create_app
|
||||||
|
|
||||||
|
from .conftest import write_task
|
||||||
|
from .test_runner import wait_for
|
||||||
|
|
||||||
|
|
||||||
|
def start_review(settings, *, sensitive=False, fail=False):
|
||||||
|
task = write_task(settings.task_root, "imports", "apply.py", '''
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||||
|
print(json.dumps({"mode": values["mode"], "csv": Path(values["csv"]).read_text()}))
|
||||||
|
''' + ("raise SystemExit(1)\n" if fail else ""))
|
||||||
|
(task.parent / "task-inputs.json").write_text(json.dumps({
|
||||||
|
"version": 1,
|
||||||
|
"tasks": {"apply.py": {
|
||||||
|
"inputs": [
|
||||||
|
{"name": "csv", "label": "CSV", "type": "file", "required": True,
|
||||||
|
"accept": [".csv"], "maximum_bytes": 1024, "sensitive": sensitive},
|
||||||
|
{"name": "choices", "label": "Choices", "type": "multi_choice",
|
||||||
|
"options": ["one"], "default": ["one"]},
|
||||||
|
],
|
||||||
|
"execution": {"field": "mode", "dry_run_value": "review", "dry_run_label": "Review",
|
||||||
|
"execute_value": "apply", "execute_label": "Execute"},
|
||||||
|
}},
|
||||||
|
}))
|
||||||
|
app = create_app(settings)
|
||||||
|
client = app.test_client()
|
||||||
|
response = client.post("/tasks/imports/apply.py/run", data={
|
||||||
|
"csv": (BytesIO(b"answer\nsame\n"), "decisions.csv"), "choices": "",
|
||||||
|
})
|
||||||
|
run_id = int(response.location.rsplit("/", 1)[1])
|
||||||
|
manager = app.extensions["run_manager"]
|
||||||
|
assert wait_for(manager, run_id, {"failed" if fail else "succeeded"}) == ("failed" if fail else "succeeded")
|
||||||
|
manager.cleanup_transient_files()
|
||||||
|
return app, client, run_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_uploaded_review_can_execute_with_the_same_csv_and_keeps_its_output(settings):
|
||||||
|
app, client, review_id = start_review(settings)
|
||||||
|
store = app.extensions["run_store"]
|
||||||
|
original = Path(json.loads(store.get(review_id).input_json)["csv"])
|
||||||
|
assert original.is_file()
|
||||||
|
assert original.stat().st_mode & 0o777 == 0o600
|
||||||
|
assert f'action="/runs/{review_id}/execute"'.encode() in client.get(f"/runs/{review_id}").data
|
||||||
|
|
||||||
|
response = client.post(f"/runs/{review_id}/execute")
|
||||||
|
assert response.status_code == 302
|
||||||
|
executed_id = int(response.location.rsplit("/", 1)[1])
|
||||||
|
assert executed_id != review_id
|
||||||
|
assert wait_for(app.extensions["run_manager"], executed_id, {"succeeded"}) == "succeeded"
|
||||||
|
app.extensions["run_manager"].cleanup_transient_files()
|
||||||
|
|
||||||
|
review_output = json.loads(client.get(f"/runs/{review_id}/output").get_json()["output"])
|
||||||
|
execute_output = json.loads(client.get(f"/runs/{executed_id}/output").get_json()["output"])
|
||||||
|
assert review_output == {"mode": "review", "csv": "answer\nsame\n"}
|
||||||
|
assert execute_output == {"mode": "apply", "csv": "answer\nsame\n"}
|
||||||
|
assert json.loads(store.get(executed_id).input_json)["choices"] == []
|
||||||
|
assert not original.parent.exists()
|
||||||
|
assert not Path(json.loads(store.get(executed_id).input_json)["csv"]).parent.exists()
|
||||||
|
assert client.post(f"/runs/{review_id}/execute").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_review_upload_survives_application_restart(settings):
|
||||||
|
app, _, run_id = start_review(settings)
|
||||||
|
original = Path(json.loads(app.extensions["run_store"].get(run_id).input_json)["csv"])
|
||||||
|
restarted = create_app(settings)
|
||||||
|
assert original.is_file()
|
||||||
|
assert f'action="/runs/{run_id}/execute"'.encode() in restarted.test_client().get(f"/runs/{run_id}").data
|
||||||
|
|
||||||
|
|
||||||
|
def test_unused_review_upload_expires_after_one_day(settings, monkeypatch):
|
||||||
|
app, client, run_id = start_review(settings)
|
||||||
|
original = Path(json.loads(app.extensions["run_store"].get(run_id).input_json)["csv"])
|
||||||
|
future = time.time() + 86401
|
||||||
|
monkeypatch.setattr("app.runner.time.time", lambda: future)
|
||||||
|
assert client.post(f"/runs/{run_id}/execute").status_code == 404
|
||||||
|
app.extensions["run_manager"].cleanup_transient_files()
|
||||||
|
assert not original.parent.exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("sensitive,fail", [(True, False), (False, True)])
|
||||||
|
def test_sensitive_or_failed_review_does_not_retain_uploads(settings, sensitive, fail):
|
||||||
|
_, client, run_id = start_review(settings, sensitive=sensitive, fail=fail)
|
||||||
|
assert not (settings.state_root / "runs" / f"{run_id}.uploads").exists()
|
||||||
|
assert client.post(f"/runs/{run_id}/execute").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("invalid_upload", ["missing", "symlink", "outside"])
|
||||||
|
def test_missing_or_unsafe_review_upload_cannot_execute(settings, invalid_upload):
|
||||||
|
app, client, run_id = start_review(settings)
|
||||||
|
store = app.extensions["run_store"]
|
||||||
|
values = json.loads(store.get(run_id).input_json)
|
||||||
|
original = Path(values["csv"])
|
||||||
|
original.unlink()
|
||||||
|
if invalid_upload != "missing":
|
||||||
|
outside = settings.state_root / "other.csv"
|
||||||
|
outside.write_text("not the reviewed upload")
|
||||||
|
if invalid_upload == "symlink":
|
||||||
|
original.symlink_to(outside)
|
||||||
|
else:
|
||||||
|
values["csv"] = str(outside)
|
||||||
|
store.set_input(run_id, values)
|
||||||
|
assert client.post(f"/runs/{run_id}/execute").status_code == 404
|
||||||
|
assert f'action="/runs/{run_id}/execute"'.encode() not in client.get(f"/runs/{run_id}").data
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_execution_enqueue_keeps_original_review_upload(settings, monkeypatch):
|
||||||
|
app, client, run_id = start_review(settings)
|
||||||
|
manager = app.extensions["run_manager"]
|
||||||
|
original = Path(json.loads(manager.store.get(run_id).input_json)["csv"])
|
||||||
|
|
||||||
|
def cannot_save(*_):
|
||||||
|
raise OSError("test: disk full")
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager, "_save_uploads", cannot_save)
|
||||||
|
response = client.post(f"/runs/{run_id}/execute")
|
||||||
|
failed_id = int(response.location.rsplit("/", 1)[1])
|
||||||
|
assert manager.store.get(failed_id).status == "failed"
|
||||||
|
assert original.is_file()
|
||||||
|
assert f'action="/runs/{run_id}/execute"'.encode() in client.get(f"/runs/{run_id}").data
|
||||||
|
|
||||||
|
|
||||||
|
def test_removing_file_input_from_task_cleans_up_unused_review_upload(settings):
|
||||||
|
app, _, run_id = start_review(settings)
|
||||||
|
original = Path(json.loads(app.extensions["run_store"].get(run_id).input_json)["csv"])
|
||||||
|
manifest = settings.task_root / "imports" / "task-inputs.json"
|
||||||
|
definition = json.loads(manifest.read_text())
|
||||||
|
definition["tasks"]["apply.py"]["inputs"] = []
|
||||||
|
manifest.write_text(json.dumps(definition))
|
||||||
|
app.extensions["run_manager"].cleanup_transient_files()
|
||||||
|
assert not original.parent.exists()
|
||||||
Reference in new issue
Block a user