Simplify task input handling and tighten runner coordination
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8
context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; }
name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18
POSTGIS_VERSION=3
VCHORD_VERSION=0.5.3
context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3
context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim
context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; }
name:timepoll oci_labels:org.opencontai… (push) Successful in 42s
This commit is contained in:
1 parent
3bfd199b9c
commit
1c713ff635
7 files changed
+130
-67
No files matched your search
@@ -103,7 +103,7 @@ Every input has `name`, `label`, and `type`; `required` defaults to `false`.
|
|||||||
|
|
||||||
Set `sensitive` to `true` on text or file inputs that must not appear in run
|
Set `sensitive` to `true` on text or file inputs that must not appear in run
|
||||||
history. Text is masked, the database records `[redacted]`, and the original
|
history. Text is masked, the database records `[redacted]`, and the original
|
||||||
value is available only through a private execution-input file. Interrupted
|
value is available only through the run's private work-directory input file. Interrupted
|
||||||
recoverable tasks retain that input and their uploads until recovery finishes.
|
recoverable tasks retain that input and their uploads until recovery finishes.
|
||||||
|
|
||||||
For review-first tasks, use `execution`. It produces one split button: click
|
For review-first tasks, use `execution`. It produces one split button: click
|
||||||
@@ -219,13 +219,14 @@ For simple item-count percentages and per-step estimates:
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20})
|
tracker = runtime.Progress({"Scanning files": 100, "Updating records": 20})
|
||||||
tracker.update("Scanning files", 0, force=True)
|
tracker.update("Scanning files", 0)
|
||||||
# After processing five files:
|
# After processing five files:
|
||||||
tracker.update("Scanning files", 5)
|
tracker.update("Scanning files", 5)
|
||||||
```
|
```
|
||||||
|
|
||||||
Each step has its own percentage and an approximate ETA based only on processed
|
Each step shows completed/total items, a percentage and an approximate ETA based
|
||||||
items and elapsed time. Pending steps have no ETA. Tasks may instead publish their
|
on items processed since starting or resuming. Stage starts and completion are
|
||||||
|
published immediately. Pending steps have no ETA. Tasks may instead publish their
|
||||||
own text when totals are unknown; no global estimate is required.
|
own text when totals are unknown; no global estimate is required.
|
||||||
|
|
||||||
A group may also include a private `task-config.json` for stable local details
|
A group may also include a private `task-config.json` for stable local details
|
||||||
|
|||||||
+35
-56
@@ -50,6 +50,7 @@ class RunManager:
|
|||||||
self._processes: dict[int, subprocess.Popen[bytes]] = {}
|
self._processes: dict[int, subprocess.Popen[bytes]] = {}
|
||||||
self._background_processes: set[subprocess.Popen[bytes]] = set()
|
self._background_processes: set[subprocess.Popen[bytes]] = set()
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
|
self._cleanup_lock = threading.Lock()
|
||||||
self._closing = threading.Event()
|
self._closing = threading.Event()
|
||||||
|
|
||||||
def run_task(
|
def run_task(
|
||||||
@@ -58,8 +59,6 @@ class RunManager:
|
|||||||
input_data: dict[str, str | int | list[str]] | None = None,
|
input_data: dict[str, str | int | list[str]] | None = None,
|
||||||
uploads: Mapping[str, PendingUpload] | None = None,
|
uploads: Mapping[str, PendingUpload] | None = None,
|
||||||
) -> tuple[Run, bool]:
|
) -> tuple[Run, bool]:
|
||||||
if self._closing.is_set():
|
|
||||||
raise RuntimeError("The application is shutting down; try again after it restarts.")
|
|
||||||
sensitive_names = {field.name for field in task.inputs if field.sensitive}
|
sensitive_names = {field.name for field in task.inputs if field.sensitive}
|
||||||
stored_input = self._redact_input(input_data or {}, sensitive_names)
|
stored_input = self._redact_input(input_data or {}, sensitive_names)
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -72,33 +71,28 @@ class RunManager:
|
|||||||
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
target_kind="task", target_path=task.id, queue_group=task.group_id,
|
||||||
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
|
input_data=stored_input, resources=task.resources or (f"group:{task.group_id}",),
|
||||||
)
|
)
|
||||||
|
if created:
|
||||||
|
try:
|
||||||
|
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
||||||
|
execution_input = {**(input_data or {}), **uploaded_inputs}
|
||||||
|
run = self.store.set_input(run.id, self._redact_input(execution_input, sensitive_names))
|
||||||
|
self._save_work_input(run, execution_input)
|
||||||
|
except (OSError, ValueError) as error:
|
||||||
|
self._delete_uploads(run.id)
|
||||||
|
self._discard_work_input(run)
|
||||||
|
return self.store.finish(
|
||||||
|
run.id, status="failed", message=f"Could not save task input: {error}",
|
||||||
|
), True
|
||||||
if created:
|
if created:
|
||||||
try:
|
|
||||||
uploaded_inputs = self._save_uploads(run.id, uploads or {})
|
|
||||||
execution_input = {**(input_data or {}), **uploaded_inputs}
|
|
||||||
stored_input = self._redact_input(execution_input, sensitive_names)
|
|
||||||
run = self.store.set_input(run.id, stored_input)
|
|
||||||
if sensitive_names:
|
|
||||||
self._write_execution_input(run.id, execution_input)
|
|
||||||
self._save_work_input(run, execution_input)
|
|
||||||
except (OSError, ValueError) as error:
|
|
||||||
self._delete_uploads(run.id)
|
|
||||||
self._delete_execution_input(run.id)
|
|
||||||
self._discard_work_input(run)
|
|
||||||
return self.store.finish(
|
|
||||||
run.id,
|
|
||||||
status="failed",
|
|
||||||
message=f"Could not save uploaded file: {error}",
|
|
||||||
), True
|
|
||||||
self._wake_worker(task.group_id)
|
self._wake_worker(task.group_id)
|
||||||
return run, created
|
return run, created
|
||||||
|
|
||||||
def run_group(self, group: Group) -> tuple[Run, bool]:
|
def run_group(self, group: Group) -> tuple[Run, bool]:
|
||||||
if self._closing.is_set():
|
|
||||||
raise RuntimeError("The application is shutting down.")
|
|
||||||
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
|
resources = tuple(dict.fromkeys(resource for task in group.tasks for resource in task.resources))
|
||||||
resources = resources or (f"group:{group.id}",)
|
resources = resources or (f"group:{group.id}",)
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
if self._closing.is_set():
|
||||||
|
raise RuntimeError("The application is shutting down.")
|
||||||
for active in self.store.active_runs():
|
for active in self.store.active_runs():
|
||||||
if active.target_kind == "group" and active.target_path == group.id:
|
if active.target_kind == "group" and active.target_path == group.id:
|
||||||
return active, False
|
return active, False
|
||||||
@@ -152,10 +146,10 @@ class RunManager:
|
|||||||
|
|
||||||
def shutdown(self, timeout: float | None = None) -> None:
|
def shutdown(self, timeout: float | None = None) -> None:
|
||||||
self._closing.set()
|
self._closing.set()
|
||||||
for event in self._wake_events.values():
|
|
||||||
event.set()
|
|
||||||
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
|
deadline = time.monotonic() + (self.settings.shutdown_grace_seconds if timeout is None else timeout)
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
for event in self._wake_events.values():
|
||||||
|
event.set()
|
||||||
processes = [*self._processes.values(), *self._background_processes]
|
processes = [*self._processes.values(), *self._background_processes]
|
||||||
for process in processes:
|
for process in processes:
|
||||||
if process.poll() is None:
|
if process.poll() is None:
|
||||||
@@ -199,7 +193,6 @@ class RunManager:
|
|||||||
completed = self.store.get(run.id)
|
completed = self.store.get(run.id)
|
||||||
if completed is not None and completed.status == "cancelled":
|
if completed is not None and completed.status == "cancelled":
|
||||||
self._delete_uploads(run.id)
|
self._delete_uploads(run.id)
|
||||||
self._delete_execution_input(run.id)
|
|
||||||
self._discard_work_input(completed)
|
self._discard_work_input(completed)
|
||||||
return completed
|
return completed
|
||||||
|
|
||||||
@@ -271,7 +264,10 @@ class RunManager:
|
|||||||
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
|
orphaned_task_state_retention_seconds: int = ORPHANED_TASK_STATE_RETENTION_SECONDS,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Remove transient run data and state for task groups removed long ago."""
|
"""Remove transient run data and state for task groups removed long ago."""
|
||||||
|
with self._cleanup_lock:
|
||||||
|
self._cleanup_transient_files(artifact_max_age_seconds, orphaned_task_state_retention_seconds)
|
||||||
|
|
||||||
|
def _cleanup_transient_files(self, artifact_max_age_seconds: int, orphaned_task_state_retention_seconds: int) -> None:
|
||||||
runs_root = self.settings.state_root / "runs"
|
runs_root = self.settings.state_root / "runs"
|
||||||
cutoff = time.time() - artifact_max_age_seconds
|
cutoff = time.time() - artifact_max_age_seconds
|
||||||
if runs_root.is_dir():
|
if runs_root.is_dir():
|
||||||
@@ -438,6 +434,8 @@ class RunManager:
|
|||||||
|
|
||||||
def _wake_worker(self, group_id: str) -> None:
|
def _wake_worker(self, group_id: str) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
if self._closing.is_set():
|
||||||
|
return
|
||||||
event = self._wake_events.setdefault(group_id, threading.Event())
|
event = self._wake_events.setdefault(group_id, threading.Event())
|
||||||
worker = self._workers.get(group_id)
|
worker = self._workers.get(group_id)
|
||||||
if worker is None or not worker.is_alive():
|
if worker is None or not worker.is_alive():
|
||||||
@@ -454,15 +452,20 @@ class RunManager:
|
|||||||
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
|
def _worker_loop(self, group_id: str, event: threading.Event) -> None:
|
||||||
while not self._closing.is_set():
|
while not self._closing.is_set():
|
||||||
event.clear()
|
event.clear()
|
||||||
run = self.store.claim_next(group_id)
|
# A queued run is claimable only after its private input and uploads are saved.
|
||||||
|
with self._lock:
|
||||||
|
if self._closing.is_set():
|
||||||
|
return
|
||||||
|
run = self.store.claim_next(group_id)
|
||||||
if run is None:
|
if run is None:
|
||||||
event.wait()
|
event.wait()
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
self._execute_root_run(run)
|
self._execute_root_run(run)
|
||||||
finally:
|
finally:
|
||||||
for waiting in self._wake_events.values():
|
with self._lock:
|
||||||
waiting.set()
|
for waiting in self._wake_events.values():
|
||||||
|
waiting.set()
|
||||||
|
|
||||||
def _execute_root_run(self, run: Run) -> None:
|
def _execute_root_run(self, run: Run) -> None:
|
||||||
if run.target_kind == "task":
|
if run.target_kind == "task":
|
||||||
@@ -529,23 +532,17 @@ class RunManager:
|
|||||||
|
|
||||||
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
|
log_path = self.settings.state_root / "runs" / f"{run.id}.log"
|
||||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
input_path = self._execution_input_path(run.id)
|
|
||||||
artifact_root = self._artifact_root(run.id)
|
artifact_root = self._artifact_root(run.id)
|
||||||
artifact_root.mkdir(parents=True, exist_ok=True)
|
artifact_root.mkdir(parents=True, exist_ok=True)
|
||||||
try:
|
try:
|
||||||
work = self._work_root(run)
|
work = self._work_root(run)
|
||||||
saved_input = work / "input.json"
|
input_path = work / "input.json"
|
||||||
if run.recovery_of_run_id is not None:
|
if not input_path.is_file():
|
||||||
input_path = saved_input
|
if run.recovery_of_run_id is not None:
|
||||||
elif not input_path.exists():
|
raise FileNotFoundError("The original recovery input is no longer available")
|
||||||
input_path = self.settings.state_root / "runs" / f"{run.id}.input.json"
|
self._save_work_input(run, json.loads(run.input_json))
|
||||||
input_path.write_text(run.input_json + "\n", encoding="utf-8")
|
|
||||||
input_path.chmod(0o600)
|
|
||||||
if not saved_input.exists():
|
|
||||||
self._save_work_input(run, json.loads(input_path.read_text()))
|
|
||||||
except OSError as error:
|
except OSError as error:
|
||||||
self._delete_uploads(run.id)
|
self._delete_uploads(run.id)
|
||||||
self._delete_execution_input(run.id)
|
|
||||||
self._discard_work_input(run)
|
self._discard_work_input(run)
|
||||||
shutil.rmtree(artifact_root, ignore_errors=True)
|
shutil.rmtree(artifact_root, ignore_errors=True)
|
||||||
return self.store.finish(
|
return self.store.finish(
|
||||||
@@ -595,8 +592,6 @@ class RunManager:
|
|||||||
owned = self._processes.pop(run.id, None)
|
owned = self._processes.pop(run.id, None)
|
||||||
if owned is not None:
|
if owned is not None:
|
||||||
kill_remaining_group(owned)
|
kill_remaining_group(owned)
|
||||||
self._delete_execution_input(run.id)
|
|
||||||
(self.settings.state_root / "runs" / f"{run.id}.input.json").unlink(missing_ok=True)
|
|
||||||
|
|
||||||
if self.store.is_cancel_requested(run.id) or (
|
if self.store.is_cancel_requested(run.id) or (
|
||||||
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
parent_run_id is not None and self.store.is_cancel_requested(parent_run_id)
|
||||||
@@ -659,22 +654,6 @@ class RunManager:
|
|||||||
def _artifact_root(self, run_id: int) -> Path:
|
def _artifact_root(self, run_id: int) -> Path:
|
||||||
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
|
return self.settings.state_root / "runs" / f"{run_id}.artifacts"
|
||||||
|
|
||||||
def _execution_input_path(self, run_id: int) -> Path:
|
|
||||||
return self.settings.state_root / "runs" / f"{run_id}.execution-input.json"
|
|
||||||
|
|
||||||
def _write_execution_input(
|
|
||||||
self,
|
|
||||||
run_id: int,
|
|
||||||
input_data: Mapping[str, str | int | list[str]],
|
|
||||||
) -> None:
|
|
||||||
path = self._execution_input_path(run_id)
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(input_data, sort_keys=True, separators=(",", ":")) + "\n", encoding="utf-8")
|
|
||||||
path.chmod(0o600)
|
|
||||||
|
|
||||||
def _delete_execution_input(self, run_id: int) -> None:
|
|
||||||
self._execution_input_path(run_id).unlink(missing_ok=True)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _redact_input(
|
def _redact_input(
|
||||||
input_data: Mapping[str, str | int | list[str]],
|
input_data: Mapping[str, str | int | list[str]],
|
||||||
|
|||||||
@@ -144,18 +144,21 @@ class Progress:
|
|||||||
self.baseline: dict[str, int] = {}
|
self.baseline: dict[str, int] = {}
|
||||||
|
|
||||||
def update(self, label: str, done: int, *, force: bool = False) -> None:
|
def update(self, label: str, done: int, *, force: bool = False) -> None:
|
||||||
|
force = force or label not in self.started or done >= self.totals[label]
|
||||||
self.started.setdefault(label, time.monotonic())
|
self.started.setdefault(label, time.monotonic())
|
||||||
self.baseline.setdefault(label, done)
|
self.baseline.setdefault(label, done)
|
||||||
self.done[label] = done
|
self.done[label] = done
|
||||||
lines = []
|
lines = []
|
||||||
for name, total in self.totals.items():
|
for name, total in self.totals.items():
|
||||||
count = self.done[name]
|
count = self.done[name]
|
||||||
percent = min(100, int(count * 100 / total)) if total else (100 if name in self.started else 0)
|
if total:
|
||||||
line = f"{name}: {percent}%"
|
line = f"{name}: {count}/{total} ({min(100, int(count * 100 / total))}%)"
|
||||||
|
else:
|
||||||
|
line = f"{name}: {'no items' if name in self.started else 'waiting'}"
|
||||||
processed = count - self.baseline.get(name, 0)
|
processed = count - self.baseline.get(name, 0)
|
||||||
if processed > 0 and count < total and name in self.started:
|
if processed > 0 and count < total and name in self.started:
|
||||||
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
|
seconds = max(0, round((time.monotonic() - self.started[name]) / processed * (total - count)))
|
||||||
estimate = f"{seconds} seconds" if seconds < 60 else f"{max(1, round(seconds / 60))} minutes"
|
value, unit = (seconds, "second") if seconds < 60 else (round(seconds / 60), "minute")
|
||||||
line += f", ETA {estimate}"
|
line += f", ETA {value} {unit}{'' if value == 1 else 's'}"
|
||||||
lines.append(line)
|
lines.append(line)
|
||||||
progress("\n".join(lines), force=force)
|
progress("\n".join(lines), force=force)
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -379,3 +381,38 @@ def test_upload_is_removed_when_task_process_cannot_start(manager: RunManager) -
|
|||||||
assert created is True
|
assert created is True
|
||||||
assert wait_for(broken_manager, run.id, {"failed"}) == "failed"
|
assert wait_for(broken_manager, run.id, {"failed"}) == "failed"
|
||||||
assert not (broken_manager.settings.state_root / "runs" / f"{run.id}.uploads").exists()
|
assert not (broken_manager.settings.state_root / "runs" / f"{run.id}.uploads").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_idle_worker_cannot_start_before_uploaded_input_is_ready(manager, monkeypatch):
|
||||||
|
write_task(manager.settings.task_root, "imports", "read.py", '''
|
||||||
|
import json, os
|
||||||
|
from pathlib import Path
|
||||||
|
values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
||||||
|
assert Path(values["document"]).read_text() == "uploaded content"
|
||||||
|
''')
|
||||||
|
task = manager.catalog.task("imports/read.py")
|
||||||
|
manager._wake_worker(task.group_id)
|
||||||
|
saving = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
save_uploads = manager._save_uploads
|
||||||
|
|
||||||
|
def slow_upload(run_id, uploads):
|
||||||
|
saving.set()
|
||||||
|
assert release.wait(5)
|
||||||
|
return save_uploads(run_id, uploads)
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager, '_save_uploads', slow_upload)
|
||||||
|
upload = PendingUpload('document', '.txt', 1024, BytesIO(b'uploaded content'))
|
||||||
|
with ThreadPoolExecutor() as pool:
|
||||||
|
submitted = pool.submit(manager.run_task, task, uploads={'document': upload})
|
||||||
|
try:
|
||||||
|
assert saving.wait(5)
|
||||||
|
run = manager.store.active_runs()[0]
|
||||||
|
manager._wake_events[task.group_id].set()
|
||||||
|
time.sleep(.1)
|
||||||
|
assert manager.store.get(run.id).status == 'queued'
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
run, created = submitted.result(timeout=5)
|
||||||
|
assert created
|
||||||
|
assert wait_for(manager, run.id, {'succeeded', 'failed'}) == 'succeeded'
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from app.web import create_app
|
from app.web import create_app
|
||||||
@@ -8,6 +10,32 @@ from .conftest import write_task
|
|||||||
from .test_runner import wait_for
|
from .test_runner import wait_for
|
||||||
|
|
||||||
|
|
||||||
|
def test_concurrent_cleanup_passes_do_not_overlap(manager, monkeypatch):
|
||||||
|
entered = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def cleanup(*args):
|
||||||
|
calls.append(args)
|
||||||
|
if len(calls) == 1:
|
||||||
|
entered.set()
|
||||||
|
assert release.wait(5)
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager, '_cleanup_transient_files', cleanup)
|
||||||
|
with ThreadPoolExecutor() as pool:
|
||||||
|
first = pool.submit(manager.cleanup_transient_files)
|
||||||
|
assert entered.wait(5)
|
||||||
|
second = pool.submit(manager.cleanup_transient_files)
|
||||||
|
try:
|
||||||
|
time.sleep(.1)
|
||||||
|
assert len(calls) == 1
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
first.result(timeout=5)
|
||||||
|
second.result(timeout=5)
|
||||||
|
assert len(calls) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
|
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
|
||||||
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
|
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
|
||||||
manager.store.claim_next("example")
|
manager.store.claim_next("example")
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
|
|||||||
clock[0]+=30
|
clock[0]+=30
|
||||||
progress.update('Removing files',5,force=True)
|
progress.update('Removing files',5,force=True)
|
||||||
text=json.loads((work/'progress.json').read_text())['text']
|
text=json.loads((work/'progress.json').read_text())['text']
|
||||||
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
|
assert text=='Removing files: 5/10 (50%), ETA 30 seconds\nLinking files: 0/20 (0%)'
|
||||||
assert (work/'progress.json').stat().st_mode&0o777==0o600
|
assert (work/'progress.json').stat().st_mode&0o777==0o600
|
||||||
|
|
||||||
|
|
||||||
@@ -78,4 +78,18 @@ def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
|
|||||||
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
|
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
|
||||||
clock[0]+=60
|
clock[0]+=60
|
||||||
progress.update('Items',40,force=True)
|
progress.update('Items',40,force=True)
|
||||||
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
|
assert json.loads((work/'progress.json').read_text())['text']=='Items: 40/50 (80%), ETA 10 minutes'
|
||||||
|
|
||||||
|
|
||||||
|
def test_stage_start_and_completion_bypass_progress_throttle(work, monkeypatch):
|
||||||
|
monkeypatch.setattr(runtime.time, 'monotonic', lambda: 100.)
|
||||||
|
progress = runtime.Progress({'Items': 2, 'Empty': 0})
|
||||||
|
progress.update('Items', 0)
|
||||||
|
assert json.loads((work / 'progress.json').read_text())['text'] == (
|
||||||
|
'Items: 0/2 (0%)\nEmpty: waiting'
|
||||||
|
)
|
||||||
|
progress.update('Items', 1)
|
||||||
|
progress.update('Items', 2)
|
||||||
|
assert json.loads((work / 'progress.json').read_text())['text'].startswith('Items: 2/2 (100%)')
|
||||||
|
progress.update('Empty', 0)
|
||||||
|
assert json.loads((work / 'progress.json').read_text())['text'].endswith('Empty: no items')
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
import json
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from app.web import create_app
|
from app.web import create_app
|
||||||
@@ -254,7 +255,7 @@ print(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
|
|||||||
assert executed.input_json == '{"mode":"apply"}'
|
assert executed.input_json == '{"mode":"apply"}'
|
||||||
assert executed_id != run_id
|
assert executed_id != run_id
|
||||||
assert app.extensions["run_store"].get(run_id).input_json == '{"mode":"report"}'
|
assert app.extensions["run_store"].get(run_id).input_json == '{"mode":"report"}'
|
||||||
assert '"mode":"report"' in client.get(f"/runs/{run_id}/output").get_json()["output"]
|
assert json.loads(client.get(f"/runs/{run_id}/output").get_json()["output"]) == {"mode": "report"}
|
||||||
executed_detail = client.get(f"/runs/{executed_id}")
|
executed_detail = client.get(f"/runs/{executed_id}")
|
||||||
assert b'href="/history/task/examples/migrate.py"' in executed_detail.data
|
assert b'href="/history/task/examples/migrate.py"' in executed_detail.data
|
||||||
history = client.get("/history/task/examples/migrate.py")
|
history = client.get("/history/task/examples/migrate.py")
|
||||||
|
|||||||
Reference in new issue
Block a user