Simplify task input handling and tighten runner coordination
Build custom container images / build (map[base_image:php:8-fpm-alpine build_args:PHP_VERSION=8 context:php8-pgsql fingerprint_command:{ apk info -v | LC_ALL=C sort; find /usr/local/lib/php/extensions /usr/local/etc/php/conf.d -type f -exec sha256sum {} + | LC_ALL=C sort; } name:ph… (push) Successful in 50s
Build custom container images / build (map[base_image:postgres:18 build_args:PG_VERSION=18 POSTGIS_VERSION=3 VCHORD_VERSION=0.5.3 context:postgres fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; find /usr/lib/postgresql -type f -exec sha256su… (push) Successful in 50s
Build custom container images / build (map[base_image:python:3 build_args:PYTHON_VERSION=3 context:python-tools fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:python-tools oci_labels:org.opencontainers.ima… (push) Successful in 1m37s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:linkki-tiedotus fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:linkki-tiedotus oci_labels:… (push) Successful in 36s
Build custom container images / build (map[base_image:python:3.12-slim build_args:PYTHON_VERSION=3.12-slim context:timepoll fingerprint_command:{ dpkg-query -W -f='${binary:Package}=${Version}\n' | LC_ALL=C sort; pip freeze | LC_ALL=C sort; } name:timepoll oci_labels:org.opencontai… (push) Successful in 42s

This commit is contained in:
ajp_anton committed 2026-11-01 16:44:11 +00:00
1 parent 3bfd199b9c
commit 1c713ff635
7 files changed
+130 -67

No files matched your search

+37
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import time
import threading
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from io import BytesIO
from pathlib import Path
@@ -379,3 +381,38 @@ def test_upload_is_removed_when_task_process_cannot_start(manager: RunManager) -
assert created is True
assert wait_for(broken_manager, run.id, {"failed"}) == "failed"
assert not (broken_manager.settings.state_root / "runs" / f"{run.id}.uploads").exists()
def test_idle_worker_cannot_start_before_uploaded_input_is_ready(manager, monkeypatch):
write_task(manager.settings.task_root, "imports", "read.py", '''
import json, os
from pathlib import Path
values = json.loads(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
assert Path(values["document"]).read_text() == "uploaded content"
''')
task = manager.catalog.task("imports/read.py")
manager._wake_worker(task.group_id)
saving = threading.Event()
release = threading.Event()
save_uploads = manager._save_uploads
def slow_upload(run_id, uploads):
saving.set()
assert release.wait(5)
return save_uploads(run_id, uploads)
monkeypatch.setattr(manager, '_save_uploads', slow_upload)
upload = PendingUpload('document', '.txt', 1024, BytesIO(b'uploaded content'))
with ThreadPoolExecutor() as pool:
submitted = pool.submit(manager.run_task, task, uploads={'document': upload})
try:
assert saving.wait(5)
run = manager.store.active_runs()[0]
manager._wake_events[task.group_id].set()
time.sleep(.1)
assert manager.store.get(run.id).status == 'queued'
finally:
release.set()
run, created = submitted.result(timeout=5)
assert created
assert wait_for(manager, run.id, {'succeeded', 'failed'}) == 'succeeded'
@@ -1,6 +1,8 @@
import json
import os
import time
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from app.web import create_app
@@ -8,6 +10,32 @@ from .conftest import write_task
from .test_runner import wait_for
def test_concurrent_cleanup_passes_do_not_overlap(manager, monkeypatch):
entered = threading.Event()
release = threading.Event()
calls = []
def cleanup(*args):
calls.append(args)
if len(calls) == 1:
entered.set()
assert release.wait(5)
monkeypatch.setattr(manager, '_cleanup_transient_files', cleanup)
with ThreadPoolExecutor() as pool:
first = pool.submit(manager.cleanup_transient_files)
assert entered.wait(5)
second = pool.submit(manager.cleanup_transient_files)
try:
time.sleep(.1)
assert len(calls) == 1
finally:
release.set()
first.result(timeout=5)
second.result(timeout=5)
assert len(calls) == 2
def test_active_inputs_and_artifacts_never_expire(manager, monkeypatch):
run, _ = manager.store.enqueue(target_kind="task", target_path="example/task.py", queue_group="example")
manager.store.claim_next("example")
+16 -2
View File
@@ -57,7 +57,7 @@ def test_multiline_progress_has_per_step_eta_only(work,monkeypatch):
clock[0]+=30
progress.update('Removing files',5,force=True)
text=json.loads((work/'progress.json').read_text())['text']
assert text=='Removing files: 50%, ETA 30 seconds\nLinking files: 0%'
assert text=='Removing files: 5/10 (50%), ETA 30 seconds\nLinking files: 0/20 (0%)'
assert (work/'progress.json').stat().st_mode&0o777==0o600
@@ -78,4 +78,18 @@ def test_recovery_eta_uses_only_work_done_since_resume(work,monkeypatch):
assert 'ETA' not in json.loads((work/'progress.json').read_text())['text']
clock[0]+=60
progress.update('Items',40,force=True)
assert json.loads((work/'progress.json').read_text())['text']=='Items: 80%, ETA 10 minutes'
assert json.loads((work/'progress.json').read_text())['text']=='Items: 40/50 (80%), ETA 10 minutes'
def test_stage_start_and_completion_bypass_progress_throttle(work, monkeypatch):
monkeypatch.setattr(runtime.time, 'monotonic', lambda: 100.)
progress = runtime.Progress({'Items': 2, 'Empty': 0})
progress.update('Items', 0)
assert json.loads((work / 'progress.json').read_text())['text'] == (
'Items: 0/2 (0%)\nEmpty: waiting'
)
progress.update('Items', 1)
progress.update('Items', 2)
assert json.loads((work / 'progress.json').read_text())['text'].startswith('Items: 2/2 (100%)')
progress.update('Empty', 0)
assert json.loads((work / 'progress.json').read_text())['text'].endswith('Empty: no items')
+2 -1
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from io import BytesIO
import json
import time
from app.web import create_app
@@ -254,7 +255,7 @@ print(Path(os.environ["SERVER_MAINTENANCE_INPUT"]).read_text())
assert executed.input_json == '{"mode":"apply"}'
assert executed_id != run_id
assert app.extensions["run_store"].get(run_id).input_json == '{"mode":"report"}'
assert '"mode":"report"' in client.get(f"/runs/{run_id}/output").get_json()["output"]
assert json.loads(client.get(f"/runs/{run_id}/output").get_json()["output"]) == {"mode": "report"}
executed_detail = client.get(f"/runs/{executed_id}")
assert b'href="/history/task/examples/migrate.py"' in executed_detail.data
history = client.get("/history/task/examples/migrate.py")