"""Safe, non-executing task discovery.""" from __future__ import annotations import ast import json import re from dataclasses import dataclass from pathlib import Path from typing import Any FIELD_NAME = re.compile(r"[a-z][a-z0-9_]*\Z") FIELD_TYPES = frozenset({"text", "integer", "date", "datetime", "choice", "multi_choice", "file"}) DISPLAY_ACRONYMS = { "api": "API", "csv": "CSV", "iata": "IATA", "icao": "ICAO", "id": "ID", "ids": "IDs", "url": "URL", "urls": "URLs", "uuid": "UUID", } def display_name(value: str) -> str: titled = value.replace("_", " ").title() return re.sub( r"\b[A-Za-z]+\b", lambda match: DISPLAY_ACRONYMS.get(match.group().casefold(), match.group()), titled, ) @dataclass(frozen=True) class Task: id: str group_id: str filename: str path: Path display_name: str description: str | None inputs: tuple["InputField", ...] wait_for_result: bool = False download_artifacts: bool = False execution: "ExecutionMode | None" = None @dataclass(frozen=True) class BackgroundTask: """A declared internal task that is never exposed through the web UI.""" id: str group_id: str path: Path interval_seconds: int @dataclass(frozen=True) class InputOption: value: str label: str @dataclass(frozen=True) class InputField: name: str label: str type: str required: bool default: str | int | tuple[str, ...] | None minimum: int | None maximum: int | None step: int | None pattern: str | None options: tuple[InputOption, ...] accept: tuple[str, ...] maximum_bytes: int | None sensitive: bool = False @dataclass(frozen=True) class ExecutionMode: """A task-declared report/apply mode exposed as a generated choice field.""" field: InputField dry_run_value: str execute_value: str @dataclass(frozen=True) class Group: id: str path: Path display_name: str tasks: tuple[Task, ...] @property def can_run_as_group(self) -> bool: return all(not task.inputs for task in self.tasks) class TaskCatalog: """Discovers only direct Python children of non-hidden group directories.""" def __init__(self, task_root: Path) -> None: self.task_root = task_root def groups(self) -> tuple[Group, ...]: if not self.task_root.is_dir(): return () groups = [] for group_path in sorted(self.task_root.iterdir(), key=lambda path: path.name): if ( group_path.name.startswith((".", "_")) or group_path.is_symlink() or not group_path.is_dir() ): continue manifest = load_input_manifest( group_path, { path.name for path in group_path.iterdir() if path.is_file() and not path.is_symlink() and path.suffix == ".py" and not path.name.startswith((".", "_")) }, ) tasks = tuple(self._tasks_in_group(group_path, manifest.declarations)) groups.append( Group( id=group_path.name, path=group_path, display_name=manifest.label or display_name(group_path.name), tasks=tasks, ) ) return tuple(groups) def group(self, group_id: str) -> Group | None: return next((group for group in self.groups() if group.id == group_id), None) def task(self, task_id: str) -> Task | None: for group in self.groups(): for task in group.tasks: if task.id == task_id: return task return None def background_tasks(self) -> tuple[BackgroundTask, ...]: if not self.task_root.is_dir(): return () tasks = [] for group_path in sorted(self.task_root.iterdir(), key=lambda path: path.name): if ( group_path.name.startswith((".", "_")) or group_path.is_symlink() or not group_path.is_dir() ): continue tasks.extend(load_background_tasks(group_path)) return tuple(tasks) def _tasks_in_group( self, group_path: Path, declarations_by_filename: dict[str, "TaskDeclaration"] | None = None, ) -> list[Task]: task_paths = [ path for path in sorted(group_path.iterdir(), key=lambda candidate: candidate.name) if not ( path.name.startswith((".", "_")) or path.is_symlink() or not path.is_file() or path.suffix != ".py" ) ] if declarations_by_filename is None: declarations_by_filename = load_input_manifest( group_path, {path.name for path in task_paths}, ).declarations paths_by_filename = {path.name: path for path in task_paths} ordered_filenames = [ *declarations_by_filename, *(path.name for path in task_paths if path.name not in declarations_by_filename), ] tasks = [] for filename in ordered_filenames: path = paths_by_filename[filename] declaration = declarations_by_filename.get(path.name) tasks.append( Task( id=f"{group_path.name}/{path.name}", group_id=group_path.name, filename=path.name, path=path, display_name=display_name(path.stem), description=read_description(path), inputs=( (*declaration.inputs, declaration.execution.field) if declaration and declaration.execution else declaration.inputs if declaration else () ), wait_for_result=declaration.wait_for_result if declaration else False, download_artifacts=declaration.download_artifacts if declaration else False, execution=declaration.execution if declaration else None, ) ) return tasks def load_background_tasks(group_path: Path) -> list[BackgroundTask]: """Read optional internal recurring-task declarations for one group.""" manifest_path = group_path / "background-tasks.json" if not manifest_path.exists(): return [] if manifest_path.is_symlink(): raise ValueError(f"Background task manifest cannot be a symlink: {manifest_path}") try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: raise ValueError(f"Could not read background task manifest {manifest_path}: {error}") from error if not isinstance(data, dict) or set(data) != {"version", "tasks"}: raise ValueError( f"Background task manifest {manifest_path} must contain only version and tasks." ) if data["version"] != 1 or not isinstance(data["tasks"], list): raise ValueError(f"Background task manifest {manifest_path} has an unsupported structure.") tasks = [] seen_ids = set() for declaration in data["tasks"]: if not isinstance(declaration, dict) or set(declaration) != {"id", "script", "interval_seconds"}: raise ValueError(f"Background task manifest {manifest_path} has an invalid task declaration.") task_id = declaration["id"] script = declaration["script"] interval_seconds = declaration["interval_seconds"] if not isinstance(task_id, str) or not FIELD_NAME.fullmatch(task_id) or task_id in seen_ids: raise ValueError(f"Background task manifest {manifest_path} has an invalid or duplicate task id.") if ( not isinstance(script, str) or not script.startswith("internal/") or Path(script).suffix != ".py" or Path(script).is_absolute() or ".." in Path(script).parts ): raise ValueError(f"Background task manifest {manifest_path} task {task_id} has an invalid script.") if not isinstance(interval_seconds, int) or isinstance(interval_seconds, bool) or interval_seconds < 1: raise ValueError(f"Background task manifest {manifest_path} task {task_id} has an invalid interval.") path = group_path / script try: path.relative_to(group_path) except ValueError as error: raise ValueError(f"Background task manifest {manifest_path} task {task_id} escapes its group.") from error if path.is_symlink() or not path.is_file(): raise ValueError(f"Background task manifest {manifest_path} task {task_id} script is unavailable.") parent = path.parent while parent != group_path: if parent.is_symlink(): raise ValueError(f"Background task manifest {manifest_path} task {task_id} script uses a symlink.") parent = parent.parent tasks.append( BackgroundTask( id=f"{group_path.name}/{task_id}", group_id=group_path.name, path=path, interval_seconds=interval_seconds, ) ) seen_ids.add(task_id) return tasks def read_description(path: Path) -> str | None: """Return the first docstring line without importing or executing a task.""" try: module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) except (OSError, SyntaxError, UnicodeDecodeError): return None docstring = ast.get_docstring(module, clean=True) if not docstring: return None return next( (line.strip() for line in docstring.splitlines() if line.strip()), None, ) @dataclass(frozen=True) class GroupManifest: label: str | None declarations: dict[str, "TaskDeclaration"] def load_input_manifest( group_path: Path, task_filenames: set[str], ) -> GroupManifest: """Read the optional data-only input declaration for one task group.""" manifest_path = group_path / "task-inputs.json" if not manifest_path.exists(): return GroupManifest(label=None, declarations={}) if manifest_path.is_symlink(): raise ValueError(f"Input manifest cannot be a symlink: {manifest_path}") try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: raise ValueError(f"Could not read input manifest {manifest_path}: {error}") from error if not isinstance(data, dict) or set(data) - {"version", "label", "tasks"}: raise ValueError( f"Input manifest {manifest_path} has an invalid structure." ) if data.get("version") != 1 or not isinstance(data.get("tasks"), dict): raise ValueError(f"Input manifest {manifest_path} has an unsupported structure.") label = data.get("label") if label is not None and (not isinstance(label, str) or not label.strip()): raise ValueError(f"Input manifest {manifest_path} label must be non-empty text.") unknown_tasks = set(data["tasks"]) - task_filenames if unknown_tasks: names = ", ".join(sorted(unknown_tasks)) raise ValueError(f"Input manifest {manifest_path} names unknown tasks: {names}.") return GroupManifest( label=label, declarations={ filename: parse_task_declaration(manifest_path, filename, declaration) for filename, declaration in data["tasks"].items() }, ) @dataclass(frozen=True) class TaskDeclaration: inputs: tuple[InputField, ...] wait_for_result: bool download_artifacts: bool execution: ExecutionMode | None def parse_task_declaration( manifest_path: Path, filename: str, declaration: Any, ) -> TaskDeclaration: if ( not isinstance(declaration, dict) or set(declaration) - {"inputs", "wait_for_result", "download_artifacts", "execution"} ): raise ValueError(f"{manifest_path} task {filename} has an invalid declaration.") raw_inputs = declaration.get("inputs", []) if not isinstance(raw_inputs, list): raise ValueError(f"{manifest_path} task {filename} inputs must be a list.") fields = tuple(parse_input_field(manifest_path, filename, raw) for raw in raw_inputs) names = [field.name for field in fields] if len(names) != len(set(names)): raise ValueError(f"{manifest_path} task {filename} has duplicate input names.") execution = parse_execution_mode(manifest_path, filename, declaration.get("execution"), set(names)) wait_for_result = declaration.get("wait_for_result", False) if not isinstance(wait_for_result, bool): raise ValueError(f"{manifest_path} task {filename} wait_for_result must be true or false.") download_artifacts = declaration.get("download_artifacts", False) if not isinstance(download_artifacts, bool): raise ValueError(f"{manifest_path} task {filename} download_artifacts must be true or false.") if download_artifacts and not wait_for_result: raise ValueError(f"{manifest_path} task {filename} download_artifacts requires wait_for_result.") return TaskDeclaration( inputs=fields, wait_for_result=wait_for_result, download_artifacts=download_artifacts, execution=execution, ) def parse_execution_mode( manifest_path: Path, filename: str, raw: Any, input_names: set[str], ) -> ExecutionMode | None: if raw is None: return None required_keys = { "field", "label", "dry_run_value", "dry_run_label", "execute_value", "execute_label", } if not isinstance(raw, dict) or set(raw) != required_keys: raise ValueError(f"{manifest_path} task {filename} has an invalid execution declaration.") values = {key: raw[key] for key in required_keys} if not all(isinstance(value, str) and value.strip() for value in values.values()): raise ValueError(f"{manifest_path} task {filename} execution values must be non-empty text.") field_name = values["field"] if not FIELD_NAME.fullmatch(field_name) or field_name in input_names: raise ValueError(f"{manifest_path} task {filename} has an invalid execution field.") if values["dry_run_value"] == values["execute_value"]: raise ValueError(f"{manifest_path} task {filename} execution values must differ.") field = InputField( name=field_name, label=values["label"], type="choice", required=True, default=values["dry_run_value"], minimum=None, maximum=None, step=None, pattern=None, options=( InputOption(values["dry_run_value"], values["dry_run_label"]), InputOption(values["execute_value"], values["execute_label"]), ), accept=(), maximum_bytes=None, ) return ExecutionMode( field=field, dry_run_value=values["dry_run_value"], execute_value=values["execute_value"], ) def parse_input_field(manifest_path: Path, filename: str, raw: Any) -> InputField: if not isinstance(raw, dict): raise ValueError(f"{manifest_path} task {filename} has a non-object input field.") allowed = { "name", "label", "type", "required", "default", "minimum", "maximum", "step", "pattern", "options", "accept", "maximum_bytes", "sensitive", } unknown = set(raw) - allowed if unknown: raise ValueError( f"{manifest_path} task {filename} input has unknown keys: {', '.join(sorted(unknown))}." ) name = raw.get("name") label = raw.get("label") field_type = raw.get("type") if not isinstance(name, str) or not FIELD_NAME.fullmatch(name): raise ValueError(f"{manifest_path} task {filename} has an invalid input name.") if not isinstance(label, str) or not label.strip(): raise ValueError(f"{manifest_path} task {filename} input {name} needs a label.") if field_type not in FIELD_TYPES: raise ValueError(f"{manifest_path} task {filename} input {name} has an invalid type.") required = raw.get("required", False) if not isinstance(required, bool): raise ValueError(f"{manifest_path} task {filename} input {name} required must be true or false.") sensitive = raw.get("sensitive", False) if not isinstance(sensitive, bool): raise ValueError(f"{manifest_path} task {filename} input {name} sensitive must be true or false.") numeric_keys = ("minimum", "maximum", "step") numeric_values = {key: raw.get(key) for key in numeric_keys} if field_type == "integer": for key, value in numeric_values.items(): if value is not None and (not isinstance(value, int) or isinstance(value, bool)): raise ValueError(f"{manifest_path} task {filename} input {name} {key} must be an integer.") if numeric_values["minimum"] is not None and numeric_values["maximum"] is not None and numeric_values["minimum"] > numeric_values["maximum"]: raise ValueError(f"{manifest_path} task {filename} input {name} minimum exceeds maximum.") if numeric_values["step"] is not None and numeric_values["step"] <= 0: raise ValueError(f"{manifest_path} task {filename} input {name} step must be positive.") elif any(value is not None for value in numeric_values.values()): raise ValueError(f"{manifest_path} task {filename} input {name} only integer fields accept numeric limits.") pattern = raw.get("pattern") if pattern is not None: if field_type != "text" or not isinstance(pattern, str): raise ValueError(f"{manifest_path} task {filename} input {name} has an invalid pattern.") try: re.compile(pattern) except re.error as error: raise ValueError(f"{manifest_path} task {filename} input {name} has an invalid pattern: {error}.") from error accept = parse_file_accept(manifest_path, filename, name, field_type, raw.get("accept")) maximum_bytes = raw.get("maximum_bytes") if field_type == "file": if pattern is not None or raw.get("default") is not None: raise ValueError(f"{manifest_path} task {filename} file input {name} cannot define a pattern or default.") if maximum_bytes is not None and ( not isinstance(maximum_bytes, int) or isinstance(maximum_bytes, bool) or maximum_bytes < 1 ): raise ValueError(f"{manifest_path} task {filename} file input {name} maximum_bytes must be positive.") elif maximum_bytes is not None: raise ValueError(f"{manifest_path} task {filename} input {name} only file fields accept maximum_bytes.") options = parse_options(manifest_path, filename, name, field_type, raw.get("options")) default = parse_default(manifest_path, filename, name, field_type, options, raw.get("default")) validate_default( manifest_path, filename, name, field_type, default, numeric_values["minimum"], numeric_values["maximum"], numeric_values["step"], pattern, ) return InputField( name=name, label=label, type=field_type, required=required, default=default, minimum=numeric_values["minimum"], maximum=numeric_values["maximum"], step=numeric_values["step"], pattern=pattern, options=options, accept=accept, maximum_bytes=maximum_bytes, sensitive=sensitive, ) def parse_file_accept( manifest_path: Path, filename: str, name: str, field_type: str, raw_accept: Any, ) -> tuple[str, ...]: if field_type != "file": if raw_accept is not None: raise ValueError(f"{manifest_path} task {filename} input {name} only file fields accept accept.") return () if raw_accept is None: return () if ( not isinstance(raw_accept, list) or not raw_accept or not all(isinstance(value, str) and re.fullmatch(r"\.[a-z0-9]{1,10}", value) for value in raw_accept) or len(set(raw_accept)) != len(raw_accept) ): raise ValueError(f"{manifest_path} task {filename} file input {name} has invalid accept values.") return tuple(raw_accept) def parse_options( manifest_path: Path, filename: str, name: str, field_type: str, raw_options: Any, ) -> tuple[InputOption, ...]: if field_type not in {"choice", "multi_choice"}: if raw_options is not None: raise ValueError(f"{manifest_path} task {filename} input {name} only choice fields accept options.") return () if not isinstance(raw_options, list) or not raw_options: raise ValueError(f"{manifest_path} task {filename} input {name} needs non-empty options.") options = [] for raw_option in raw_options: if isinstance(raw_option, str) and raw_option: options.append(InputOption(value=raw_option, label=raw_option)) elif ( isinstance(raw_option, dict) and set(raw_option) == {"value", "label"} and isinstance(raw_option["value"], str) and raw_option["value"] and isinstance(raw_option["label"], str) and raw_option["label"].strip() ): options.append(InputOption(value=raw_option["value"], label=raw_option["label"])) else: raise ValueError(f"{manifest_path} task {filename} input {name} has an invalid option.") if len({option.value for option in options}) != len(options): raise ValueError(f"{manifest_path} task {filename} input {name} has duplicate option values.") return tuple(options) def parse_default( manifest_path: Path, filename: str, name: str, field_type: str, options: tuple[InputOption, ...], default: Any, ) -> str | int | tuple[str, ...] | None: if default is None: return None if field_type == "integer": if not isinstance(default, int) or isinstance(default, bool): raise ValueError(f"{manifest_path} task {filename} input {name} default must be an integer.") return default if field_type == "multi_choice": if not isinstance(default, list) or not all(isinstance(value, str) for value in default): raise ValueError(f"{manifest_path} task {filename} input {name} default must be a list of options.") values = tuple(default) if len(values) != len(set(values)) or not set(values) <= {option.value for option in options}: raise ValueError(f"{manifest_path} task {filename} input {name} default contains an unknown option.") return values if not isinstance(default, str): raise ValueError(f"{manifest_path} task {filename} input {name} default must be text.") if field_type == "choice" and default not in {option.value for option in options}: raise ValueError(f"{manifest_path} task {filename} input {name} default is not an option.") return default def validate_default( manifest_path: Path, filename: str, name: str, field_type: str, default: str | int | tuple[str, ...] | None, minimum: int | None, maximum: int | None, step: int | None, pattern: str | None, ) -> None: if default is None or field_type not in {"integer", "text"}: return if field_type == "text": if pattern and not re.fullmatch(pattern, default): raise ValueError(f"{manifest_path} task {filename} input {name} default does not match its pattern.") return assert isinstance(default, int) if minimum is not None and default < minimum: raise ValueError(f"{manifest_path} task {filename} input {name} default is below minimum.") if maximum is not None and default > maximum: raise ValueError(f"{manifest_path} task {filename} input {name} default is above maximum.") if step is not None and (default - (minimum or 0)) % step: raise ValueError(f"{manifest_path} task {filename} input {name} default does not match step.")