"""Delivery worker — executes queued CI/CD runs and polls remotes for pushes. Owned by Agent 3 (see docs/plan/PLAN-v2.md). Two jobs per pass: 1. **Drain the queue.** Every queued run, oldest first, executed to completion. A run is claimed by moving it to `running` before any work happens, so two worker processes never execute the same build. 2. **Poll the remotes.** A push made *outside* the platform (a user pushing to their own git, or to our Gitea through its web UI) is invisible to the engine, which only reports pushes it makes itself. Comparing the remote `dev` head against the last successful run's sha is what closes that gap. Polling is gated by `DELIVERY_POLL_SECONDS` rather than running on every pass: `git ls-remote` against every project on every tick is a lot of network for a signal that changes rarely. """ from __future__ import annotations import logging import time from collections.abc import Callable from app.core.config import settings from app.core.errors import AppError from app.infrastructure.error_log import record_error from app.workers.base import WorkerOutcome, worker_session logger = logging.getLogger("agy.delivery.worker") class DeliveryWorker: name = "delivery" def __init__( self, *, poll_seconds: int | None = None, clock: Callable[[], float] = time.monotonic, ) -> None: self.poll_seconds = int( settings.delivery_poll_seconds if poll_seconds is None else poll_seconds ) self._clock = clock self._last_poll: float | None = None async def run_pass(self, *, limit: int = 10, poll: bool = True) -> WorkerOutcome: """One pass: drain the queue, then (maybe) poll the remotes.""" outcome = WorkerOutcome(name=self.name) from app.application.container import build_delivery_service async with worker_session() as session: service = build_delivery_service(session) queued = await service.runs.list_queued(limit=max(1, limit)) if not queued: outcome.skipped += 1 for run in queued: try: finished = await service.execute_run(run.id) except Exception as exc: # noqa: BLE001 - one bad run must not stop the pass logger.exception("delivery: run %s raised", run.id) await record_error( kind="delivery_run_failed", message=str(exc), exc=exc, organization_id=run.organization_id, metadata={"run_id": run.id, "worker": self.name}, ) outcome.failed += 1 continue outcome.processed += 1 outcome.note(f"run {finished.id} -> {finished.status}") if poll and self._poll_due(): polled = await self._poll_remotes(limit=limit) outcome.processed += polled if polled: outcome.note(f"queued {polled} run(s) from remote pushes") return outcome def _poll_due(self) -> bool: now = self._clock() if self._last_poll is not None and (now - self._last_poll) < self.poll_seconds: return False self._last_poll = now return True async def _poll_remotes(self, *, limit: int) -> int: """Queue a run for every project whose remote `dev` moved without us.""" from app.application.container import build_delivery_service queued = 0 try: async with worker_session() as session: service = build_delivery_service(session) runtimes = await service.runtimes.list_all(limit=limit * 20) for runtime in runtimes: if queued >= limit: break try: run = await service.poll_remote( organization_id=runtime.organization_id, project_id=runtime.project_id, ) except AppError: # A project without a repo yet is normal, not an error. continue except Exception as exc: # noqa: BLE001 - keep polling the rest logger.warning( "delivery: poll failed for project %s: %s", runtime.project_id, exc, ) await record_error( kind="delivery_poll_failed", message=str(exc), exc=exc, organization_id=runtime.organization_id, metadata={"project_id": runtime.project_id, "worker": self.name}, ) continue if run is not None: queued += 1 logger.info( "delivery: poll queued run %s for %s", run.id, runtime.project_id ) except Exception as exc: # noqa: BLE001 - polling is best-effort housekeeping logger.warning("delivery: remote poll pass failed: %s", exc) return queued __all__ = ["DeliveryWorker"]