mjalmoudy94
f1e5df29ce
- Backend: ConnectionManager, Redis-backed bus with in-process fallback, channel authorization, and typed emit helpers wired into workers (ai_runtime, sandbox_gateway), workforce/sandbox/billing/audit/error services. - Frontend: single shared WS client with backoff + heartbeat, RealtimeProvider, useLiveChannel/useLiveEvent hooks, live indicators on task detail, manager chat, dashboard, traces, sandbox terminal, admin errors, and billing. - Contracts: WsEvent/WsServerFrame/WsClientFrame types mirroring the port. All 255 backend tests pass; web typecheck and build pass.
103 lines
4.4 KiB
Python
103 lines
4.4 KiB
Python
"""AI runtime worker — drives queued tasks through the run engine.
|
|
|
|
The HTTP API can start a run inline (that is what the "اجرا" button does), but
|
|
a platform that only ever runs work on the request thread has two problems: a
|
|
slow model call holds a connection open, and a task created by anything other
|
|
than a button press never runs at all. This worker is the second driver.
|
|
|
|
It deliberately owns **no** orchestration logic. It finds runnable tasks and
|
|
hands each one to ``WorkforceService.start_run`` — the same call the router
|
|
makes. Model routing, usage accounting, the timeline and the artifact are all
|
|
the service's business.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from app.application.container import build_workforce_service
|
|
from app.application.ports.realtime import emit_worker_status
|
|
from app.core.errors import AppError
|
|
from app.domain.audit.models import ErrorSeverity, ErrorSource
|
|
from app.domain.workforce.models import TaskStatus
|
|
from app.infrastructure.error_log import record_error
|
|
from app.infrastructure.repositories.workforce import TaskRepository
|
|
from app.workers.base import WorkerOutcome, worker_session
|
|
|
|
|
|
class AiRuntimeWorker:
|
|
"""Drains the queue of tasks that nobody has run yet."""
|
|
|
|
name = "ai-runtime"
|
|
|
|
async def drain(self, *, limit: int = 10) -> WorkerOutcome:
|
|
"""Run up to ``limit`` queued tasks, oldest first.
|
|
|
|
A task that fails is recorded and the pass continues: one poisoned task
|
|
must not block the queue behind it.
|
|
"""
|
|
outcome = WorkerOutcome(name=self.name)
|
|
|
|
async with worker_session() as session:
|
|
tasks = TaskRepository(session)
|
|
workforce = build_workforce_service(session)
|
|
|
|
for task in await tasks.list_runnable(limit=limit):
|
|
# Re-read through the service so the tenant check and the
|
|
# "already closed" guard both apply. The worker gets no
|
|
# privileged bypass.
|
|
try:
|
|
current = await workforce.get_task_or_404(
|
|
organization_id=task.organization_id, task_id=task.id
|
|
)
|
|
except AppError:
|
|
outcome.skipped += 1
|
|
outcome.note(f"task {task.id}: vanished before the run")
|
|
continue
|
|
|
|
if current.status not in (TaskStatus.NEW, TaskStatus.TRIAGED):
|
|
outcome.skipped += 1
|
|
continue
|
|
|
|
try:
|
|
run = await workforce.start_run(
|
|
organization_id=task.organization_id,
|
|
task_id=task.id,
|
|
user_id=task.created_by_user_id or "",
|
|
employee_id=task.assigned_employee_id,
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - keep draining
|
|
outcome.failed += 1
|
|
outcome.note(f"task {task.id}: {type(exc).__name__}: {exc}")
|
|
# Persist with the tenant attached: a task that fails to run
|
|
# is a tenant-visible problem, and the console must be able
|
|
# to show that organization what happened.
|
|
await record_error(
|
|
kind="task_run_failed",
|
|
message=str(exc) or type(exc).__name__,
|
|
exc=exc,
|
|
source=ErrorSource.WORKER,
|
|
severity=ErrorSeverity.ERROR,
|
|
organization_id=task.organization_id,
|
|
user_id=task.created_by_user_id,
|
|
metadata={
|
|
"worker": self.name,
|
|
"task_id": task.id,
|
|
"employee_id": task.assigned_employee_id,
|
|
},
|
|
)
|
|
continue
|
|
|
|
outcome.processed += 1
|
|
outcome.note(f"task {task.id} -> run {run.id} ({run.status})")
|
|
# This worker is its own process: without a push here, a run
|
|
# started by the queue is invisible until someone reloads.
|
|
await emit_worker_status(
|
|
organization_id=task.organization_id,
|
|
worker=self.name,
|
|
message=f"کار «{current.title}» در حال اجراست (اجرا {run.id}).",
|
|
)
|
|
|
|
return outcome
|
|
|
|
|
|
__all__ = ["AiRuntimeWorker"]
|