feat: realtime WebSocket layer for live events and streaming logs

- Backend: ConnectionManager, Redis-backed bus with in-process fallback, channel authorization, and typed emit helpers wired into workers (ai_runtime, sandbox_gateway), workforce/sandbox/billing/audit/error services. - Frontend: single shared WS client with backoff + heartbeat, RealtimeProvider, useLiveChannel/useLiveEvent hooks, live indicators on task detail, manager chat, dashboard, traces, sandbox terminal, admin errors, and billing. - Contracts: WsEvent/WsServerFrame/WsClientFrame types mirroring the port. All 255 backend tests pass; web typecheck and build pass.
2026-10-13 12:56:36 +03:30
parent afdef4b444
commit f1e5df29ce
31 changed files with 2981 additions and 28 deletions
@@ -8,6 +8,7 @@ from __future__ import annotations

 from sqlalchemy.ext.asyncio import AsyncSession

+from app.application.ports.realtime import emit_audit_recorded
 from app.domain.audit.models import AuditEvent
 from app.infrastructure.repositories.audit import AuditRepository

@@ -34,4 +35,13 @@ async def record_audit(
        ip_address=ip,
        user_agent=user_agent,
    )
-    return await AuditRepository(session).record(event=event)
+    recorded = await AuditRepository(session).record(event=event)
+    # Live push: the audit timelines (dashboard, traces) update as actions
+    # happen instead of waiting for the next reload.
+    await emit_audit_recorded(
+        organization_id=organization_id,
+        action=action,
+        target_type=target_type,
+        target_id=target_id,
+    )
+    return recorded
@@ -14,6 +14,7 @@ from __future__ import annotations
 from datetime import datetime

 from app.application.ports import PaymentProviderPort
+from app.application.ports.realtime import emit_wallet_changed
 from app.core.errors import NotFoundError, PermissionDeniedError, ValidationError
 from app.domain.billing.models import (
    CreditWallet,
@@ -121,6 +122,11 @@ class BillingService:
            # Refuse to leave a wallet in debt; undo and surface the conflict.
            await self.wallets.adjust(organization_id=organization_id, delta=-amount)
            raise ValidationError("اعتبار کافی برای این کسر وجود ندارد.")
+        # Live push: the sidebar credit badge and the billing page follow the
+        # wallet as it moves, whoever moved it.
+        await emit_wallet_changed(
+            organization_id=organization_id, balance=wallet.balance
+        )
        return wallet

    async def start_checkout(
@@ -177,8 +183,13 @@ class BillingService:
            return payment

        if succeeded:
-            await self.wallets.adjust(organization_id=organization_id, delta=payment.amount)
+            wallet = await self.wallets.adjust(
+                organization_id=organization_id, delta=payment.amount
+            )
            final = await self.payments.set_status(payment_id, PaymentStatus.SUCCEEDED)
+            await emit_wallet_changed(
+                organization_id=organization_id, balance=wallet.balance
+            )
        else:
            final = await self.payments.set_status(payment_id, PaymentStatus.FAILED)
        return final or payment
@@ -18,6 +18,10 @@ import contextlib
 from datetime import datetime

 from app.application.ports import CommandResult, SandboxProviderPort, UsageRecord
+from app.application.ports.realtime import (
+    emit_sandbox_command,
+    emit_sandbox_session,
+)
 from app.core.config import settings
 from app.core.errors import ConflictError, NotFoundError, ProviderUnavailableError, ValidationError
 from app.domain.ai_control.models import RateUnit
@@ -210,6 +214,12 @@ class SandboxService:
            )
        )
        await self.workspaces.set_status(workspace.id, WorkspaceStatus.READY)
+        await emit_sandbox_session(
+            session_id=session.id,
+            organization_id=organization_id,
+            status=SessionStatus.RUNNING.value,
+            workspace_id=workspace.id,
+        )
        return session

    async def get_session_or_404(
@@ -229,6 +239,11 @@ class SandboxService:
        await self.provider.destroy(session_external_id=session.external_session_id)
        await self.sessions.set_status(session.id, SessionStatus.STOPPED, stopped=True)
        refreshed = await self.sessions.get(session.id)
+        await emit_sandbox_session(
+            session_id=session.id,
+            organization_id=organization_id,
+            status=SessionStatus.STOPPED.value,
+        )
        return refreshed or session

    # -- commands ------------------------------------------------------------
@@ -285,6 +300,20 @@ class SandboxService:
        )
        await self.sessions.set_status(session.id, SessionStatus.RUNNING)

+        # Live push: the terminal panel on the project page streams the command
+        # and its output as the row is written, not on the next poll.
+        await emit_sandbox_command(
+            session_id=session.id,
+            organization_id=organization_id,
+            command_run_id=run.id,
+            command=cleaned,
+            exit_code=result.exit_code,
+            stdout=run.stdout_ref,
+            stderr=run.stderr_ref,
+            duration_ms=result.duration_ms,
+            truncated=result.truncated,
+        )
+
        # CPU seconds are billed. 1 second of wall time ~= 1 CPU second at MVP.
        cpu_seconds = max(0.0, result.duration_ms / 1000.0)
        if cpu_seconds > 0:
@@ -24,6 +24,17 @@ from app.application.ports import (
    ModelProviderPort,
    UsageRecord,
 )
+from app.application.ports.realtime import (
+    emit_approval_pending,
+    emit_artifact_created,
+    emit_chat_pending,
+    emit_message_created,
+    emit_run_status,
+    emit_run_step,
+    emit_task_created,
+    emit_task_status,
+    emit_usage_recorded,
+)
 from app.core.errors import (
    ConflictError,
    NotFoundError,
@@ -324,6 +335,10 @@ class WorkforceService:
        else:
            employee = await self._default_manager(organization_id)

+        # The manager is about to call a model. Tell the thread so the UI can
+        # show a thinking indicator immediately.
+        await emit_chat_pending(thread_id=thread.id, organization_id=organization_id)
+
        history = await self.messages.list_for_thread(thread.id)
        system_prompt = await self._system_prompt_for(employee)
        reply = await self._complete(
@@ -347,6 +362,16 @@ class WorkforceService:
                employee_id=employee.id if employee else None,
            )
        )
+        # Live push: any browser on this thread sees the reply arrive, and the
+        # pending indicator clears, without waiting for a reload.
+        await emit_message_created(
+            thread_id=thread.id,
+            organization_id=organization_id,
+            message_id=assistant_message.id,
+            role=MessageRole.ASSISTANT.value,
+            content=reply,
+            employee_id=employee.id if employee else None,
+        )
        return user_message, assistant_message

    # ------------------------------------------------------------------ #
@@ -371,7 +396,7 @@ class WorkforceService:
            await self.get_employee_or_404(
                organization_id=organization_id, employee_id=assigned_employee_id
            )
-        return await self.tasks.create(
+        task = await self.tasks.create(
            Task(
                organization_id=organization_id,
                title=title.strip(),
@@ -382,6 +407,15 @@ class WorkforceService:
                created_by_user_id=created_by_user_id,
            )
        )
+        # Live push: the tasks list and the manager-chat sidebar update the
+        # moment the task exists, whoever created it and from where.
+        await emit_task_created(
+            task_id=task.id,
+            organization_id=organization_id,
+            title=task.title,
+            status=task.status.value,
+        )
+        return task

    async def list_tasks(
        self,
@@ -435,21 +469,39 @@ class WorkforceService:
            Run(task_id=task.id, employee_id=employee.id if employee else None)
        )
        await self.tasks.set_status(task.id, TaskStatus.RUNNING)
+        await emit_task_status(
+            task_id=task.id, organization_id=organization_id, status=TaskStatus.RUNNING.value
+        )

-        await self.run_steps.append(
-            run_id=run.id, kind="queued", title="کار در صف اجرا قرار گرفت.",
-            payload={"task_id": task.id},
+        await self._publish_step(
+            organization_id=organization_id,
+            run_id=run.id,
+            task_id=task.id,
+            step=await self.run_steps.append(
+                run_id=run.id, kind="queued", title="کار در صف اجرا قرار گرفت.",
+                payload={"task_id": task.id},
+            ),
        )

        try:
            run = await self._advance(run, RunStatus.PLANNING)
-            await self.run_steps.append(
-                run_id=run.id, kind="planning", title="برنامه‌ریزی مرحلهٔ اجرا."
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="planning", title="برنامه‌ریزی مرحلهٔ اجرا."
+                ),
            )

            run = await self._advance(run, RunStatus.RUNNING)
-            await self.run_steps.append(
-                run_id=run.id, kind="running", title="اجرای کار آغاز شد."
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="running", title="اجرای کار آغاز شد."
+                ),
            )

            reply = await self._complete(
@@ -463,9 +515,14 @@ class WorkforceService:
                    ChatMessage(role="user", content=self._task_brief(task)),
                ],
            )
-            await self.run_steps.append(
-                run_id=run.id, kind="output", title="خروجی آماده شد.",
-                payload={"chars": len(reply)},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="output", title="خروجی آماده شد.",
+                    payload={"chars": len(reply)},
+                ),
            )

            artifact = await self.artifacts.create(
@@ -479,30 +536,106 @@ class WorkforceService:
                    size_bytes=len(reply.encode("utf-8")),
                )
            )
-            await self.run_steps.append(
-                run_id=run.id, kind="artifact", title="سند خروجی ساخته شد.",
-                payload={"artifact_id": artifact.id},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="artifact", title="سند خروجی ساخته شد.",
+                    payload={"artifact_id": artifact.id},
+                ),
+            )
+            await emit_artifact_created(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                artifact_id=artifact.id,
+                name=artifact.name,
            )

            run = await self._finish(run, RunStatus.COMPLETED)
            await self.tasks.set_status(task.id, TaskStatus.DONE)
+            await emit_run_status(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                status=RunStatus.COMPLETED.value,
+            )
+            await emit_task_status(
+                task_id=task.id, organization_id=organization_id, status=TaskStatus.DONE.value
+            )
        except ProviderUnavailableError as exc:
            run = await self._finish(run, RunStatus.FAILED, error=str(exc))
            await self.tasks.set_status(task.id, TaskStatus.FAILED)
-            await self.run_steps.append(
-                run_id=run.id, kind="error", title="اجرا ناموفق بود.",
-                payload={"error": str(exc)},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="error", title="اجرا ناموفق بود.",
+                    payload={"error": str(exc)},
+                ),
+            )
+            await emit_run_status(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                status=RunStatus.FAILED.value,
+                error=str(exc),
+            )
+            await emit_task_status(
+                task_id=task.id, organization_id=organization_id, status=TaskStatus.FAILED.value
            )
        except Exception as exc:  # noqa: BLE001 - the run must always terminate
            run = await self._finish(run, RunStatus.FAILED, error=str(exc))
            await self.tasks.set_status(task.id, TaskStatus.FAILED)
-            await self.run_steps.append(
-                run_id=run.id, kind="error", title="خطای غیرمنتظره در اجرا.",
-                payload={"error": str(exc)},
+            await self._publish_step(
+                organization_id=organization_id,
+                run_id=run.id,
+                task_id=task.id,
+                step=await self.run_steps.append(
+                    run_id=run.id, kind="error", title="خطای غیرمنتظره در اجرا.",
+                    payload={"error": str(exc)},
+                ),
+            )
+            await emit_run_status(
+                run_id=run.id,
+                task_id=task.id,
+                organization_id=organization_id,
+                status=RunStatus.FAILED.value,
+                error=str(exc),
+            )
+            await emit_task_status(
+                task_id=task.id, organization_id=organization_id, status=TaskStatus.FAILED.value
            )

        return run

+    async def _publish_step(
+        self,
+        *,
+        organization_id: str,
+        run_id: str,
+        task_id: str,
+        step: RunStep,
+    ) -> None:
+        """Push one timeline step to the run, the task and the tenant feed.
+
+        The step is already persisted by the repository; this only broadcasts
+        it. Kept as a helper so the run engine above reads as the plain
+        sequence of transitions it is.
+        """
+        await emit_run_step(
+            run_id=run_id,
+            task_id=task_id,
+            organization_id=organization_id,
+            step_id=step.id,
+            index=step.index,
+            kind=step.kind,
+            title=step.title,
+            payload=step.payload_json,
+        )
+
    async def get_run_or_404(
        self, *, organization_id: str, run_id: str
    ) -> tuple[Run, list[RunStep], list[ToolCall], list[Artifact]]:
@@ -530,12 +663,28 @@ class WorkforceService:
            raise ConflictError("از وضعیت فعلی امکان لغو وجود ندارد.")

        run = await self._finish(run, RunStatus.CANCELLED)
-        await self.run_steps.append(
-            run_id=run.id, kind="cancelled", title="اجرا توسط کاربر لغو شد."
+        await self._publish_step(
+            organization_id=organization_id,
+            run_id=run.id,
+            task_id=run.task_id,
+            step=await self.run_steps.append(
+                run_id=run.id, kind="cancelled", title="اجرا توسط کاربر لغو شد."
+            ),
        )
        task = await self.tasks.get(run.task_id)
        if task is not None and task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
            await self.tasks.set_status(task.id, TaskStatus.CANCELLED)
+            await emit_task_status(
+                task_id=task.id,
+                organization_id=organization_id,
+                status=TaskStatus.CANCELLED.value,
+            )
+        await emit_run_status(
+            run_id=run.id,
+            task_id=run.task_id,
+            organization_id=organization_id,
+            status=RunStatus.CANCELLED.value,
+        )
        return run

    async def list_pending_approvals(self, organization_id: str) -> list[ToolCall]:
@@ -627,6 +776,14 @@ class WorkforceService:
            ),
            payload={"tool_call_id": call.id, "requires_approval": needs_approval},
        )
+        await emit_approval_pending(
+            run_id=run.id,
+            task_id=run.task_id,
+            organization_id=organization_id,
+            tool_call_id=call.id,
+            tool_key=tool_key,
+            requires_approval=needs_approval,
+        )
        return call

    async def decide_approval(
@@ -655,12 +812,25 @@ class WorkforceService:
            result=result,
            approved_by_user_id=decided_by_user_id,
        )
+        run = await self.runs.get(call.run_id)
        await self.run_steps.append(
            run_id=call.run_id,
            kind="approval",
            title=("درخواست تأیید شد." if approve else "درخواست رد شد."),
            payload={"tool_call_id": call.id, "tool_key": call.tool_key},
        )
+        # Live push: the task page clears the pending card and the run timeline
+        # gains the decision without a reload.
+        await emit_run_step(
+            run_id=call.run_id,
+            task_id=run.task_id if run else "",
+            organization_id=organization_id,
+            step_id=call.id,
+            index=0,
+            kind="approval",
+            title=("درخواست تأیید شد." if approve else "درخواست رد شد."),
+            payload={"tool_call_id": call.id, "tool_key": call.tool_key},
+        )
        return updated or call

    # ------------------------------------------------------------------ #
@@ -776,6 +946,15 @@ class WorkforceService:
                metadata_json={"priced": card is not None},
            )
        )
+        # Live push: the billing page and the sidebar credit badge react to
+        # consumption as it happens instead of at the next page load.
+        await emit_usage_recorded(
+            organization_id=organization_id,
+            unit=unit.value if hasattr(unit, "value") else str(unit),
+            quantity=quantity,
+            run_id=run_id,
+            employee_id=employee_id,
+        )

    async def _advance(self, run: Run, target: RunStatus) -> Run:
        if not can_transition_run(run.status, target):