feat: realtime WebSocket layer for live events and streaming logs
- Backend: ConnectionManager, Redis-backed bus with in-process fallback, channel authorization, and typed emit helpers wired into workers (ai_runtime, sandbox_gateway), workforce/sandbox/billing/audit/error services. - Frontend: single shared WS client with backoff + heartbeat, RealtimeProvider, useLiveChannel/useLiveEvent hooks, live indicators on task detail, manager chat, dashboard, traces, sandbox terminal, admin errors, and billing. - Contracts: WsEvent/WsServerFrame/WsClientFrame types mirroring the port. All 255 backend tests pass; web typecheck and build pass.
This commit is contained in:
mjalmoudy94
@@ -42,5 +42,22 @@ async def readiness(session: AsyncSession = Depends(get_session)) -> dict:
|
||||
"sandbox": settings.sandbox_provider,
|
||||
"payment": settings.payment_provider,
|
||||
}
|
||||
|
||||
# Live-feed visibility: how many browser sockets this process is serving,
|
||||
# and whether the cross-process relay (Redis) is connected. An operator
|
||||
# wondering "is realtime working?" should not have to read the logs.
|
||||
try:
|
||||
from app.interfaces.realtime.bus import get_bus
|
||||
from app.interfaces.realtime.router import realtime_manager
|
||||
|
||||
manager = realtime_manager()
|
||||
checks["realtime"] = {
|
||||
"connections": manager.connection_count,
|
||||
"channels": manager.channel_count,
|
||||
"redis_relay": "connected" if get_bus().redis_healthy else "in_process_only",
|
||||
}
|
||||
except Exception: # noqa: BLE001 - health must answer regardless
|
||||
checks["realtime"] = "unavailable"
|
||||
|
||||
healthy = checks.get("database") == "ok"
|
||||
return {"status": "ok" if healthy else "degraded", "checks": checks}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Realtime push layer — the WebSocket *interface* adapter.
|
||||
|
||||
The adapter is transport and plumbing only: who is connected to which channel
|
||||
(:mod:`manager`), how an event travels between processes (:mod:`bus`), which
|
||||
channels a socket may subscribe to (:mod:`authz`), and the wire protocol
|
||||
(:mod:`router`).
|
||||
|
||||
The event contract and the typed publishers live one layer in, in
|
||||
``app.application.ports.realtime`` — services depend on the port, this package
|
||||
implements it. ``app.main`` is the composition root that installs the bus as
|
||||
the process publisher and starts the cross-process relay.
|
||||
"""
|
||||
|
||||
from app.interfaces.realtime.bus import get_bus, start_relay, stop_relay
|
||||
|
||||
__all__ = [
|
||||
"get_bus",
|
||||
"start_relay",
|
||||
"stop_relay",
|
||||
]
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Channel authorization — the only gate on what a socket may receive.
|
||||
|
||||
Authorization happens once, at subscribe time. After that the connection holds
|
||||
plain channel names and the manager routes on them, so there is no code path
|
||||
that can deliver an event to a connection that was never checked.
|
||||
|
||||
Every rule resolves the channel's tenant through the repositories and then the
|
||||
caller's membership in it — exactly like the HTTP layer's ``get_org_context``.
|
||||
A tenant that does not own the object gets a rejection, not an empty stream:
|
||||
the difference matters when debugging "why is my page not live?".
|
||||
|
||||
The caller's identity arrives as two primitives (id, platform-owner flag) —
|
||||
what an authenticated socket actually knows — rather than a ``User`` row, so a
|
||||
subscribe costs one fewer database read.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.infrastructure.repositories.identity import MembershipRepository
|
||||
from app.infrastructure.repositories.sandbox import (
|
||||
SandboxSessionRepository,
|
||||
WorkspaceRepository,
|
||||
)
|
||||
from app.infrastructure.repositories.workforce import (
|
||||
RunRepository,
|
||||
TaskRepository,
|
||||
ThreadRepository,
|
||||
)
|
||||
|
||||
|
||||
async def authorize_channel(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
user_id: str,
|
||||
is_platform_owner: bool,
|
||||
channel: str,
|
||||
) -> bool:
|
||||
"""May this identity subscribe to ``channel``? Deny by default."""
|
||||
kind, _, identifier = channel.partition(":")
|
||||
if not identifier and kind != "admin":
|
||||
return False
|
||||
|
||||
if kind == "admin":
|
||||
return is_platform_owner
|
||||
if kind == "user":
|
||||
return identifier == user_id
|
||||
if kind == "org":
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=identifier
|
||||
)
|
||||
if kind == "task":
|
||||
return await _task_is_visible(
|
||||
session=session, user_id=user_id, task_id=identifier
|
||||
)
|
||||
if kind == "run":
|
||||
return await _run_is_visible(
|
||||
session=session, user_id=user_id, run_id=identifier
|
||||
)
|
||||
if kind == "thread":
|
||||
return await _thread_is_visible(
|
||||
session=session, user_id=user_id, thread_id=identifier
|
||||
)
|
||||
if kind == "session":
|
||||
return await _session_is_visible(
|
||||
session=session, user_id=user_id, session_id=identifier
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def _is_member(*, session: AsyncSession, user_id: str, organization_id: str) -> bool:
|
||||
membership = await MembershipRepository(session).get(organization_id, user_id)
|
||||
return membership is not None and membership.is_active
|
||||
|
||||
|
||||
async def _task_is_visible(*, session: AsyncSession, user_id: str, task_id: str) -> bool:
|
||||
task = await TaskRepository(session).get(task_id)
|
||||
if task is None:
|
||||
return False
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=task.organization_id
|
||||
)
|
||||
|
||||
|
||||
async def _run_is_visible(*, session: AsyncSession, user_id: str, run_id: str) -> bool:
|
||||
run = await RunRepository(session).get(run_id)
|
||||
if run is None:
|
||||
return False
|
||||
return await _task_is_visible(session=session, user_id=user_id, task_id=run.task_id)
|
||||
|
||||
|
||||
async def _thread_is_visible(*, session: AsyncSession, user_id: str, thread_id: str) -> bool:
|
||||
thread = await ThreadRepository(session).get(thread_id)
|
||||
if thread is None:
|
||||
return False
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=thread.organization_id
|
||||
)
|
||||
|
||||
|
||||
async def _session_is_visible(*, session: AsyncSession, user_id: str, session_id: str) -> bool:
|
||||
"""A sandbox session belongs to a workspace, which belongs to a tenant."""
|
||||
sandbox_session = await SandboxSessionRepository(session).get(session_id)
|
||||
if sandbox_session is None:
|
||||
return False
|
||||
workspace = await WorkspaceRepository(session).get(sandbox_session.workspace_id)
|
||||
if workspace is None:
|
||||
return False
|
||||
return await _is_member(
|
||||
session=session, user_id=user_id, organization_id=workspace.organization_id
|
||||
)
|
||||
@@ -0,0 +1,260 @@
|
||||
"""The event bus: how a published event reaches connected browsers.
|
||||
|
||||
Two transports, in this order:
|
||||
|
||||
1. **In-process.** Always on. When the API itself records the event (a task run
|
||||
driven by a button, a sandbox command, an error handler), the fan-out is a
|
||||
direct call into the ``ConnectionManager`` — microseconds, no broker.
|
||||
2. **Redis pub/sub.** Carries events published by *other* processes: the
|
||||
standalone workers (``pnpm worker:ai-runtime``) run tasks in their own
|
||||
process, and without a broker their events would die there. The API process
|
||||
relays whatever arrives on the shared channel into the same manager.
|
||||
|
||||
Redis is a bonus, never a dependency:
|
||||
|
||||
* The bus tries it once per cooldown window. If nothing is listening, the
|
||||
attempt fails quietly and in-process delivery continues — the product stays
|
||||
fully live for every API-driven action, only cross-process events are lost.
|
||||
* The relay task owns the subscription lifecycle; it starts with the app and
|
||||
stops on shutdown, and reconnects on its own after a broker restart.
|
||||
|
||||
Nothing here ever raises. Publishing is called from request handlers, from
|
||||
inside exception handlers and from worker passes; an exception in the feed
|
||||
must not be able to fail any of those.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
|
||||
from app.application.ports.realtime import RealtimeEvent, set_publisher
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
#: Redis pub/sub channel shared by every process of one deployment.
|
||||
DEFAULT_REDIS_CHANNEL = "agy:realtime"
|
||||
|
||||
#: After a failed broker attempt, wait this long before trying again. Keeps a
|
||||
#: dev machine without Redis from adding a connection timeout to every event.
|
||||
REDIS_RETRY_COOLDOWN_SECONDS = 10.0
|
||||
|
||||
#: How long a single connection attempt may take before it is abandoned.
|
||||
REDIS_CONNECT_TIMEOUT_SECONDS = 1.0
|
||||
|
||||
#: Listener signature: ``await listener(event)``. The ConnectionManager
|
||||
#: registers one; tests register fakes. The return value is ignored so a
|
||||
#: listener may report how many connections it fanned out to.
|
||||
EventListener = Callable[[RealtimeEvent], Awaitable[object]]
|
||||
|
||||
|
||||
class RealtimeBus:
|
||||
"""Fan-out hub shared by every publisher in this process."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
redis_url: str | None = None,
|
||||
channel: str = DEFAULT_REDIS_CHANNEL,
|
||||
) -> None:
|
||||
self._listeners: set[EventListener] = set()
|
||||
self._redis_url = redis_url if redis_url is not None else settings.redis_url
|
||||
self._channel = channel
|
||||
self._redis = None
|
||||
self._pubsub = None
|
||||
self._relay_task: asyncio.Task | None = None
|
||||
self._next_redis_attempt = 0.0
|
||||
self._redis_healthy = False
|
||||
|
||||
# -- listeners (in-process fan-out) --------------------------------------
|
||||
|
||||
def attach(self, listener: EventListener) -> None:
|
||||
self._listeners.add(listener)
|
||||
|
||||
def detach(self, listener: EventListener) -> None:
|
||||
self._listeners.discard(listener)
|
||||
|
||||
# -- publishing -----------------------------------------------------------
|
||||
|
||||
async def publish(self, event: RealtimeEvent) -> None:
|
||||
"""Deliver locally, then mirror to other processes. Never raises."""
|
||||
try:
|
||||
await self._dispatch_local(event)
|
||||
except Exception: # noqa: BLE001 - the feed must never fail a caller
|
||||
logger.exception("realtime: local dispatch failed for %s", event.type)
|
||||
|
||||
if not self._redis_enabled():
|
||||
return
|
||||
try:
|
||||
await self._publish_redis(event)
|
||||
except Exception: # noqa: BLE001 - same rule
|
||||
logger.exception("realtime: redis publish failed for %s", event.type)
|
||||
|
||||
async def _dispatch_local(self, event: RealtimeEvent) -> None:
|
||||
for listener in list(self._listeners):
|
||||
try:
|
||||
await listener(event)
|
||||
except Exception: # noqa: BLE001 - one bad listener is not fatal
|
||||
logger.exception("realtime: listener failed for %s", event.type)
|
||||
|
||||
def _redis_enabled(self) -> bool:
|
||||
# "local" pins the bus to in-process delivery — used by tests and by
|
||||
# single-process deployments that have no broker on purpose.
|
||||
return getattr(settings, "realtime_transport", "auto").lower() != "local"
|
||||
|
||||
async def _publish_redis(self, event: RealtimeEvent) -> None:
|
||||
client = await self._redis_client()
|
||||
if client is None:
|
||||
return
|
||||
await client.publish(self._channel, event.to_json())
|
||||
|
||||
async def _redis_client(self):
|
||||
"""Lazily connect, with a cooldown after failure. ``None`` if down."""
|
||||
if self._redis is not None:
|
||||
return self._redis
|
||||
now = time.monotonic()
|
||||
if now < self._next_redis_attempt:
|
||||
return None
|
||||
self._next_redis_attempt = now + REDIS_RETRY_COOLDOWN_SECONDS
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
client = aioredis.from_url(
|
||||
self._redis_url,
|
||||
encoding="utf-8",
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=REDIS_CONNECT_TIMEOUT_SECONDS,
|
||||
socket_timeout=REDIS_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
await client.ping()
|
||||
self._redis = client
|
||||
self._redis_healthy = True
|
||||
logger.info("realtime: redis pub/sub connected (%s)", self._channel)
|
||||
except Exception as exc: # noqa: BLE001 - broker is optional
|
||||
self._redis = None
|
||||
self._redis_healthy = False
|
||||
logger.debug("realtime: redis unavailable (%s); in-process only", exc)
|
||||
return None
|
||||
return self._redis
|
||||
|
||||
# -- relay (other processes -> this one's browsers) ----------------------
|
||||
|
||||
async def start_relay(self) -> None:
|
||||
"""Subscribe to the shared channel and feed the local manager."""
|
||||
if self._relay_task is not None and not self._relay_task.done():
|
||||
return
|
||||
self._relay_task = asyncio.create_task(self._relay_loop(), name="realtime-relay")
|
||||
|
||||
async def stop_relay(self) -> None:
|
||||
task = self._relay_task
|
||||
self._relay_task = None
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
await self._close_redis()
|
||||
|
||||
async def _relay_loop(self) -> None:
|
||||
while True:
|
||||
client = await self._redis_client()
|
||||
if client is None:
|
||||
# Broker absent (or down mid-flight): wait out the cooldown and
|
||||
# try again — a Redis started after the API must still work.
|
||||
await asyncio.sleep(REDIS_RETRY_COOLDOWN_SECONDS)
|
||||
continue
|
||||
try:
|
||||
pubsub = client.pubsub()
|
||||
await pubsub.subscribe(self._channel)
|
||||
self._pubsub = pubsub
|
||||
logger.info("realtime: relay subscribed to %s", self._channel)
|
||||
async for message in pubsub.listen():
|
||||
if message.get("type") != "message":
|
||||
continue
|
||||
event = _decode_message(message.get("data"))
|
||||
if event is not None:
|
||||
# Dedupe by id happens in the manager, so an event this
|
||||
# process published itself is not delivered twice.
|
||||
await self._dispatch_local(event)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - relay must survive
|
||||
self._redis_healthy = False
|
||||
logger.warning("realtime: relay interrupted (%s); reconnecting", exc)
|
||||
finally:
|
||||
if self._pubsub is not None:
|
||||
with suppress(Exception):
|
||||
await self._pubsub.unsubscribe(self._channel)
|
||||
self._pubsub = None
|
||||
self._redis = None
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
async def _close_redis(self) -> None:
|
||||
client, self._redis = self._redis, None
|
||||
if client is not None:
|
||||
with suppress(Exception):
|
||||
await client.aclose()
|
||||
self._redis_healthy = False
|
||||
|
||||
# -- introspection -------------------------------------------------------
|
||||
|
||||
@property
|
||||
def redis_healthy(self) -> bool:
|
||||
return self._redis_healthy
|
||||
|
||||
@property
|
||||
def listener_count(self) -> int:
|
||||
return len(self._listeners)
|
||||
|
||||
|
||||
def _decode_message(raw: object) -> RealtimeEvent | None:
|
||||
if not isinstance(raw, (str, bytes)):
|
||||
return None
|
||||
try:
|
||||
return RealtimeEvent.from_json(raw)
|
||||
except Exception: # noqa: BLE001 - a bad frame is skipped, not fatal
|
||||
logger.warning("realtime: undecodable relay frame dropped")
|
||||
return None
|
||||
|
||||
|
||||
_bus: RealtimeBus | None = None
|
||||
|
||||
|
||||
def get_bus() -> RealtimeBus:
|
||||
"""Process-wide bus. Created lazily so importing never opens a socket."""
|
||||
global _bus
|
||||
if _bus is None:
|
||||
_bus = RealtimeBus()
|
||||
return _bus
|
||||
|
||||
|
||||
def install() -> RealtimeBus:
|
||||
"""Make this process's bus the application-layer publisher.
|
||||
|
||||
Called by the composition roots (``app.main`` for the API,
|
||||
``app.workers.__main__`` for the workers). Idempotent, so a worker that
|
||||
imports several modules pays nothing for calling it more than once.
|
||||
"""
|
||||
bus = get_bus()
|
||||
set_publisher(bus)
|
||||
return bus
|
||||
|
||||
|
||||
async def publish(event: RealtimeEvent) -> None:
|
||||
"""Publish through the shared bus. Never raises."""
|
||||
bus = get_bus()
|
||||
try:
|
||||
await bus.publish(event)
|
||||
except Exception: # noqa: BLE001 - belt and braces; publish is best-effort
|
||||
logger.exception("realtime: publish failed for %s", event.type)
|
||||
|
||||
|
||||
async def start_relay() -> None:
|
||||
await get_bus().start_relay()
|
||||
|
||||
|
||||
async def stop_relay() -> None:
|
||||
await get_bus().stop_relay()
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Connection bookkeeping: who is connected, and to which channels.
|
||||
|
||||
The manager is deliberately transport-agnostic — it does not know it is talking
|
||||
to a WebSocket, only to an object with ``send_text``. That keeps it testable
|
||||
without a socket and lets a future transport (SSE, push bridge) reuse it.
|
||||
|
||||
Two invariants:
|
||||
|
||||
* **A connection only ever receives channels it was authorized for.** The
|
||||
authorizer runs at subscribe time; the routing table built afterwards can
|
||||
only contain pre-approved channel names.
|
||||
* **An event is delivered at most once.** ``dispatch`` de-duplicates by event
|
||||
id, because the same event reaches a process twice in the normal case: once
|
||||
in-process and once relayed from Redis.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from starlette.websockets import WebSocket, WebSocketState
|
||||
|
||||
from app.application.ports.realtime import RealtimeEvent
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
#: Bounded dedupe memory. Large enough that a relayed duplicate is always
|
||||
#: caught, small enough that a chatty day costs a few kilobytes.
|
||||
MAX_SEEN_EVENTS = 2048
|
||||
|
||||
#: Per-connection outbox. If a browser stops reading (a frozen tab), a
|
||||
#: producer must not be able to grow this without bound. Overflow drops the
|
||||
#: oldest frame — ordering of what follows stays meaningful.
|
||||
MAX_QUEUE_DEPTH = 256
|
||||
|
||||
#: What a subscribe check looks like: ``(connection, channel) -> allowed``.
|
||||
ChannelAuthorizer = Callable[["Connection", str], Awaitable[bool]]
|
||||
|
||||
|
||||
#: ``eq=False`` on purpose: a connection is identified by object identity. The
|
||||
#: default dataclass ``__eq__`` would also make it unhashable (a ``set`` field
|
||||
#: cannot be compared safely), and the manager keys connections by identity in
|
||||
#: both its connection set and its channel routing table.
|
||||
@dataclass(slots=True, eq=False)
|
||||
class Connection:
|
||||
"""One authenticated browser socket."""
|
||||
|
||||
user_id: str
|
||||
is_platform_owner: bool
|
||||
websocket: WebSocket
|
||||
channels: set[str] = field(default_factory=set)
|
||||
_queue: asyncio.Queue[str] = field(
|
||||
default_factory=lambda: asyncio.Queue(maxsize=MAX_QUEUE_DEPTH)
|
||||
)
|
||||
_sender: asyncio.Task | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Single writer task per connection.
|
||||
|
||||
One task owns the socket: concurrent ``send_text`` calls on one
|
||||
WebSocket interleave frames and corrupt the stream, so everything goes
|
||||
through this queue instead.
|
||||
"""
|
||||
if self._sender is None or self._sender.done():
|
||||
self._sender = asyncio.create_task(self._drain(), name="realtime-sender")
|
||||
|
||||
async def _drain(self) -> None:
|
||||
while True:
|
||||
payload = await self._queue.get()
|
||||
if payload is None: # shutdown sentinel
|
||||
return
|
||||
try:
|
||||
if self.websocket.client_state == WebSocketState.CONNECTED:
|
||||
await self.websocket.send_text(payload)
|
||||
except Exception: # noqa: BLE001 - a dead socket ends the connection
|
||||
logger.debug("realtime: send failed; closing connection", exc_info=True)
|
||||
return
|
||||
|
||||
def offer(self, payload: str) -> None:
|
||||
"""Queue a frame, never blocking the publisher.
|
||||
|
||||
A slow consumer loses the oldest frames rather than stalling every
|
||||
other tenant's stream behind it.
|
||||
"""
|
||||
if self._queue.full():
|
||||
with suppress(asyncio.QueueFull):
|
||||
# Drop the oldest frame to make room: ordering of what follows
|
||||
# stays meaningful, which is what the UI renders.
|
||||
self._queue.get_nowait()
|
||||
with suppress(asyncio.QueueFull): # pragma: no cover - defensive
|
||||
self._queue.put_nowait(payload)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._sender is not None:
|
||||
self._sender.cancel()
|
||||
self._sender = None
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
"""The routing table between events and open sockets."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._connections: set[Connection] = set()
|
||||
self._by_channel: dict[str, set[Connection]] = {}
|
||||
self._seen: deque[str] = deque(maxlen=MAX_SEEN_EVENTS)
|
||||
self._seen_set: set[str] = set()
|
||||
|
||||
# -- lifecycle -----------------------------------------------------------
|
||||
|
||||
async def register(self, connection: Connection) -> None:
|
||||
self._connections.add(connection)
|
||||
await connection.start()
|
||||
|
||||
async def unregister(self, connection: Connection) -> None:
|
||||
self._connections.discard(connection)
|
||||
for channel in list(connection.channels):
|
||||
subscribers = self._by_channel.get(channel)
|
||||
if subscribers is not None:
|
||||
subscribers.discard(connection)
|
||||
if not subscribers:
|
||||
self._by_channel.pop(channel, None)
|
||||
connection.channels.clear()
|
||||
await connection.close()
|
||||
|
||||
# -- subscriptions ---------------------------------------------------------
|
||||
|
||||
async def subscribe(
|
||||
self,
|
||||
connection: Connection,
|
||||
channels: list[str],
|
||||
authorizer: ChannelAuthorizer,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Authorize and subscribe. Returns ``(accepted, rejected)``."""
|
||||
accepted: list[str] = []
|
||||
rejected: list[str] = []
|
||||
for channel in channels:
|
||||
if channel in connection.channels:
|
||||
accepted.append(channel)
|
||||
continue
|
||||
try:
|
||||
allowed = await authorizer(connection, channel)
|
||||
except Exception: # noqa: BLE001 - a broken check denies, never allows
|
||||
logger.exception("realtime: channel authorization crashed")
|
||||
allowed = False
|
||||
if not allowed:
|
||||
rejected.append(channel)
|
||||
continue
|
||||
connection.channels.add(channel)
|
||||
self._by_channel.setdefault(channel, set()).add(connection)
|
||||
accepted.append(channel)
|
||||
return accepted, rejected
|
||||
|
||||
async def unsubscribe(self, connection: Connection, channels: list[str]) -> None:
|
||||
for channel in channels:
|
||||
connection.channels.discard(channel)
|
||||
subscribers = self._by_channel.get(channel)
|
||||
if subscribers is not None:
|
||||
subscribers.discard(connection)
|
||||
if not subscribers:
|
||||
self._by_channel.pop(channel, None)
|
||||
|
||||
# -- fan-out ---------------------------------------------------------------
|
||||
|
||||
async def dispatch(self, event: RealtimeEvent) -> int:
|
||||
"""Deliver to every subscriber of every channel. Returns fan-out count.
|
||||
|
||||
The id check is what makes "publish locally, then receive the same event
|
||||
from Redis" deliver once instead of twice.
|
||||
"""
|
||||
if event.id in self._seen_set:
|
||||
return 0
|
||||
self._remember(event.id)
|
||||
|
||||
recipients: set[Connection] = set()
|
||||
for channel in event.channels:
|
||||
recipients.update(self._by_channel.get(channel, ()))
|
||||
|
||||
envelope = event.envelope()
|
||||
# Control frames and event frames share one top-level discriminator
|
||||
# (``type``), so the client switches on one field for everything.
|
||||
payload = _dumps({"type": "event", "event": envelope})
|
||||
delivered = 0
|
||||
for connection in recipients:
|
||||
connection.offer(payload)
|
||||
delivered += 1
|
||||
return delivered
|
||||
|
||||
def _remember(self, event_id: str) -> None:
|
||||
if len(self._seen) == self._seen.maxlen:
|
||||
evicted = self._seen[0]
|
||||
self._seen_set.discard(evicted)
|
||||
self._seen.append(event_id)
|
||||
self._seen_set.add(event_id)
|
||||
|
||||
# -- introspection -------------------------------------------------------
|
||||
|
||||
@property
|
||||
def connection_count(self) -> int:
|
||||
return len(self._connections)
|
||||
|
||||
@property
|
||||
def channel_count(self) -> int:
|
||||
return len(self._by_channel)
|
||||
|
||||
def channel_load(self) -> dict[str, int]:
|
||||
return {channel: len(subs) for channel, subs in self._by_channel.items()}
|
||||
|
||||
|
||||
def _dumps(envelope: dict) -> str:
|
||||
import json
|
||||
|
||||
return json.dumps(envelope, ensure_ascii=False, default=str)
|
||||
@@ -0,0 +1,175 @@
|
||||
"""The WebSocket endpoint — one socket per browser, every live feed on it.
|
||||
|
||||
``GET /api/ws?token=<jwt>``
|
||||
|
||||
The token travels as a query parameter because the browser WebSocket API cannot
|
||||
set an ``Authorization`` header. That is a deliberate, contained exception to
|
||||
the header-only rule: the token is short-lived, the socket is TLS-terminated in
|
||||
any real deployment, and the alternative (a cookie) would drag CSRF into a
|
||||
protocol that has no CSRF story.
|
||||
|
||||
Protocol — small, symmetric, JSON in both directions:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
client -> {"action": "subscribe", "channels": ["org:…", "task:…"]}
|
||||
client -> {"action": "unsubscribe", "channels": ["task:…"]}
|
||||
client -> {"action": "ping"}
|
||||
server -> {"type": "hello", "user_id": "…"}
|
||||
server -> {"type": "subscribed", "accepted": [...], "rejected": [...]}
|
||||
server -> {"type": "event", "event": {…envelope…}}
|
||||
server -> {"type": "pong"}
|
||||
server -> {"type": "error", "message": "…"}
|
||||
|
||||
A rejected channel is reported, not hidden: "my task page is not live" must be
|
||||
answerable from the browser console alone.
|
||||
|
||||
Authorization queries open their own short-lived session: a socket can stay
|
||||
open for hours, and one SQLAlchemy session per open tab would quietly exhaust
|
||||
the connection pool.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Query, WebSocket, WebSocketDisconnect
|
||||
|
||||
from app.application.ports.realtime import dedupe_channels
|
||||
from app.core.database import SessionFactory
|
||||
from app.core.errors import AuthenticationError
|
||||
from app.core.security import decode_access_token
|
||||
from app.infrastructure.repositories.identity import UserRepository
|
||||
from app.interfaces.realtime.authz import authorize_channel
|
||||
from app.interfaces.realtime.manager import Connection, ConnectionManager
|
||||
|
||||
logger = logging.getLogger("agy.realtime")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
#: The manager instance this process fans out with. Created at import: the
|
||||
#: endpoint and the lifecycle hooks can share it without a wiring ceremony.
|
||||
_manager = ConnectionManager()
|
||||
|
||||
|
||||
def realtime_manager() -> ConnectionManager:
|
||||
"""The process-wide manager — attached to the bus by ``app.main``."""
|
||||
return _manager
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def realtime_socket(
|
||||
websocket: WebSocket,
|
||||
token: str | None = Query(default=None),
|
||||
) -> None:
|
||||
connection = await _authenticate(websocket, token)
|
||||
if connection is None:
|
||||
return # _authenticate already closed with a policy-violation code
|
||||
|
||||
await _manager.register(connection)
|
||||
|
||||
await _send(
|
||||
websocket,
|
||||
{"type": "hello", "user_id": connection.user_id, "server_time": _now_iso()},
|
||||
)
|
||||
|
||||
try:
|
||||
while True:
|
||||
raw = await websocket.receive_text()
|
||||
await _handle_message(websocket, connection, raw)
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception: # noqa: BLE001 - any read failure ends the socket cleanly
|
||||
logger.debug("realtime: socket loop ended", exc_info=True)
|
||||
finally:
|
||||
await _manager.unregister(connection)
|
||||
|
||||
|
||||
async def _authenticate(websocket: WebSocket, token: str | None) -> Connection | None:
|
||||
"""Accept only a valid, active user. Closes 1008 otherwise."""
|
||||
user = None
|
||||
if token:
|
||||
try:
|
||||
payload = decode_access_token(token)
|
||||
user_id = payload.get("sub")
|
||||
if user_id:
|
||||
# Own session: the socket must not depend on request scope.
|
||||
async with SessionFactory() as session:
|
||||
user = await UserRepository(session).get(str(user_id))
|
||||
except AuthenticationError:
|
||||
user = None
|
||||
if user is None or not user.is_active:
|
||||
# 1008 (policy violation) rather than a bare close: the client's
|
||||
# reconnect logic can then tell "bad token" from "network dropped".
|
||||
await websocket.close(code=1008, reason="unauthorized")
|
||||
return None
|
||||
await websocket.accept()
|
||||
return Connection(
|
||||
user_id=user.id,
|
||||
is_platform_owner=user.is_platform_owner,
|
||||
websocket=websocket,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_message(websocket: WebSocket, connection: Connection, raw: str) -> None:
|
||||
try:
|
||||
message = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
await _send(websocket, {"type": "error", "message": "malformed JSON frame"})
|
||||
return
|
||||
if not isinstance(message, dict):
|
||||
await _send(websocket, {"type": "error", "message": "frame must be an object"})
|
||||
return
|
||||
|
||||
action = message.get("action")
|
||||
|
||||
if action == "ping":
|
||||
await _send(websocket, {"type": "pong"})
|
||||
return
|
||||
|
||||
channels = dedupe_channels(
|
||||
str(channel) for channel in (message.get("channels") or []) if isinstance(channel, str)
|
||||
)
|
||||
|
||||
if action == "subscribe":
|
||||
accepted, rejected = await _subscribe(connection, channels)
|
||||
await _send(
|
||||
websocket, {"type": "subscribed", "accepted": accepted, "rejected": rejected}
|
||||
)
|
||||
return
|
||||
|
||||
if action == "unsubscribe":
|
||||
await _manager.unsubscribe(connection, list(channels))
|
||||
await _send(websocket, {"type": "unsubscribed", "channels": channels})
|
||||
return
|
||||
|
||||
await _send(websocket, {"type": "error", "message": f"unknown action: {action!r}"})
|
||||
|
||||
|
||||
async def _subscribe(
|
||||
connection: Connection, channels: tuple[str, ...]
|
||||
) -> tuple[list[str], list[str]]:
|
||||
if not channels:
|
||||
return [], []
|
||||
|
||||
async def _authorize(conn: Connection, channel: str) -> bool:
|
||||
async with SessionFactory() as session:
|
||||
return await authorize_channel(
|
||||
session,
|
||||
user_id=conn.user_id,
|
||||
is_platform_owner=conn.is_platform_owner,
|
||||
channel=channel,
|
||||
)
|
||||
|
||||
return await _manager.subscribe(connection, list(channels), _authorize)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from app.domain.base import utcnow
|
||||
|
||||
return utcnow().isoformat()
|
||||
|
||||
|
||||
async def _send(websocket: WebSocket, payload: dict) -> None:
|
||||
await websocket.send_text(json.dumps(payload, ensure_ascii=False, default=str))
|
||||
Reference in New Issue
Block a user