feat(queue): integrate redis queue and 2-minute paced ai consumer worker

2026-08-27 21:14:51 +03:30
parent f23ce3a383
commit 1510af647c
8 changed files with 136 additions and 5 deletions
@@ -0,0 +1,49 @@
+import os
+import asyncio
+import logging
+from typing import Optional
+from core.queue import RedisQueue
+from services.ai_processor import AIProcessor
+from core.metrics import QUEUE_POSTS_GAUGE
+
+logger = logging.getLogger(__name__)
+
+FETCH_INTERVAL_SECONDS = int(os.getenv("AI_PROCESSING_INTERVAL_SECONDS", "120"))
+
+class QueueConsumerService:
+    def __init__(self, queue: RedisQueue, ai_processor: AIProcessor, fetch_interval: int = FETCH_INTERVAL_SECONDS):
+        self.queue = queue
+        self.ai_processor = ai_processor
+        self.fetch_interval = fetch_interval
+        self._running = False
+        self._task: Optional[asyncio.Task] = None
+
+    async def start(self):
+        logger.info(f"Starting Redis Queue Consumer Service (interval: {self.fetch_interval}s / {self.fetch_interval // 60}m)...")
+        await self.queue.connect()
+        self._running = True
+        self._task = asyncio.create_task(self._consumer_loop())
+
+    async def _consumer_loop(self):
+        while self._running:
+            try:
+                qsize = await self.queue.qsize()
+                QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(qsize)
+
+                if qsize > 0:
+                    post_id = await self.queue.pop()
+                    if post_id:
+                        logger.info(f"Paced Consumer: processing post ID {post_id} from Redis queue (remaining: {qsize - 1})")
+                        await self.ai_processor.process_post(post_id)
+                        new_qsize = await self.queue.qsize()
+                        QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(new_qsize)
+            except Exception as e:
+                logger.error(f"Error in queue consumer loop: {e}", exc_info=True)
+
+            await asyncio.sleep(self.fetch_interval)
+
+    async def stop(self):
+        self._running = False
+        if self._task:
+            self._task.cancel()
+        logger.info("Redis Queue Consumer Service stopped.")