import os import asyncio import logging from typing import Optional from core.queue import RedisQueue from services.ai_processor import AIProcessor from core.metrics import QUEUE_POSTS_GAUGE logger = logging.getLogger(__name__) FETCH_INTERVAL_SECONDS = int(os.getenv("AI_PROCESSING_INTERVAL_SECONDS", "120")) class QueueConsumerService: def __init__(self, queue: RedisQueue, ai_processor: AIProcessor, fetch_interval: int = FETCH_INTERVAL_SECONDS): self.queue = queue self.ai_processor = ai_processor self.fetch_interval = fetch_interval self._running = False self._task: Optional[asyncio.Task] = None async def start(self): logger.info(f"Starting Redis Queue Consumer Service (interval: {self.fetch_interval}s / {self.fetch_interval // 60}m)...") await self.queue.connect() self._running = True self._task = asyncio.create_task(self._consumer_loop()) async def _consumer_loop(self): while self._running: try: qsize = await self.queue.qsize() QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(qsize) if qsize > 0: post_id = await self.queue.pop() if post_id: logger.info(f"Paced Consumer: processing post ID {post_id} from Redis queue (remaining: {qsize - 1})") await self.ai_processor.process_post(post_id) new_qsize = await self.queue.qsize() QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(new_qsize) except Exception as e: logger.error(f"Error in queue consumer loop: {e}", exc_info=True) await asyncio.sleep(self.fetch_interval) async def stop(self): self._running = False if self._task: self._task.cancel() logger.info("Redis Queue Consumer Service stopped.")