feat(queue): integrate redis queue and 2-minute paced ai consumer worker

services/admin_bot.py

@@ -165,9 +165,11 @@ class AdminBotService:
             approved = len(await self.repo.get_posts_by_status("approved", limit=1000))
             published = len(await self.repo.get_posts_by_status("published", limit=1000))
             rejected = len(await self.repo.get_posts_by_status("rejected", limit=1000))
+            redis_q = await self.collector.queue.qsize() if (self.collector and self.collector.queue) else 0
 
             text = (
                 "📊 <b>Copykar Fleet Metrics</b>\n\n"
+                f"• 📥 <b>Redis Incoming Queue:</b> {redis_q} (Pacing: 1 post / 2m)\n"
                 f"• ⏳ <b>Pending AI:</b> {pending_ai}\n"
                 f"• 📋 <b>Pending Review:</b> {pending_review}\n"
                 f"• 🚀 <b>Approved (In Queue):</b> {approved}\n"

services/collector.py

@@ -7,6 +7,7 @@ from telethon.tl.types import MessageMediaPhoto, MessageMediaDocument
 from db.repository import Repository
 from core.dedup import compute_content_hash, compute_file_hash
 from services.ai_processor import AIProcessor
+from core.queue import RedisQueue
 from core.metrics import COLLECTED_POSTS_TOTAL
 from core.proxy import get_telegram_proxy
 
@@ -20,6 +21,7 @@ class CollectorService:
         self,
         repo: Repository,
         ai_processor: AIProcessor,
+        queue: Optional[RedisQueue] = None,
         api_id: Optional[int] = None,
         api_hash: Optional[str] = None,
         phone: Optional[str] = None,
@@ -27,6 +29,7 @@ class CollectorService:
     ):
         self.repo = repo
         self.ai_processor = ai_processor
+        self.queue = queue
         self.api_id = api_id or int(os.getenv("API_ID", "0"))
         self.api_hash = api_hash or os.getenv("API_HASH", "")
         self.phone = phone or os.getenv("PHONE")
@@ -139,7 +142,10 @@ class CollectorService:
             if post_id:
                 COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(chat_id)).inc()
                 logger.info(f"Collected new post ID {post_id} from channel {chat_id}")
-                await self.ai_processor.process_post(post_id)
+                if self.queue:
+                    await self.queue.push(post_id)
+                else:
+                    await self.ai_processor.process_post(post_id)
         except Exception as e:
             logger.error(f"Error handling message from {event.chat_id}: {e}", exc_info=True)
 
@@ -205,13 +211,16 @@ class CollectorService:
                     collected_count += 1
                     COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(channel_id)).inc()
                     logger.info(f"Backfilled historical post ID {post_id} from {channel_id}")
-                    await self.ai_processor.process_post(post_id)
+                    if self.queue:
+                        await self.queue.push(post_id)
+                    else:
+                        await self.ai_processor.process_post(post_id)
                 else:
                     skipped_count += 1
 
             if progress_callback:
                 await progress_callback(
-                    f"✅ Scraped <b>{collected_count}</b> new historical posts from <code>{channel_id}</code> (Skipped {skipped_count} existing/empty)."
+                    f"✅ Scraped <b>{collected_count}</b> new posts from <code>{channel_id}</code> and queued in Redis! (Skipped {skipped_count} existing/empty)."
                 )
             return collected_count
         except Exception as e:

services/queue_consumer.py

@@ -0,0 +1,49 @@
+import os
+import asyncio
+import logging
+from typing import Optional
+from core.queue import RedisQueue
+from services.ai_processor import AIProcessor
+from core.metrics import QUEUE_POSTS_GAUGE
+
+logger = logging.getLogger(__name__)
+
+FETCH_INTERVAL_SECONDS = int(os.getenv("AI_PROCESSING_INTERVAL_SECONDS", "120"))
+
+class QueueConsumerService:
+    def __init__(self, queue: RedisQueue, ai_processor: AIProcessor, fetch_interval: int = FETCH_INTERVAL_SECONDS):
+        self.queue = queue
+        self.ai_processor = ai_processor
+        self.fetch_interval = fetch_interval
+        self._running = False
+        self._task: Optional[asyncio.Task] = None
+
+    async def start(self):
+        logger.info(f"Starting Redis Queue Consumer Service (interval: {self.fetch_interval}s / {self.fetch_interval // 60}m)...")
+        await self.queue.connect()
+        self._running = True
+        self._task = asyncio.create_task(self._consumer_loop())
+
+    async def _consumer_loop(self):
+        while self._running:
+            try:
+                qsize = await self.queue.qsize()
+                QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(qsize)
+
+                if qsize > 0:
+                    post_id = await self.queue.pop()
+                    if post_id:
+                        logger.info(f"Paced Consumer: processing post ID {post_id} from Redis queue (remaining: {qsize - 1})")
+                        await self.ai_processor.process_post(post_id)
+                        new_qsize = await self.queue.qsize()
+                        QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(new_qsize)
+            except Exception as e:
+                logger.error(f"Error in queue consumer loop: {e}", exc_info=True)
+
+            await asyncio.sleep(self.fetch_interval)
+
+    async def stop(self):
+        self._running = False
+        if self._task:
+            self._task.cancel()
+        logger.info("Redis Queue Consumer Service stopped.")