services: update ingestion collector, paced publisher, and deduplication

services/collector.py

@@ -1,5 +1,6 @@
 import os
 import logging
+from dataclasses import dataclass
 from typing import Optional, Callable, Awaitable
 from telethon import TelegramClient, events
 from telethon.errors import SessionPasswordNeededError
@@ -7,7 +8,7 @@ from telethon.tl.types import MessageMediaPhoto, MessageMediaDocument
 from db.repository import Repository
 from core.dedup import compute_content_hash, compute_file_hash
 from core.queue import RedisQueue
-from core.metrics import COLLECTED_POSTS_TOTAL, SOURCE_ACTIVITY_TOTAL
+from core.metrics import SOURCE_ACTIVITY_TOTAL, DUPLICATES_DETECTED_TOTAL
 from core.proxy import get_telegram_proxy
 from core.error_logger import log_exception
 
@@ -16,6 +17,40 @@ logger = logging.getLogger(__name__)
 SESSION_DIR = os.getenv("SESSION_DIR", "/app/sessions" if os.path.exists("/app") else "/projects/telegram-bots/copykar/sessions")
 MEDIA_DIR = os.getenv("MEDIA_DIR", "/app/data/media" if os.path.exists("/app") else "/projects/telegram-bots/copykar/data/media")
 
+# Bounds for the "custom count" prompt in the admin bot.
+MIN_FETCH_LIMIT = 1
+MAX_FETCH_LIMIT = 500
+
+
+@dataclass
+class ScrapeResult:
+    """Outcome of a history scrape.
+
+    'collected' alone is misleading: a repeat scrape of the same window legitimately
+    adds nothing, which reads as a broken button unless the other counters are shown.
+    """
+    scanned: int = 0
+    collected: int = 0
+    already_stored: int = 0
+    duplicates: int = 0
+    error: Optional[str] = None
+
+    def summary_fa(self, channel_id: int) -> str:
+        if self.error:
+            return f"❌ خطا در دریافت پست‌های کانال <code>{channel_id}</code>: {self.error}"
+        header = (
+            f"✅ <b>{self.collected}</b> پست جدید از <code>{channel_id}</code> دریافت و به کانال ادمین ارسال شد."
+            if self.collected
+            else f"ℹ️ هیچ پست <b>جدیدی</b> در <code>{channel_id}</code> پیدا نشد."
+        )
+        return (
+            f"{header}\n\n"
+            f"• 🔍 پیام بررسی‌شده: <b>{self.scanned}</b>\n"
+            f"• 🆕 پست جدید: <b>{self.collected}</b>\n"
+            f"• 🗂 قبلا ذخیره شده: <b>{self.already_stored}</b>\n"
+            f"• ♻️ محتوای تکراری: <b>{self.duplicates}</b>"
+        )
+
 class CollectorService:
     def __init__(
         self,
@@ -136,6 +171,10 @@ class CollectorService:
 
     async def _handle_message(self, event: events.NewMessage.Event):
         try:
+            if await self.repo.is_system_paused():
+                logger.info(f"[collector] System is paused by admin. Ignoring incoming post from chat {event.chat_id}")
+                return
+
             chat_id = event.chat_id
             source = await self.repo.get_source_by_channel_id(chat_id)
             if not source or not source.is_active:
@@ -162,6 +201,7 @@ class CollectorService:
                     media_hash = compute_file_hash(downloaded_file)
 
             content_hash = compute_content_hash(raw_text, media_hash)
+            duplicate_of = await self.repo.find_duplicate_post(content_hash)
 
             post_id = await self.repo.create_raw_post(
                 source_channel_id=chat_id,
@@ -170,10 +210,15 @@ class CollectorService:
                 media_path=media_path,
                 media_type=media_type,
                 content_hash=content_hash,
+                is_duplicate=duplicate_of is not None,
+                duplicate_of_id=duplicate_of.id if duplicate_of else None,
+                similarity_reason=f"content hash matches post #{duplicate_of.id}" if duplicate_of else None,
             )
 
+            if post_id and duplicate_of:
+                DUPLICATES_DETECTED_TOTAL.labels(method="content_hash").inc()
+
             if post_id:
-                COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(chat_id)).inc()
                 SOURCE_ACTIVITY_TOTAL.labels(channel_id=str(chat_id), title=source.title or 'Unknown').inc()
                 logger.info(f"Collected raw post ID {post_id} from source channel {chat_id}")
 
@@ -187,20 +232,21 @@ class CollectorService:
         channel_id: int,
         limit: int = 20,
         progress_callback: Optional[Callable[[str], Awaitable[None]]] = None
-    ) -> int:
+    ) -> ScrapeResult:
         """Scrape historical messages from a source channel."""
+        result = ScrapeResult()
+
         if not self.client.is_connected() or not await self.client.is_user_authorized():
             if progress_callback:
                 await progress_callback("❌ ربات متصل نیست. لطفا ابتدا لاگین کنید.")
-            return 0
-
-        source = await self.repo.get_source_by_channel_id(channel_id)
-        source_title = source.title if source else str(channel_id)
-        username = source.username if source else None
-        collected_count = 0
-        skipped_count = 0
+            result.error = "not connected"
+            return result
 
         try:
+            source = await self.repo.get_source_by_channel_id(channel_id)
+            source_title = source.title if source else str(channel_id)
+            username = source.username if source else None
+
             entity = await self._resolve_channel_entity(channel_id, username)
             messages = []
             async for msg in self.client.iter_messages(entity, limit=limit):
@@ -208,6 +254,8 @@ class CollectorService:
 
             messages.reverse()
 
+            result.scanned = len(messages)
+
             for message in messages:
                 raw_text = message.raw_text or ""
                 if not raw_text and not message.media:
@@ -233,6 +281,10 @@ class CollectorService:
                         media_hash = compute_file_hash(downloaded_file)
 
                 content_hash = compute_content_hash(raw_text, media_hash)
+                duplicate_of = await self.repo.find_duplicate_post(content_hash)
+                if duplicate_of:
+                    result.duplicates += 1
+                    DUPLICATES_DETECTED_TOTAL.labels(method="content_hash").inc()
 
                 post_id = await self.repo.create_raw_post(
                     source_channel_id=channel_id,
@@ -241,29 +293,30 @@ class CollectorService:
                     media_path=media_path,
                     media_type=media_type,
                     content_hash=content_hash,
+                    is_duplicate=duplicate_of is not None,
+                    duplicate_of_id=duplicate_of.id if duplicate_of else None,
+                    similarity_reason=f"content hash matches post #{duplicate_of.id}" if duplicate_of else None,
                 )
 
                 if post_id:
-                    collected_count += 1
-                    COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(channel_id)).inc()
+                    result.collected += 1
                     SOURCE_ACTIVITY_TOTAL.labels(channel_id=str(channel_id), title=source_title).inc()
                     logger.info(f"Backfilled raw post ID {post_id} from {channel_id}")
 
                     if self.on_post_received:
                         await self.on_post_received(post_id)
                 else:
-                    skipped_count += 1
+                    result.already_stored += 1
 
             if progress_callback:
-                await progress_callback(
-                    f"✅ تعداد <b>{collected_count}</b> پست جدید از <code>{channel_id}</code> دریافت و در کانال ادمین قرار گرفت! (رد شده تکراری: {skipped_count})."
-                )
-            return collected_count
+                await progress_callback(result.summary_fa(channel_id))
+            return result
         except Exception as e:
             await log_exception("collector.scrape_history", e, {"channel_id": channel_id, "limit": limit})
+            result.error = str(e)
             if progress_callback:
-                await progress_callback(f"❌ خطا در دریافت پست‌های کانال <code>{channel_id}</code>: {e}")
-            return collected_count
+                await progress_callback(result.summary_fa(channel_id))
+            return result
 
     async def stop(self):
         if self.client.is_connected():