root/copykar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

services: update ingestion collector, paced publisher, and deduplication

Browse Source
This commit is contained in:
mamad
2026-08-28 19:33:29 +03:30
parent db2e726a57
commit 2137a1158d
9 changed files with 494 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/collector.py
+72 -19
View File
@@ -1,5 +1,6 @@
import os
import logging
from dataclasses import dataclass
from typing import Optional, Callable, Awaitable
from telethon import TelegramClient, events
from telethon.errors import SessionPasswordNeededError
@@ -7,7 +8,7 @@ from telethon.tl.types import MessageMediaPhoto, MessageMediaDocument
from db.repository import Repository
from core.dedup import compute_content_hash, compute_file_hash
from core.queue import RedisQueue
from core.metrics import COLLECTED_POSTS_TOTAL, SOURCE_ACTIVITY_TOTAL
from core.metrics import SOURCE_ACTIVITY_TOTAL, DUPLICATES_DETECTED_TOTAL
from core.proxy import get_telegram_proxy
from core.error_logger import log_exception
@@ -16,6 +17,40 @@ logger = logging.getLogger(__name__)
SESSION_DIR = os.getenv("SESSION_DIR", "/app/sessions" if os.path.exists("/app") else "/projects/telegram-bots/copykar/sessions")
MEDIA_DIR = os.getenv("MEDIA_DIR", "/app/data/media" if os.path.exists("/app") else "/projects/telegram-bots/copykar/data/media")
# Bounds for the "custom count" prompt in the admin bot.
MIN_FETCH_LIMIT = 1
MAX_FETCH_LIMIT = 500
@dataclass
class ScrapeResult:
"""Outcome of a history scrape.
'collected' alone is misleading: a repeat scrape of the same window legitimately
adds nothing, which reads as a broken button unless the other counters are shown.
"""
scanned: int = 0
collected: int = 0
already_stored: int = 0
duplicates: int = 0
error: Optional[str] = None
def summary_fa(self, channel_id: int) -> str:
if self.error:
return f"❌ خطا در دریافت پست‌های کانال <code>{channel_id}</code>: {self.error}"
header = (
f"✅ <b>{self.collected}</b> پست جدید از <code>{channel_id}</code> دریافت و به کانال ادمین ارسال شد."
if self.collected
else f"ℹ️ هیچ پست <b>جدیدی</b> در <code>{channel_id}</code> پیدا نشد."
)
return (
f"{header}\n\n"
f"• 🔍 پیام بررسی‌شده: <b>{self.scanned}</b>\n"
f"• 🆕 پست جدید: <b>{self.collected}</b>\n"
f"• 🗂 قبلا ذخیره شده: <b>{self.already_stored}</b>\n"
f"• ♻️ محتوای تکراری: <b>{self.duplicates}</b>"
)
class CollectorService:
def __init__(
self,
@@ -136,6 +171,10 @@ class CollectorService:
async def _handle_message(self, event: events.NewMessage.Event):
try:
if await self.repo.is_system_paused():
logger.info(f"[collector] System is paused by admin. Ignoring incoming post from chat {event.chat_id}")
return
chat_id = event.chat_id
source = await self.repo.get_source_by_channel_id(chat_id)
if not source or not source.is_active:
@@ -162,6 +201,7 @@ class CollectorService:
media_hash = compute_file_hash(downloaded_file)
content_hash = compute_content_hash(raw_text, media_hash)
duplicate_of = await self.repo.find_duplicate_post(content_hash)
post_id = await self.repo.create_raw_post(
source_channel_id=chat_id,
@@ -170,10 +210,15 @@ class CollectorService:
media_path=media_path,
media_type=media_type,
content_hash=content_hash,
is_duplicate=duplicate_of is not None,
duplicate_of_id=duplicate_of.id if duplicate_of else None,
similarity_reason=f"content hash matches post #{duplicate_of.id}" if duplicate_of else None,
)
if post_id and duplicate_of:
DUPLICATES_DETECTED_TOTAL.labels(method="content_hash").inc()
if post_id:
COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(chat_id)).inc()
SOURCE_ACTIVITY_TOTAL.labels(channel_id=str(chat_id), title=source.title or 'Unknown').inc()
logger.info(f"Collected raw post ID {post_id} from source channel {chat_id}")
@@ -187,20 +232,21 @@ class CollectorService:
channel_id: int,
limit: int = 20,
progress_callback: Optional[Callable[[str], Awaitable[None]]] = None
) -> int:
) -> ScrapeResult:
"""Scrape historical messages from a source channel."""
result = ScrapeResult()
if not self.client.is_connected() or not await self.client.is_user_authorized():
if progress_callback:
await progress_callback("❌ ربات متصل نیست. لطفا ابتدا لاگین کنید.")
return 0
source = await self.repo.get_source_by_channel_id(channel_id)
source_title = source.title if source else str(channel_id)
username = source.username if source else None
collected_count = 0
skipped_count = 0
result.error = "not connected"
return result
try:
source = await self.repo.get_source_by_channel_id(channel_id)
source_title = source.title if source else str(channel_id)
username = source.username if source else None
entity = await self._resolve_channel_entity(channel_id, username)
messages = []
async for msg in self.client.iter_messages(entity, limit=limit):
@@ -208,6 +254,8 @@ class CollectorService:
messages.reverse()
result.scanned = len(messages)
for message in messages:
raw_text = message.raw_text or ""
if not raw_text and not message.media:
@@ -233,6 +281,10 @@ class CollectorService:
media_hash = compute_file_hash(downloaded_file)
content_hash = compute_content_hash(raw_text, media_hash)
duplicate_of = await self.repo.find_duplicate_post(content_hash)
if duplicate_of:
result.duplicates += 1
DUPLICATES_DETECTED_TOTAL.labels(method="content_hash").inc()
post_id = await self.repo.create_raw_post(
source_channel_id=channel_id,
@@ -241,29 +293,30 @@ class CollectorService:
media_path=media_path,
media_type=media_type,
content_hash=content_hash,
is_duplicate=duplicate_of is not None,
duplicate_of_id=duplicate_of.id if duplicate_of else None,
similarity_reason=f"content hash matches post #{duplicate_of.id}" if duplicate_of else None,
)
if post_id:
collected_count += 1
COLLECTED_POSTS_TOTAL.labels(source_channel_id=str(channel_id)).inc()
result.collected += 1
SOURCE_ACTIVITY_TOTAL.labels(channel_id=str(channel_id), title=source_title).inc()
logger.info(f"Backfilled raw post ID {post_id} from {channel_id}")
if self.on_post_received:
await self.on_post_received(post_id)
else:
skipped_count += 1
result.already_stored += 1
if progress_callback:
await progress_callback(
f"✅ تعداد <b>{collected_count}</b> پست جدید از <code>{channel_id}</code> دریافت و در کانال ادمین قرار گرفت! (رد شده تکراری: {skipped_count})."
)
return collected_count
await progress_callback(result.summary_fa(channel_id))
return result
except Exception as e:
await log_exception("collector.scrape_history", e, {"channel_id": channel_id, "limit": limit})
result.error = str(e)
if progress_callback:
await progress_callback(f"❌ خطا در دریافت پست‌های کانال <code>{channel_id}</code>: {e}")
return collected_count
await progress_callback(result.summary_fa(channel_id))
return result
async def stop(self):
if self.client.is_connected():
services/publisher.py
+82 -11
View File
@@ -1,13 +1,20 @@
import os
import asyncio
import logging
from datetime import datetime, timezone
from typing import Optional, List
from datetime import datetime, timedelta, timezone
from typing import Optional, List, Awaitable, Callable, Set, Tuple
from telethon import TelegramClient
from telethon.errors import (
ChannelPrivateError,
ChatAdminRequiredError,
ChatWriteForbiddenError,
PeerIdInvalidError,
UserBannedInChannelError,
)
from db.models import TargetChannel
from db.repository import Repository
from core.queue import RedisQueue
from core.metrics import TARGET_ACTIVITY_TOTAL, QUEUE_POSTS_GAUGE, REDIS_QUEUE_SIZE_GAUGE
from core.metrics import TARGET_ACTIVITY_TOTAL, QUEUE_POSTS_GAUGE
from core.proxy import get_telegram_proxy
from core.error_logger import log_exception
@@ -15,6 +22,19 @@ logger = logging.getLogger(__name__)
SESSION_DIR = os.getenv("SESSION_DIR", "/app/sessions" if os.path.exists("/app") else "/projects/telegram-bots/copykar/sessions")
# Sleep windows are configured in the operator's wall-clock time, not UTC.
TIMEZONE_OFFSET_HOURS = float(os.getenv("TIMEZONE_OFFSET_HOURS", "3.5"))
# Failures that will never resolve by retrying: the account simply cannot post there.
# Re-queueing these would spin the same post through the loop forever.
PERMANENT_DELIVERY_ERRORS = (
ChatAdminRequiredError,
ChatWriteForbiddenError,
ChannelPrivateError,
UserBannedInChannelError,
PeerIdInvalidError,
)
class PublisherService:
def __init__(
self,
@@ -25,10 +45,15 @@ class PublisherService:
api_hash: Optional[str] = None,
bot_token: Optional[str] = None,
session_name: Optional[str] = None,
notify_fn: Optional[Callable[[str], Awaitable[None]]] = None,
):
self.repo = repo
self.queue = queue
self.client = client
self.notify_fn = notify_fn
# (target_id, error type) pairs already reported, so a broken target is
# announced once instead of every polling cycle.
self._reported_failures: Set[Tuple[int, str]] = set()
self.api_id = api_id or int(os.getenv("API_ID", "0"))
self.api_hash = api_hash or os.getenv("API_HASH", "")
self.bot_token = bot_token or os.getenv("BOT_TOKEN")
@@ -42,11 +67,13 @@ class PublisherService:
async def start(self):
logger.info("Starting Paced Target Publisher Service...")
if not self.client.is_connected():
# Only owns the connection when it built its own client; a shared client
# (the admin bot's) is already connected by its owner.
if self.bot_token:
await self.client.start(bot_token=self.bot_token)
else:
await self.client.start()
logger.info("Paced Target Publisher Service connected.")
logger.info("Paced Target Publisher Service connected (delivering as the bot account).")
self._running = True
self._task = asyncio.create_task(self._publisher_loop())
@@ -66,7 +93,10 @@ class PublisherService:
async def _publisher_loop(self):
while self._running:
try:
await self._process_all_target_queues()
if await self.repo.is_system_paused():
logger.debug("[publisher] System is paused by admin. Skipping queue processing.")
else:
await self._process_all_target_queues()
except Exception as e:
logger.error(f"Error in target publisher loop: {e}", exc_info=True)
await asyncio.sleep(15)
@@ -74,13 +104,11 @@ class PublisherService:
async def _process_all_target_queues(self):
targets = await self.repo.get_active_targets()
now = datetime.now(timezone.utc)
current_hour_local = (now.hour + 3) % 24 # UTC+3:30 approx hour
total_queued = 0
local_now = now + timedelta(hours=TIMEZONE_OFFSET_HOURS)
current_hour_local = local_now.hour
for target in targets:
qsize = await self.queue.get_target_queue_size(target.id)
total_queued += qsize
QUEUE_POSTS_GAUGE.labels(status=f"target_{target.id}").set(qsize)
if qsize == 0:
@@ -130,9 +158,52 @@ class PublisherService:
TARGET_ACTIVITY_TOTAL.labels(channel_id=str(target.channel_id), title=target.title or '').inc()
logger.info(f"Published post ID {post_id} to Target {target.title} ({target.channel_id})")
except Exception as e:
await log_exception("publisher.publish", e, {"post_id": post_id, "target_id": target.id, "channel_id": target.channel_id})
permanent = isinstance(e, PERMANENT_DELIVERY_ERRORS)
if permanent:
# Dropping the payload is deliberate: the post stays 'pending_review'
# so an admin can re-send it once the permission problem is resolved.
await self._report_broken_target(target, e)
else:
# The payload was already popped; putting it back keeps the post from
# being silently lost on a transient Telegram failure.
try:
await self.queue.push_target_post(target.id, payload)
except Exception as requeue_err:
logger.critical(f"Failed to requeue post {post_id} for target {target.id}: {requeue_err}")
REDIS_QUEUE_SIZE_GAUGE.set(total_queued)
await log_exception("publisher.publish", e, {
"post_id": post_id,
"target_id": target.id,
"channel_id": target.channel_id,
"permanent": permanent,
})
async def _report_broken_target(self, target: TargetChannel, error: Exception) -> None:
"""Tell the admins once that a target channel is unreachable for this account."""
key = (target.id, type(error).__name__)
if key in self._reported_failures:
return
self._reported_failures.add(key)
logger.error(
f"Target #{target.id} ({target.title}) rejected delivery permanently: "
f"{type(error).__name__}. Queue drained for this target until it is fixed."
)
if not self.notify_fn:
return
try:
await self.notify_fn(
f"⛔ <b>ارسال به کانال مقصد «{target.title}» ممکن نیست!</b>\n\n"
f"• 🆔 شناسه کانال: <code>{target.channel_id}</code>\n"
f"• ❗️ خطا: <code>{type(error).__name__}</code>\n\n"
"<b>ربات</b> در این کانال عضو یا ادمین با دسترسی ارسال پیام نیست.\n"
"لطفا ربات را در کانال <b>ادمین</b> کنید و دسترسی <b>ارسال پیام (Post Messages)</b> بدهید، "
"سپس پست را دوباره به صف بفرستید.\n\n"
"<i>تا رفع این مشکل، پست‌های این کانال ارسال نمی‌شوند و در وضعیت بررسی باقی می‌مانند.</i>"
)
except Exception as notify_err:
logger.error(f"Could not notify admins about broken target {target.id}: {notify_err}")
async def stop(self):
self._running = False
services/queue_consumer.py
-50
View File
@@ -1,50 +0,0 @@
import os
import asyncio
import logging
from typing import Optional
from core.queue import RedisQueue
from core.metrics import QUEUE_POSTS_GAUGE, REDIS_QUEUE_SIZE_GAUGE
logger = logging.getLogger(__name__)
FETCH_INTERVAL_SECONDS = int(os.getenv("AI_PROCESSING_INTERVAL_SECONDS", "120"))
class QueueConsumerService:
def __init__(self, queue: RedisQueue, on_post_popped = None, fetch_interval: int = FETCH_INTERVAL_SECONDS):
self.queue = queue
self.on_post_popped = on_post_popped
self.fetch_interval = fetch_interval
self._running = False
self._task: Optional[asyncio.Task] = None
async def start(self):
logger.info(f"Starting Redis Queue Consumer Service (interval: {self.fetch_interval}s / {self.fetch_interval // 60}m)...")
await self.queue.connect()
self._running = True
self._task = asyncio.create_task(self._consumer_loop())
async def _consumer_loop(self):
while self._running:
try:
qsize = await self.queue.qsize()
QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(qsize)
REDIS_QUEUE_SIZE_GAUGE.set(qsize)
if qsize > 0:
post_id = await self.queue.pop()
if post_id and self.on_post_popped:
logger.info(f"Dispatching raw post ID {post_id} to Admin Review channel (remaining in Redis: {qsize - 1})")
await self.on_post_popped(post_id)
new_qsize = await self.queue.qsize()
QUEUE_POSTS_GAUGE.labels(status="redis_incoming").set(new_qsize)
REDIS_QUEUE_SIZE_GAUGE.set(new_qsize)
except Exception as e:
logger.error(f"Error in queue consumer loop: {e}", exc_info=True)
await asyncio.sleep(self.fetch_interval)
async def stop(self):
self._running = False
if self._task:
self._task.cancel()
logger.info("Redis Queue Consumer Service stopped.")