root/copykar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(publishing): implement immediate raw ingestion, per-target Redis queues, and configurable post intervals & sleep windows

Browse Source
This commit is contained in:
mamad
2026-08-27 22:21:29 +03:30
parent e66ed361c3
commit 7e5e679917
8 changed files with 288 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/publisher.py
+70 -32
View File
@@ -2,10 +2,12 @@ import os
import asyncio
import logging
from datetime import datetime, timezone
from typing import Optional
from typing import Optional, List
from telethon import TelegramClient
from db.models import TargetChannel
from db.repository import Repository
from core.metrics import POSTS_PUBLISHED_TOTAL, QUEUE_POSTS_GAUGE
from core.queue import RedisQueue
from core.metrics import TARGET_ACTIVITY_TOTAL, QUEUE_POSTS_GAUGE, REDIS_QUEUE_SIZE_GAUGE
from core.proxy import get_telegram_proxy
logger = logging.getLogger(__name__)
@@ -16,48 +18,79 @@ class PublisherService:
def __init__(
self,
repo: Repository,
queue: RedisQueue,
client: Optional[TelegramClient] = None,
api_id: Optional[int] = None,
api_hash: Optional[str] = None,
session_name: Optional[str] = None,
bot_token: Optional[str] = None,
session_name: Optional[str] = None,
):
self.repo = repo
self.queue = queue
self.client = client
self.api_id = api_id or int(os.getenv("API_ID", "0"))
self.api_hash = api_hash or os.getenv("API_HASH", "")
self.bot_token = bot_token or os.getenv("BOT_TOKEN")
self.session_name = session_name or os.path.join(SESSION_DIR, "publisher.session")
os.makedirs(os.path.dirname(self.session_name), exist_ok=True)
self.client = TelegramClient(self.session_name, self.api_id, self.api_hash, proxy=get_telegram_proxy())
if not self.client:
self.client = TelegramClient(self.session_name, self.api_id, self.api_hash, proxy=get_telegram_proxy())
self._running = False
self._task: Optional[asyncio.Task] = None
async def start(self):
logger.info("Starting Publisher Service...")
if self.bot_token:
await self.client.start(bot_token=self.bot_token)
else:
await self.client.start()
logger.info("Publisher Service connected successfully.")
logger.info("Starting Paced Target Publisher Service...")
if not self.client.is_connected():
if self.bot_token:
await self.client.start(bot_token=self.bot_token)
else:
await self.client.start()
logger.info("Paced Target Publisher Service connected.")
self._running = True
self._task = asyncio.create_task(self._publisher_loop())
def _is_in_sleep_window(self, target: TargetChannel, current_hour: int) -> bool:
if not target.is_sleep_enabled:
return False
start = target.sleep_start_hour
end = target.sleep_end_hour
if start == end:
return False
if start < end:
return start <= current_hour < end
else:
# Overnight sleep (e.g. 23:00 to 08:00)
return current_hour >= start or current_hour < end
async def _publisher_loop(self):
while self._running:
try:
await self._process_pending_queues()
await self._process_all_target_queues()
except Exception as e:
logger.error(f"Error in publisher loop: {e}", exc_info=True)
logger.error(f"Error in target publisher loop: {e}", exc_info=True)
await asyncio.sleep(15)
async def _process_pending_queues(self):
async def _process_all_target_queues(self):
targets = await self.repo.get_active_targets()
now = datetime.now(timezone.utc)
current_hour_local = (now.hour + 3) % 24 # UTC+3:30 approx hour
pending_count = len(await self.repo.get_posts_by_status("approved", limit=5000))
QUEUE_POSTS_GAUGE.labels(status="approved").set(pending_count)
total_queued = 0
for target in targets:
qsize = await self.queue.get_target_queue_size(target.id)
total_queued += qsize
QUEUE_POSTS_GAUGE.labels(status=f"target_{target.id}").set(qsize)
if qsize == 0:
continue
# 1. Check Sleep Window
if self._is_in_sleep_window(target, current_hour_local):
logger.debug(f"Target #{target.id} ({target.title}) in sleep window ({target.sleep_start_hour}:00-{target.sleep_end_hour}:00). Skipping.")
continue
# 2. Check Cooldown Interval
if target.last_post_time:
last_post = target.last_post_time
if last_post.tzinfo is None:
@@ -66,37 +99,42 @@ class PublisherService:
if diff_minutes < target.post_interval_min:
continue
post = await self.repo.get_next_approved_post_for_target(target.id)
if not post:
# 3. Pop next post payload for this target
payload = await self.queue.pop_target_post(target.id)
if not payload:
continue
post_id = payload.get("post_id")
text = payload.get("text", "")
media_path = payload.get("media_path")
try:
publish_text = post.ai_text or post.raw_text or ""
if post.media_path and os.path.exists(post.media_path):
if media_path and os.path.exists(media_path):
await self.client.send_file(
target.channel_id,
file=post.media_path,
caption=publish_text,
parse_mode="markdown"
file=media_path,
caption=text,
parse_mode="html"
)
else:
await self.client.send_message(
target.channel_id,
publish_text,
parse_mode="markdown"
text,
parse_mode="html"
)
await self.repo.mark_post_published(post.id)
# Record publication in database
await self.repo.record_post_published_to_target(post_id, target.id, target.title or "Target")
await self.repo.update_target_last_post(target.id)
POSTS_PUBLISHED_TOTAL.labels(target_channel_id=str(target.channel_id)).inc()
logger.info(f"Successfully published post {post.id} to target channel {target.title} ({target.channel_id})")
TARGET_ACTIVITY_TOTAL.labels(channel_id=str(target.channel_id), title=target.title or '').inc()
logger.info(f"Published post ID {post_id} to Target {target.title} ({target.channel_id})")
except Exception as e:
logger.error(f"Failed to publish post {post.id} to target {target.channel_id}: {e}", exc_info=True)
logger.error(f"Failed to publish queued post {post_id} to target {target.channel_id}: {e}", exc_info=True)
REDIS_QUEUE_SIZE_GAUGE.set(total_queued)
async def stop(self):
self._running = False
if self._task:
self._task.cancel()
if self.client.is_connected():
await self.client.disconnect()
logger.info("Publisher Service disconnected.")
logger.info("Publisher Service stopped.")