import logging from typing import List, Optional, Dict, Any from db.models import Post, TargetChannel from db.repository import Repository from core.llm import LLMClient from core.dedup import compute_content_hash from core.metrics import DUPLICATES_DETECTED_TOTAL logger = logging.getLogger(__name__) TAG_EXTRACTION_SYSTEM_PROMPT = """ You are an AI news analyst and classifier. Given a social media/channel post, extract: 1. "subject": A brief, specific headline/subject (3-8 words). 2. "tags": A JSON array of 3 to 6 lowercase keywords/topics/entities (e.g. ["ai", "nvidia", "gpus", "hardware"]). Respond ONLY in JSON format: { "subject": "...", "tags": ["tag1", "tag2", "tag3"] } """ DUPLICATE_CHECK_SYSTEM_PROMPT = """ You are an expert news editor checking for duplicate news stories. Given a NEW POST and a list of PREVIOUS POSTS, determine if the NEW POST is covering the same exact event, news item, or story as any of the previous posts. Respond ONLY in JSON format: { "is_duplicate": true/false, "duplicate_of_id": , "similarity_reason": "" } """ POST_REWRITE_SYSTEM_PROMPT = """ You are an expert Telegram content creator and copywriter. Rewrite the provided post to make it engaging, well-formatted, professional, and clear. Use appropriate emojis, clear paragraphs, and markdown formatting. Remove any original promotional links, author credits, or watermarks. Respond ONLY in JSON format: { "ai_text": "...", "suggested_target_id": } """ class AIProcessor: def __init__(self, repo: Repository, llm: Optional[LLMClient] = None): self.repo = repo self.llm = llm or LLMClient() async def process_post(self, post_id: int) -> Optional[Post]: post = await self.repo.get_post_by_id(post_id) if not post or not post.raw_text: return post raw_text = post.raw_text is_dup = False dup_of_id = None sim_reason = None # 1. Exact hash duplicate check content_hash = post.content_hash or compute_content_hash(raw_text) if content_hash: exact_dup = await self.repo.find_duplicate_post_by_hash(content_hash) if exact_dup and exact_dup.id != post.id: is_dup = True dup_of_id = exact_dup.id sim_reason = "Exact match on normalized text/media hash" DUPLICATES_DETECTED_TOTAL.labels(method="hash").inc() # 2. Extract Tags and Subject via AI tags = [] subject = "General News" try: tag_res = await self.llm.generate_json( prompt=f"Post content:\n\n{raw_text}", system_prompt=TAG_EXTRACTION_SYSTEM_PROMPT, action_name="extract_tags" ) subject = tag_res.get("subject", subject) tags = [t.lower().strip() for t in tag_res.get("tags", []) if isinstance(t, str)] await self.repo.update_post_tags(post.id, tags, subject) except Exception as e: logger.error(f"Tag extraction failed for post {post.id}: {e}") # 3. Candidate search & Semantic AI Deduplication check (if not already exact dup) if not is_dup and tags: candidates = await self.repo.find_candidate_posts_by_tags(tags, exclude_post_id=post.id, hours_lookback=72, limit=5) if candidates: cand_texts = "\n---\n".join([f"ID {c.id} (Subject: {c.subject}):\n{c.raw_text}" for c in candidates if c.raw_text]) prompt = f"NEW POST:\n{raw_text}\n\nPREVIOUS CANDIDATE POSTS:\n{cand_texts}" try: dup_res = await self.llm.generate_json( prompt=prompt, system_prompt=DUPLICATE_CHECK_SYSTEM_PROMPT, action_name="check_duplicate" ) if dup_res.get("is_duplicate"): is_dup = True dup_of_id = dup_res.get("duplicate_of_id") sim_reason = dup_res.get("similarity_reason", "AI detected duplicate news topic") DUPLICATES_DETECTED_TOTAL.labels(method="ai_semantic").inc() except Exception as e: logger.error(f"Semantic duplicate check failed for post {post.id}: {e}") # 4. Rewrite post for our channels ai_text = raw_text suggested_target_id = None targets = await self.repo.get_active_targets() target_info = "\n".join([f"Target ID {t.id}: {t.title} (@{t.username or 'none'})" for t in targets]) rewrite_prompt = f"TARGET CHANNELS AVAILABLE:\n{target_info or 'None'}\n\nORIGINAL POST:\n{raw_text}" try: rewrite_res = await self.llm.generate_json( prompt=rewrite_prompt, system_prompt=POST_REWRITE_SYSTEM_PROMPT, action_name="rewrite_post" ) ai_text = rewrite_res.get("ai_text", raw_text) suggested_target_id = rewrite_res.get("suggested_target_id") except Exception as e: logger.error(f"Post rewrite failed for post {post.id}: {e}") # 5. Save AI results into database await self.repo.update_ai_result( post_id=post.id, subject=subject, ai_text=ai_text, tags=tags, suggested_target_id=suggested_target_id, is_duplicate=is_dup, duplicate_of_id=dup_of_id, similarity_reason=sim_reason, ) return await self.repo.get_post_by_id(post.id)