feat: add topic tagging, semantic duplicate detection, and per-provider vision toggle

services/ai_processor.py

@@ -307,3 +307,111 @@ class AIProcessor:
             fallback = f"{fallback}\n\n{footer_text}"
         return RewriteResult(decision="accept", rejection_reason="", rewritten_text=fallback)
 
+    async def extract_tags_and_subject(
+        self,
+        raw_text: str,
+        image_path: Optional[str] = None
+    ) -> tuple[List[str], str]:
+        """Extract 3-6 topic tags and a short subject from incoming post text and/or image."""
+        if not raw_text and not image_path:
+            return [], ""
+
+        sys_prompt = (
+            "You are a Telegram post classifier and tagger. "
+            "Analyze the given post content (and optional image) and extract:\n"
+            "1. 'subject': A concise, descriptive subject or headline (in Persian, max 10 words).\n"
+            "2. 'tags': A list of 3 to 6 relevant topical keywords/tags (lowercase, normalized, without '#' prefix).\n\n"
+            "Respond ONLY in valid JSON format:\n"
+            "{\n"
+            '  "subject": "...",\n'
+            '  "tags": ["tag1", "tag2", "tag3"]\n'
+            "}"
+        )
+
+        user_prompt = f"متن پست برای دسته‌بندی و برچسب‌گذاری:\n\n{raw_text}" if raw_text else "لطفاً با توجه به تصویر پیوست، موضوع و برچسب‌های آن را استخراج کنید."
+
+        try:
+            res = await self.llm.generate_json(
+                prompt=user_prompt,
+                system_prompt=sys_prompt,
+                action_name="extract_tags_and_subject",
+                image_path=image_path
+            )
+            subject = str(res.get("subject", "")).strip()
+            raw_tags = res.get("tags") or []
+            tags: List[str] = []
+            if isinstance(raw_tags, list):
+                for t in raw_tags:
+                    clean_t = str(t).strip().lstrip("#").lower()
+                    if clean_t and clean_t not in tags:
+                        tags.append(clean_t)
+            return tags[:8], subject
+        except Exception as e:
+            logger.warning(f"Failed to extract tags and subject via AI: {e}")
+            return [], ""
+
+    async def check_semantic_duplicate(
+        self,
+        new_text: str,
+        candidate_posts: List[Post],
+        image_path: Optional[str] = None
+    ) -> tuple[bool, Optional[int], Optional[str]]:
+        """Compare a new post against candidate posts sharing overlapping tags using AI semantic similarity."""
+        if not candidate_posts or (not new_text and not image_path):
+            return False, None, None
+
+        candidates_formatted = []
+        for p in candidate_posts[:10]:
+            p_text = (p.raw_text or "").strip()
+            snippet = p_text[:250] + ("..." if len(p_text) > 250 else "")
+            candidates_formatted.append(
+                f"- [پست شناسه #{p.id}] (موضوع: {p.subject or 'نامشخص'} | برچسب‌ها: {', '.join(p.tags or [])}):\n  «{snippet}»"
+            )
+
+        candidates_block = "\n\n".join(candidates_formatted)
+
+        sys_prompt = (
+            "You are an expert news and content duplicate detection engine.\n"
+            "Compare the NEW POST against the list of CANDIDATE POSTS.\n"
+            "Determine if the NEW POST covers the EXACT SAME news event, identical story, announcement, or duplicate information.\n\n"
+            "Respond ONLY in valid JSON format:\n"
+            "{\n"
+            '  "is_duplicate": true,\n'
+            '  "duplicate_of_id": 123,\n'
+            '  "reason": "توضیح کوتاه به زبان فارسی در مورد علت تکراری بودن"\n'
+            "}\n"
+            "If it is NOT a duplicate of any candidate post:\n"
+            "{\n"
+            '  "is_duplicate": false,\n'
+            '  "duplicate_of_id": null,\n'
+            '  "reason": ""\n'
+            "}"
+        )
+
+        user_prompt = (
+            f"پست جدید برای بررسی:\n{new_text}\n\n"
+            f"لیست پست‌های مشابه قبلی برای مقایسه:\n{candidates_block}"
+        )
+
+        try:
+            res = await self.llm.generate_json(
+                prompt=user_prompt,
+                system_prompt=sys_prompt,
+                action_name="check_semantic_duplicate",
+                image_path=image_path
+            )
+            is_dup = bool(res.get("is_duplicate", False))
+            dup_id = res.get("duplicate_of_id")
+            reason = str(res.get("reason", "")).strip()
+
+            if is_dup and dup_id:
+                try:
+                    dup_id_int = int(dup_id)
+                    return True, dup_id_int, reason or f"تشخیص هوش مصنوعی: مشابه پست #{dup_id_int}"
+                except (ValueError, TypeError):
+                    pass
+            return False, None, None
+        except Exception as e:
+            logger.warning(f"AI semantic duplicate check failed: {e}")
+            return False, None, None
+