feat(dedup): implement two-stage duplicate detection and content rewriter

core/dedup.py

@@ -0,0 +1,35 @@
+import hashlib
+import re
+from typing import Optional
+
+def normalize_text(text: Optional[str]) -> str:
+    """Normalize text by removing URLs, telegram handles, hashtags, excessive punctuation/whitespace, and lowercasing."""
+    if not text:
+        return ""
+    # Remove URLs
+    text = re.sub(r'https?://\S+|www\.\S+', '', text)
+    # Remove Telegram @mentions / hashtags
+    text = re.sub(r'[@#]\w+', '', text)
+    # Normalize punctuation and whitespace
+    text = re.sub(r'[^\w\s]', ' ', text)
+    text = re.sub(r'\s+', ' ', text)
+    return text.strip().lower()
+
+def compute_content_hash(text: Optional[str], media_hash: Optional[str] = None) -> Optional[str]:
+    """Generate SHA256 hash from normalized text and/or media hash."""
+    norm_text = normalize_text(text)
+    if not norm_text and not media_hash:
+        return None
+    raw_key = f"{norm_text}|{media_hash or ''}"
+    return hashlib.sha256(raw_key.encode('utf-8')).hexdigest()
+
+def compute_file_hash(file_path: str) -> Optional[str]:
+    """Generate SHA256 hash of a media file."""
+    try:
+        hasher = hashlib.sha256()
+        with open(file_path, 'rb') as f:
+            while chunk := f.read(65536):
+                hasher.update(chunk)
+        return hasher.hexdigest()
+    except Exception:
+        return None