import hashlib import re from typing import Optional def normalize_text(text: Optional[str]) -> str: """Normalize text by removing URLs, telegram handles, hashtags, excessive punctuation/whitespace, and lowercasing.""" if not text: return "" # Remove URLs text = re.sub(r'https?://\S+|www\.\S+', '', text) # Remove Telegram @mentions / hashtags text = re.sub(r'[@#]\w+', '', text) # Normalize punctuation and whitespace text = re.sub(r'[^\w\s]', ' ', text) text = re.sub(r'\s+', ' ', text) return text.strip().lower() def compute_content_hash(text: Optional[str], media_hash: Optional[str] = None) -> Optional[str]: """Generate SHA256 hash from normalized text and/or media hash.""" norm_text = normalize_text(text) if not norm_text and not media_hash: return None raw_key = f"{norm_text}|{media_hash or ''}" return hashlib.sha256(raw_key.encode('utf-8')).hexdigest() def compute_file_hash(file_path: str) -> Optional[str]: """Generate SHA256 hash of a media file.""" try: hasher = hashlib.sha256() with open(file_path, 'rb') as f: while chunk := f.read(65536): hasher.update(chunk) return hasher.hexdigest() except Exception: return None