feat(dedup): implement two-stage duplicate detection and content rewriter

tests/test_dedup.py

@@ -0,0 +1,22 @@
+from core.dedup import normalize_text, compute_content_hash
+
+def test_normalize_text():
+    raw = "  Check out this link: https://t.me/example! @admin #tech news...  "
+    norm = normalize_text(raw)
+    assert "https" not in norm
+    assert "admin" not in norm
+    assert "tech" not in norm
+    assert norm == "check out this link news"
+
+def test_content_hash():
+    text1 = "Breaking News: Bitcoin hits $100k! Check https://example.com"
+    text2 = "Breaking News: Bitcoin hits $100k! Check https://other.com"
+    hash1 = compute_content_hash(text1)
+    hash2 = compute_content_hash(text2)
+    # Both normalize to the same text after link removal
+    assert hash1 == hash2
+
+if __name__ == "__main__":
+    test_normalize_text()
+    test_content_hash()
+    print("All deduplication unit tests passed!")