27 lines
842 B
Python
27 lines
842 B
Python
import sys
|
|
|
|
sys.path.insert(0, "/app")
|
|
|
|
from core.dedup import normalize_text, compute_content_hash
|
|
|
|
def test_normalize_text():
|
|
raw = " Check out this link: https://t.me/example! @admin #tech news... "
|
|
norm = normalize_text(raw)
|
|
assert "https" not in norm
|
|
assert "admin" not in norm
|
|
assert "tech" not in norm
|
|
assert norm == "check out this link news"
|
|
|
|
def test_content_hash():
|
|
text1 = "Breaking News: Bitcoin hits $100k! Check https://example.com"
|
|
text2 = "Breaking News: Bitcoin hits $100k! Check https://other.com"
|
|
hash1 = compute_content_hash(text1)
|
|
hash2 = compute_content_hash(text2)
|
|
# Both normalize to the same text after link removal
|
|
assert hash1 == hash2
|
|
|
|
if __name__ == "__main__":
|
|
test_normalize_text()
|
|
test_content_hash()
|
|
print("All deduplication unit tests passed!")
|