services: update ingestion collector, paced publisher, and deduplication

tests/test_scrape_history.py

@@ -0,0 +1,126 @@
+"""Covers the flow behind the '📥 استخراج ۲۰ پست' button, which used to abort silently."""
+import asyncio
+import sys
+import types
+
+sys.path.insert(0, "/app")
+
+from db.database import init_db, close_db_pool
+from db.repository import Repository
+from services.collector import CollectorService
+
+SOURCE_ID = -1009999000010
+
+
+class FakeMessage:
+    def __init__(self, msg_id, text):
+        self.id = msg_id
+        self.raw_text = text
+        self.media = None
+
+    async def download_media(self, file=None):
+        return None
+
+
+class FakeClient:
+    """Minimal stand-in for the Telethon client used by the collector."""
+
+    def __init__(self, messages):
+        self._messages = messages
+
+    def is_connected(self):
+        return True
+
+    async def is_user_authorized(self):
+        return True
+
+    async def get_entity(self, ident):
+        return types.SimpleNamespace(id=abs(SOURCE_ID), title="Scrape Source")
+
+    def iter_messages(self, entity, limit=20):
+        async def gen():
+            for m in self._messages[:limit]:
+                yield m
+        return gen()
+
+
+async def _cleanup(repo):
+    pool = await repo._get_pool()
+    async with pool.acquire() as conn:
+        await conn.execute("DELETE FROM posts WHERE source_channel_id = $1;", SOURCE_ID)
+        await conn.execute("DELETE FROM sources WHERE channel_id = $1;", SOURCE_ID)
+        # The failure-path assertion logs a real error row; don't leave it behind.
+        await conn.execute(
+            "DELETE FROM error_logs WHERE context->>'channel_id' = $1;", str(SOURCE_ID))
+
+
+async def run_tests():
+    await init_db()
+    repo = Repository()
+    await _cleanup(repo)
+    await repo.add_source(SOURCE_ID, "Scrape Source", None)
+
+    reviewed = []
+    collector = CollectorService(repo=repo, on_post_received=lambda pid: _record(reviewed, pid))
+    # 20 distinct messages plus one exact repeat of the first, to exercise dedup.
+    messages = [FakeMessage(1000 + i, f"historical post number {i}") for i in range(20)]
+    messages.append(FakeMessage(1099, "historical post number 0"))
+    collector.client = FakeClient(messages)
+
+    progress = []
+
+    async def on_progress(text):
+        progress.append(text)
+
+    res = await collector.scrape_channel_history(SOURCE_ID, limit=21, progress_callback=on_progress)
+
+    assert res.collected == 21, f"expected 21 posts collected, got {res.collected}"
+    assert res.scanned == 21, f"expected 21 messages scanned, got {res.scanned}"
+    assert res.duplicates == 1, f"expected 1 duplicate, got {res.duplicates}"
+    assert res.already_stored == 0, f"expected 0 already-stored, got {res.already_stored}"
+    assert len(reviewed) == 21, f"expected 21 review cards dispatched, got {len(reviewed)}"
+    assert progress and progress[-1].startswith("✅"), f"admin was not told the result: {progress}"
+
+    # Re-running the same window adds nothing new, and the report must say WHY
+    # rather than looking like a dead button.
+    progress.clear()
+    again = await collector.scrape_channel_history(SOURCE_ID, limit=21, progress_callback=on_progress)
+    assert again.collected == 0, f"repeat scrape should add nothing, got {again.collected}"
+    assert again.already_stored == 21, f"expected 21 already-stored, got {again.already_stored}"
+    assert progress[-1].startswith("ℹ️"), f"repeat scrape must be explained: {progress[-1]}"
+    assert "قبلا ذخیره شده" in progress[-1]
+
+    stored = await repo.get_posts_by_status("pending_review", limit=100)
+    mine = [p for p in stored if p.source_channel_id == SOURCE_ID]
+    assert len(mine) == 21, f"expected 21 stored posts, got {len(mine)}"
+
+    # The repeated text must be flagged against the original rather than stored blind.
+    dupes = [p for p in mine if p.is_duplicate]
+    assert len(dupes) == 1, f"expected exactly 1 duplicate flagged, got {len(dupes)}"
+    assert dupes[0].duplicate_of_id is not None
+
+    # A channel that cannot be resolved reports the failure instead of vanishing.
+    class BrokenClient(FakeClient):
+        async def get_entity(self, ident):
+            raise RuntimeError("channel not reachable")
+
+        async def get_dialogs(self, limit=50):
+            return []
+
+    collector.client = BrokenClient([])
+    progress.clear()
+    result = await collector.scrape_channel_history(SOURCE_ID, limit=5, progress_callback=on_progress)
+    assert result.collected == 0 and result.error
+    assert progress and progress[-1].startswith("❌"), f"failure was not reported: {progress}"
+
+    await _cleanup(repo)
+    await close_db_pool()
+    print("All scrape-history tests passed successfully!")
+
+
+async def _record(bucket, post_id):
+    bucket.append(post_id)
+
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())