services: update ingestion collector, paced publisher, and deduplication

tests/test_auto_routing.py

@@ -0,0 +1,99 @@
+"""Auto-routing must queue to subscribed targets WITHOUT replacing the admin review card."""
+import asyncio
+import sys
+
+sys.path.insert(0, "/app")
+
+from db.database import init_db, close_db_pool
+from db.repository import Repository
+from services.admin_bot import AdminBotService
+
+SRC_A = -1009999000020
+SRC_B = -1009999000021
+TRG = -1009999000022
+
+
+class FakeQueue:
+    def __init__(self):
+        self.pushed = []
+
+    async def push_target_post(self, target_id, payload):
+        self.pushed.append((target_id, payload))
+
+    async def get_target_queue_size(self, target_id):
+        return 0
+
+
+class FakeAI:
+    async def rewrite_for_target(self, raw_text, target, has_media=False, *args, **kwargs):
+        return f"[{target.title}] {raw_text}"
+
+
+
+async def _cleanup(repo):
+    pool = await repo._get_pool()
+    async with pool.acquire() as conn:
+        for cid in (SRC_A, SRC_B):
+            await conn.execute("DELETE FROM posts WHERE source_channel_id = $1;", cid)
+            await conn.execute("DELETE FROM sources WHERE channel_id = $1;", cid)
+        await conn.execute("DELETE FROM targets WHERE channel_id = $1;", TRG)
+
+
+async def run_tests():
+    await init_db()
+    repo = Repository()
+    await _cleanup(repo)
+
+    await repo.add_source(SRC_A, "Source A", None)
+    await repo.add_source(SRC_B, "Source B", None)
+    target_id = await repo.add_target(TRG, "Auto Target", None)
+
+    # A brand new target routes nothing.
+    assert await repo.get_targets_auto_routed_from(SRC_A) == []
+
+    # Toggle is a switch, not a one-way door.
+    assert await repo.toggle_target_auto_source(target_id, SRC_A) is True
+    routed = await repo.get_targets_auto_routed_from(SRC_A)
+    assert len(routed) == 1 and routed[0].id == target_id
+    assert routed[0].auto_source_ids == [SRC_A]
+    assert await repo.toggle_target_auto_source(target_id, SRC_A) is False
+    assert await repo.get_targets_auto_routed_from(SRC_A) == []
+
+    # Subscribe to A only; B must stay manual.
+    await repo.set_target_auto_sources(target_id, [SRC_A])
+
+    queue = FakeQueue()
+    bot = AdminBotService(repo=repo, ai_processor=FakeAI(), queue=queue,
+                          review_channel_id=0, admin_user_ids=[1])
+
+    post_a = await repo.create_raw_post(SRC_A, 1, "post from A", content_hash="auto_a")
+    post_b = await repo.create_raw_post(SRC_B, 2, "post from B", content_hash="auto_b")
+
+    assert await bot.auto_route_post(post_a) == 1, "subscribed source must route"
+    assert await bot.auto_route_post(post_b) == 0, "unsubscribed source must not route"
+
+    assert len(queue.pushed) == 1, f"expected 1 enqueue, got {queue.pushed}"
+    tid, payload = queue.pushed[0]
+    assert tid == target_id
+    assert payload["text"] == "[Auto Target] post from A", f"AI rewrite not applied: {payload['text']}"
+    assert payload["post_id"] == post_a
+
+    # The post is recorded against the target but must NOT be marked published yet.
+    stored = await repo.get_post_by_id(post_a)
+    assert stored.status == "pending_review", f"auto-routing must not publish, got {stored.status}"
+    assert len(stored.published_to) == 1
+    assert stored.published_to[0]["target_id"] == target_id
+    assert stored.published_to[0]["published_at"] is None
+
+    # A soft-deleted post is never auto-routed.
+    await repo.soft_delete_post(post_b)
+    await repo.set_target_auto_sources(target_id, [SRC_A, SRC_B])
+    assert await bot.auto_route_post(post_b) == 0, "deleted posts must not route"
+
+    await _cleanup(repo)
+    await close_db_pool()
+    print("All auto-routing tests passed successfully!")
+
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())

tests/test_db.py

@@ -1,63 +0,0 @@
-import asyncio
-import os
-import tempfile
-from db.database import init_db
-from db.repository import Repository
-
-async def run_tests():
-    with tempfile.NamedTemporaryFile(suffix=".db") as tmp:
-        db_path = tmp.name
-        print(f"Testing DB on {db_path}...")
-        await init_db(db_path)
-        repo = Repository(db_path)
-
-        # 1. Test Sources
-        s_id = await repo.add_source(-1001234567890, "Source Tech", "source_tech")
-        sources = await repo.get_active_sources()
-        assert len(sources) == 1
-        assert sources[0].channel_id == -1001234567890
-
-        # 2. Test Targets
-        t_id = await repo.add_target(-1009876543210, "Target Channel", "target_chan", post_interval_min=15)
-        targets = await repo.get_active_targets()
-        assert len(targets) == 1
-        assert targets[0].post_interval_min == 15
-
-        # 3. Test Raw Post & Deduplication
-        post_id = await repo.create_raw_post(
-            source_channel_id=-1001234567890,
-            source_message_id=101,
-            raw_text="Breaking news: AI update released!",
-            content_hash="hash_12345",
-            is_duplicate=False
-        )
-        assert post_id is not None
-
-        # Check duplicate lookup
-        dup_match = await repo.find_duplicate_post("hash_12345")
-        assert dup_match is not None
-        assert dup_match.id == post_id
-
-        # 4. Test AI update & Approval flow
-        await repo.update_ai_result(post_id, subject="AI/Tech", ai_text="Rewritten: AI update is now live!", suggested_target_id=t_id)
-        pending_review = await repo.get_posts_by_status("pending_review")
-        assert len(pending_review) == 1
-        assert pending_review[0].subject == "AI/Tech"
-
-        await repo.approve_post(post_id, target_channel_id=t_id)
-        approved_post = await repo.get_next_approved_post_for_target(t_id)
-        assert approved_post is not None
-        assert approved_post.id == post_id
-
-        await repo.mark_post_published(post_id)
-        assert await repo.get_next_approved_post_for_target(t_id) is None
-
-        # 5. Settings
-        await repo.set_setting("system_prompt", "You are a professional editor.")
-        val = await repo.get_setting("system_prompt")
-        assert val == "You are a professional editor."
-
-        print("All database tests passed successfully!")
-
-if __name__ == "__main__":
-    asyncio.run(run_tests())

tests/test_dedup.py

@@ -1,3 +1,7 @@
+import sys
+
+sys.path.insert(0, "/app")
+
 from core.dedup import normalize_text, compute_content_hash
 
 def test_normalize_text():

tests/test_publisher_failures.py

@@ -0,0 +1,111 @@
+"""A target the account cannot post to must not spin the same post through the queue forever."""
+import asyncio
+import sys
+
+sys.path.insert(0, "/app")
+
+from telethon.errors import ChatAdminRequiredError
+from db.models import TargetChannel
+from services.publisher import PublisherService
+
+
+class FakeQueue:
+    def __init__(self, payloads):
+        self.items = list(payloads)
+        self.pushes = 0
+
+    async def get_target_queue_size(self, target_id):
+        return len(self.items)
+
+    async def pop_target_post(self, target_id):
+        return self.items.pop(0) if self.items else None
+
+    async def push_target_post(self, target_id, payload):
+        self.pushes += 1
+        self.items.append(payload)
+
+
+class FakeRepo:
+    def __init__(self, targets):
+        self._targets = targets
+        self.published = []
+
+    async def get_active_targets(self):
+        return self._targets
+
+    async def record_post_published_to_target(self, *a):
+        self.published.append(a)
+
+    async def update_target_last_post(self, *a):
+        pass
+
+
+def make_target():
+    return TargetChannel(id=2, channel_id=-1003848540849, title="newsjoker", username=None,
+                         post_interval_min=1, last_post_time=None)
+
+
+class FailingClient:
+    def __init__(self, error):
+        self.error = error
+        self.attempts = 0
+
+    def is_connected(self):
+        return True
+
+    async def send_message(self, *a, **kw):
+        self.attempts += 1
+        raise self.error
+
+    async def send_file(self, *a, **kw):
+        self.attempts += 1
+        raise self.error
+
+
+async def drive(error, cycles=3):
+    queue = FakeQueue([{"post_id": 117, "text": "hello", "media_path": None}])
+    repo = FakeRepo([make_target()])
+    notes = []
+
+    async def notify(text):
+        notes.append(text)
+
+    pub = PublisherService(repo=repo, queue=queue, client=FailingClient(error), notify_fn=notify)
+    for _ in range(cycles):
+        await pub._process_all_target_queues()
+    return queue, notes
+
+
+async def _cleanup_error_rows():
+    """log_exception writes to the real table even from fakes; remove those rows."""
+    from db.database import init_db, get_db_pool
+    await init_db()
+    pool = await get_db_pool()
+    async with pool.acquire() as conn:
+        await conn.execute(
+            "DELETE FROM error_logs WHERE service_name = 'publisher.publish' "
+            "AND context->>'target_id' = '2' AND context->>'post_id' = '117';")
+
+
+async def run_tests():
+    # Permanent: drained after the first attempt, admins told exactly once.
+    queue, notes = await drive(ChatAdminRequiredError(request=None))
+    assert queue.pushes == 0, f"permanent failure must not requeue, got {queue.pushes} pushes"
+    assert len(queue.items) == 0, f"queue should be drained, still holds {queue.items}"
+    assert len(notes) == 1, f"admins should be warned exactly once, got {len(notes)}"
+    assert "newsjoker" in notes[0] and "-1003848540849" in notes[0]
+
+    # Transient: the post survives every failed cycle.
+    queue, notes = await drive(ConnectionError("network blip"))
+    assert queue.pushes == 3, f"transient failure must requeue each time, got {queue.pushes}"
+    assert len(queue.items) == 1, f"post must still be queued, got {queue.items}"
+    assert notes == [], "a transient blip must not raise a permanent-failure alarm"
+
+    await _cleanup_error_rows()
+    from db.database import close_db_pool
+    await close_db_pool()
+    print("All publisher failure-handling tests passed successfully!")
+
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())

tests/test_scrape_history.py

@@ -0,0 +1,126 @@
+"""Covers the flow behind the '📥 استخراج ۲۰ پست' button, which used to abort silently."""
+import asyncio
+import sys
+import types
+
+sys.path.insert(0, "/app")
+
+from db.database import init_db, close_db_pool
+from db.repository import Repository
+from services.collector import CollectorService
+
+SOURCE_ID = -1009999000010
+
+
+class FakeMessage:
+    def __init__(self, msg_id, text):
+        self.id = msg_id
+        self.raw_text = text
+        self.media = None
+
+    async def download_media(self, file=None):
+        return None
+
+
+class FakeClient:
+    """Minimal stand-in for the Telethon client used by the collector."""
+
+    def __init__(self, messages):
+        self._messages = messages
+
+    def is_connected(self):
+        return True
+
+    async def is_user_authorized(self):
+        return True
+
+    async def get_entity(self, ident):
+        return types.SimpleNamespace(id=abs(SOURCE_ID), title="Scrape Source")
+
+    def iter_messages(self, entity, limit=20):
+        async def gen():
+            for m in self._messages[:limit]:
+                yield m
+        return gen()
+
+
+async def _cleanup(repo):
+    pool = await repo._get_pool()
+    async with pool.acquire() as conn:
+        await conn.execute("DELETE FROM posts WHERE source_channel_id = $1;", SOURCE_ID)
+        await conn.execute("DELETE FROM sources WHERE channel_id = $1;", SOURCE_ID)
+        # The failure-path assertion logs a real error row; don't leave it behind.
+        await conn.execute(
+            "DELETE FROM error_logs WHERE context->>'channel_id' = $1;", str(SOURCE_ID))
+
+
+async def run_tests():
+    await init_db()
+    repo = Repository()
+    await _cleanup(repo)
+    await repo.add_source(SOURCE_ID, "Scrape Source", None)
+
+    reviewed = []
+    collector = CollectorService(repo=repo, on_post_received=lambda pid: _record(reviewed, pid))
+    # 20 distinct messages plus one exact repeat of the first, to exercise dedup.
+    messages = [FakeMessage(1000 + i, f"historical post number {i}") for i in range(20)]
+    messages.append(FakeMessage(1099, "historical post number 0"))
+    collector.client = FakeClient(messages)
+
+    progress = []
+
+    async def on_progress(text):
+        progress.append(text)
+
+    res = await collector.scrape_channel_history(SOURCE_ID, limit=21, progress_callback=on_progress)
+
+    assert res.collected == 21, f"expected 21 posts collected, got {res.collected}"
+    assert res.scanned == 21, f"expected 21 messages scanned, got {res.scanned}"
+    assert res.duplicates == 1, f"expected 1 duplicate, got {res.duplicates}"
+    assert res.already_stored == 0, f"expected 0 already-stored, got {res.already_stored}"
+    assert len(reviewed) == 21, f"expected 21 review cards dispatched, got {len(reviewed)}"
+    assert progress and progress[-1].startswith("✅"), f"admin was not told the result: {progress}"
+
+    # Re-running the same window adds nothing new, and the report must say WHY
+    # rather than looking like a dead button.
+    progress.clear()
+    again = await collector.scrape_channel_history(SOURCE_ID, limit=21, progress_callback=on_progress)
+    assert again.collected == 0, f"repeat scrape should add nothing, got {again.collected}"
+    assert again.already_stored == 21, f"expected 21 already-stored, got {again.already_stored}"
+    assert progress[-1].startswith("ℹ️"), f"repeat scrape must be explained: {progress[-1]}"
+    assert "قبلا ذخیره شده" in progress[-1]
+
+    stored = await repo.get_posts_by_status("pending_review", limit=100)
+    mine = [p for p in stored if p.source_channel_id == SOURCE_ID]
+    assert len(mine) == 21, f"expected 21 stored posts, got {len(mine)}"
+
+    # The repeated text must be flagged against the original rather than stored blind.
+    dupes = [p for p in mine if p.is_duplicate]
+    assert len(dupes) == 1, f"expected exactly 1 duplicate flagged, got {len(dupes)}"
+    assert dupes[0].duplicate_of_id is not None
+
+    # A channel that cannot be resolved reports the failure instead of vanishing.
+    class BrokenClient(FakeClient):
+        async def get_entity(self, ident):
+            raise RuntimeError("channel not reachable")
+
+        async def get_dialogs(self, limit=50):
+            return []
+
+    collector.client = BrokenClient([])
+    progress.clear()
+    result = await collector.scrape_channel_history(SOURCE_ID, limit=5, progress_callback=on_progress)
+    assert result.collected == 0 and result.error
+    assert progress and progress[-1].startswith("❌"), f"failure was not reported: {progress}"
+
+    await _cleanup(repo)
+    await close_db_pool()
+    print("All scrape-history tests passed successfully!")
+
+
+async def _record(bucket, post_id):
+    bucket.append(post_id)
+
+
+if __name__ == "__main__":
+    asyncio.run(run_tests())