feat: add source websites with automated endpoint analysis, target queue dispatch ordering, and markdown styling

db/repository.py

@@ -2,7 +2,7 @@ import json
 import asyncpg
 from datetime import datetime, timezone
 from typing import List, Optional, Dict, Any, Tuple
-from db.models import SourceChannel, TargetChannel, Post, Setting, AILog, AIProviderProfile, ChannelCategory
+from db.models import SourceChannel, SourceWebsite, TargetChannel, Post, Setting, AILog, AIProviderProfile, ChannelCategory
 from db.database import get_db_pool
 
 
@@ -17,6 +17,18 @@ def _parse_post_row(row: asyncpg.Record) -> Post:
         data["published_to"] = []
     return Post(**data)
 
+
+def _parse_source_website_row(row: asyncpg.Record) -> SourceWebsite:
+    data = dict(row)
+    if isinstance(data.get("api_config"), str):
+        try:
+            data["api_config"] = json.loads(data["api_config"])
+        except Exception:
+            data["api_config"] = {}
+    elif data.get("api_config") is None:
+        data["api_config"] = {}
+    return SourceWebsite(**data)
+
 class Repository:
     def __init__(self, dsn: Optional[str] = None):
         self.dsn = dsn
@@ -72,6 +84,117 @@ class Repository:
         async with pool.acquire() as conn:
             await conn.execute("UPDATE sources SET is_active = FALSE WHERE id = $1;", source_id)
 
+    # --- Source Websites ---
+    async def add_source_website(
+        self,
+        name: str,
+        url: str,
+        category_id: Optional[int] = None,
+        check_interval_min: int = 30,
+        auto_reanalyze_hours: int = 24,
+        api_config: Optional[Dict[str, Any]] = None
+    ) -> int:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            cfg_json = json.dumps(api_config or {})
+            row = await conn.fetchrow(
+                """
+                INSERT INTO source_websites (name, url, category_id, check_interval_min, auto_reanalyze_hours, api_config)
+                VALUES ($1, $2, $3, $4, $5, $6::jsonb)
+                ON CONFLICT(url) DO UPDATE SET
+                    name = EXCLUDED.name,
+                    category_id = COALESCE(EXCLUDED.category_id, source_websites.category_id),
+                    is_active = TRUE
+                RETURNING id;
+                """,
+                name, url, category_id, check_interval_min, auto_reanalyze_hours, cfg_json
+            )
+            return row["id"]
+
+    async def get_active_source_websites(self) -> List[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE ORDER BY id ASC;")
+            return [_parse_source_website_row(r) for r in rows]
+
+    async def get_source_websites(self) -> List[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE ORDER BY id ASC;")
+            return [_parse_source_website_row(r) for r in rows]
+
+    async def get_source_website_by_id(self, site_id: int) -> Optional[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            row = await conn.fetchrow("SELECT * FROM source_websites WHERE id = $1;", site_id)
+            return _parse_source_website_row(row) if row else None
+
+    async def get_source_websites_by_category(self, category_id: Optional[int]) -> List[SourceWebsite]:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            if category_id is None:
+                rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE AND category_id IS NULL ORDER BY id ASC;")
+            else:
+                rows = await conn.fetch("SELECT * FROM source_websites WHERE is_active = TRUE AND category_id = $1 ORDER BY id ASC;", category_id)
+            return [_parse_source_website_row(r) for r in rows]
+
+    async def update_source_website_category(self, site_id: int, category_id: Optional[int]) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET category_id = $1 WHERE id = $2;", category_id, site_id)
+
+    async def update_source_website_api_config(self, site_id: int, api_config: Dict[str, Any]) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(
+                """
+                UPDATE source_websites
+                SET api_config = $1::jsonb,
+                    last_reanalyzed_at = CURRENT_TIMESTAMP,
+                    last_error = NULL,
+                    last_error_at = NULL
+                WHERE id = $2;
+                """,
+                json.dumps(api_config), site_id
+            )
+
+    async def update_source_website_fetch_status(self, site_id: int, error: Optional[str] = None) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            if error:
+                await conn.execute(
+                    """
+                    UPDATE source_websites
+                    SET last_error = $1, last_error_at = CURRENT_TIMESTAMP
+                    WHERE id = $2;
+                    """,
+                    error, site_id
+                )
+            else:
+                await conn.execute(
+                    """
+                    UPDATE source_websites
+                    SET last_fetched_at = CURRENT_TIMESTAMP, last_error = NULL, last_error_at = NULL
+                    WHERE id = $1;
+                    """,
+                    site_id
+                )
+
+    async def update_source_website_interval(self, site_id: int, interval_min: int) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET check_interval_min = $1 WHERE id = $2;", max(1, interval_min), site_id)
+
+    async def update_source_website_reanalyze_hours(self, site_id: int, hours: int) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET auto_reanalyze_hours = $1 WHERE id = $2;", max(0, hours), site_id)
+
+    async def delete_source_website(self, site_id: int) -> None:
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute("UPDATE source_websites SET is_active = FALSE WHERE id = $1;", site_id)
+
     async def delete_target(self, target_id: int) -> None:
         pool = await self._get_pool()
         async with pool.acquire() as conn:
@@ -142,6 +265,15 @@ class Repository:
                 custom_prompt, target_id
             )
 
+    async def update_target_dispatch_order(self, target_id: int, dispatch_order: str) -> None:
+        order = "random" if str(dispatch_order).strip().lower() == "random" else "order"
+        pool = await self._get_pool()
+        async with pool.acquire() as conn:
+            await conn.execute(
+                "UPDATE targets SET dispatch_order = $1 WHERE id = $2;",
+                order, target_id
+            )
+
 
     async def update_target_schedule(
         self,