feat: add source websites with automated endpoint analysis, target queue dispatch ordering, and markdown styling

main.py

@@ -13,6 +13,8 @@ from services.ai_processor import AIProcessor
 from services.collector import CollectorService
 from services.admin_bot import AdminBotService
 from services.publisher import PublisherService
+from services.website_analyzer import WebsiteAnalyzer
+from services.website_collector import WebsiteCollectorService
 
 # Load environment variables
 load_dotenv()
@@ -51,6 +53,17 @@ async def main():
     )
     ai_processor = AIProcessor(repo=repo, llm=llm)
     admin_bot.set_ai_processor(ai_processor)
+
+    website_analyzer = WebsiteAnalyzer(llm=llm)
+    website_collector = WebsiteCollectorService(
+        repo=repo,
+        analyzer=website_analyzer,
+        ai_processor=ai_processor,
+        on_post_received=admin_bot.handle_collected_post,
+        on_error_alert=admin_bot.on_website_error_alert
+    )
+    admin_bot.set_website_services(website_analyzer, website_collector)
+
     collector = CollectorService(repo=repo, on_post_received=admin_bot.handle_collected_post, ai_processor=ai_processor)
     admin_bot.set_collector(collector)
     
@@ -67,6 +80,7 @@ async def main():
     # 4. Start all services
     await admin_bot.start()
     await collector.start(notify_fn=admin_bot.notify_admins)
+    await website_collector.start()
     await publisher.start()
 
     logger.info("All Copykar services are active and running.")
@@ -91,6 +105,7 @@ async def main():
         pass
     finally:
         logger.info("Shutting down Copykar services...")
+        await website_collector.stop()
         await collector.stop()
         await publisher.stop()
         await admin_bot.stop()