diff --git a/core/metrics.py b/core/metrics.py index df767d1..aa35272 100644 --- a/core/metrics.py +++ b/core/metrics.py @@ -1,15 +1,11 @@ from prometheus_client import Counter, Histogram, Gauge, start_http_server import logging +from typing import Optional logger = logging.getLogger(__name__) # Counters -COLLECTED_POSTS_TOTAL = Counter( - "copykar_posts_collected_total", - "Total posts collected by the Telethon Userbot", - ["source_channel_id"] -) - +# Ingest is counted once, by SOURCE_ACTIVITY_TOTAL, which also carries the channel title. SOURCE_ACTIVITY_TOTAL = Counter( "copykar_source_activity_total", "Total posts ingested per source channel", @@ -40,10 +36,11 @@ ADMIN_ACTIONS_TOTAL = Counter( ["action"] ) -POSTS_PUBLISHED_TOTAL = Counter( - "copykar_posts_published_total", - "Total posts successfully published to target channels", - ["target_channel_id"] +# Delivery is counted once, by TARGET_ACTIVITY_TOTAL. +AUTO_ROUTED_POSTS_TOTAL = Counter( + "copykar_auto_routed_posts_total", + "Posts queued automatically by a source-to-target route", + ["source_channel_id", "target_title"] ) ERRORS_TOTAL = Counter( @@ -52,6 +49,12 @@ ERRORS_TOTAL = Counter( ["service", "error_type"] ) +ERRORS_RESOLVED_TOTAL = Counter( + "copykar_errors_resolved_total", + "Errors an admin has marked as fixed", + ["service", "error_type"] +) + # Histograms AI_LATENCY_SECONDS = Histogram( "copykar_ai_latency_seconds", @@ -66,14 +69,136 @@ QUEUE_POSTS_GAUGE = Gauge( ["status"] ) -REDIS_QUEUE_SIZE_GAUGE = Gauge( - "copykar_redis_queue_size", - "Current number of posts waiting in Redis incoming queue" +ERRORS_OPEN_GAUGE = Gauge( + "copykar_errors_open", + "Unresolved errors currently recorded, by service and exception type", + ["service", "error_type"] ) -def start_metrics_server(port: int = 8000): +ERRORS_OPEN_TOTAL_GAUGE = Gauge( + "copykar_errors_open_total", + "Total unresolved errors across all services" +) + +import shutil +import asyncio + +# Disk Space Gauges per Mount Point +DISK_TOTAL_BYTES = Gauge( + "copykar_disk_total_bytes", + "Total disk capacity in bytes", + ["mountpoint", "device"] +) + +DISK_USED_BYTES = Gauge( + "copykar_disk_used_bytes", + "Used disk space in bytes", + ["mountpoint", "device"] +) + +DISK_FREE_BYTES = Gauge( + "copykar_disk_free_bytes", + "Free/available disk space in bytes", + ["mountpoint", "device"] +) + +DISK_FREE_PERCENT = Gauge( + "copykar_disk_free_percent", + "Percentage of free disk space", + ["mountpoint", "device"] +) + + +import os + +VOLUME_DEFINITIONS = [ + { + "mountpoint": "/", + "device": "/dev/sda1", + "container_paths": ["/host_os/home", "/hostfs/host_mnt/home", "/home", "/host_os_disk", "/hostfs", "/"] + }, + { + "mountpoint": "/projects", + "device": "/dev/sda2", + "container_paths": ["/host_os/projects", "/projects"] + }, + { + "mountpoint": "/boot", + "device": "/dev/sda4", + "container_paths": ["/host_os/boot", "/boot"] + }, + { + "mountpoint": "/boot/efi", + "device": "/dev/sda3", + "container_paths": ["/host_os/boot_efi", "/boot/efi"] + }, + { + "mountpoint": "/run/media/mamad/WIN11_25H2_", + "device": "/dev/sdc1", + "container_paths": ["/host_os/media/mamad/WIN11_25H2_", "/run/media/mamad/WIN11_25H2_"] + } +] + + +def update_disk_metrics(): + """Update Prometheus gauges with real host OS filesystem disk usage for each mounted volume.""" + try: + recorded_mounts = set() + for v in VOLUME_DEFINITIONS: + mnt = v["mountpoint"] + dev = v["device"] + for path in v["container_paths"]: + if os.path.exists(path) and os.path.isdir(path): + try: + total, used, free = shutil.disk_usage(path) + DISK_TOTAL_BYTES.labels(mountpoint=mnt, device=dev).set(total) + DISK_USED_BYTES.labels(mountpoint=mnt, device=dev).set(used) + DISK_FREE_BYTES.labels(mountpoint=mnt, device=dev).set(free) + free_pct = (free / total * 100.0) if total > 0 else 0.0 + DISK_FREE_PERCENT.labels(mountpoint=mnt, device=dev).set(free_pct) + recorded_mounts.add(mnt) + break + except Exception as err: + logger.debug(f"Error measuring disk usage for {path}: {err}") + + # If none of the specific mounts matched, fallback to root + if not recorded_mounts: + total, used, free = shutil.disk_usage("/") + DISK_TOTAL_BYTES.labels(mountpoint="/", device="default").set(total) + DISK_USED_BYTES.labels(mountpoint="/", device="default").set(used) + DISK_FREE_BYTES.labels(mountpoint="/", device="default").set(free) + free_pct = (free / total * 100.0) if total > 0 else 0.0 + DISK_FREE_PERCENT.labels(mountpoint="/", device="default").set(free_pct) + + except Exception as e: + logger.debug(f"Error updating host disk metrics: {e}") + + + +async def _disk_metrics_loop(interval: int = 15): + """Background task to keep disk metrics updated.""" + while True: + try: + update_disk_metrics() + except Exception as e: + logger.debug(f"Disk metrics loop error: {e}") + await asyncio.sleep(interval) + + +# The overall queue depth is sum(copykar_posts_queue_gauge) - no separate total series, +# which previously made the dashboard report the queue twice. +def start_metrics_server(port: int = 8008): try: start_http_server(port) + update_disk_metrics() + try: + loop = asyncio.get_event_loop() + if loop.is_running(): + asyncio.create_task(_disk_metrics_loop()) + except Exception: + pass logger.info(f"Prometheus metrics server running on port {port}") except Exception as e: logger.error(f"Failed to start Prometheus metrics server: {e}") + + diff --git a/monitoring/grafana/dashboards/copykar.json b/monitoring/grafana/dashboards/copykar.json index 8bda1f4..bd6b715 100644 --- a/monitoring/grafana/dashboards/copykar.json +++ b/monitoring/grafana/dashboards/copykar.json @@ -11,298 +11,1309 @@ "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 100, - "title": "📌 شاخص‌های کلیدی عملکرد و صف‌های سیستم (KPIs & Queues)", - "type": "row" + "title": "📌 شاخص‌های کلیدی سیستم (KPIs)", + "type": "row", + "panels": [] }, { - "collapsed": false, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 1 }, "id": 1, + "type": "stat", "title": "📥 کل پست‌های دریافتی از مبدا", - "description": "تعداد کل پیام‌های دریافت شده توسط ربات از کانال‌های مبدا تحت مانیتور.", - "type": "stat", + "description": "مجموع پیام‌هایی که از تمام کانال‌های مبدا دریافت و ذخیره شده‌اند.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, "targets": [ { - "expr": "sum(copykar_posts_collected_total) or vector(0)", - "legendFormat": "دریافت شده", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "sum(copykar_source_activity_total) or vector(0)", + "refId": "A", + "instant": true } ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } - } + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] } }, { - "collapsed": false, - "gridPos": { "h": 4, "w": 8, "x": 8, "y": 1 }, "id": 2, - "title": "⏳ مجموع پست‌های در صف ارسال مقصد", - "description": "تعداد پست‌های تایید شده که در صف ردیس کانال‌های مقصد منتظر رسیدن نوبت ارسال یا پایان ساعت خواب هستند.", - "type": "stat", - "targets": [ - { - "expr": "copykar_redis_queue_size or vector(0)", - "legendFormat": "در صف ارسال", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 10 }, - { "color": "red", "value": 50 } - ] - } - } - } - }, - { - "collapsed": false, - "gridPos": { "h": 4, "w": 8, "x": 16, "y": 1 }, - "id": 3, - "title": "🚀 کل پست‌های منتشر شده نهایی", - "description": "تعداد پست‌هایی که با موفقیت در کانال‌های مقصد نهایی ارسال و منتشر شده‌اند.", "type": "stat", + "title": "🚀 کل پست‌های منتشر شده", + "description": "مجموع پست‌هایی که با موفقیت در کانال‌های مقصد ارسال شده‌اند.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, "expr": "sum(copykar_target_activity_total) or vector(0)", - "legendFormat": "منتشر شده", - "refId": "A" + "refId": "A", + "instant": true } ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": null }] } - } - } - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "id": 101, - "title": "📋 تصمیمات ادمین و وضعیت سلامت خطاها (Admin Actions & Error Telemetry)", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 6 }, - "id": 4, - "title": "✅ پست‌های تایید شده ادمین", - "description": "تعداد پست‌هایی که ادمین بازنویسی آن‌ها را تایید کرده و به صف ارسال فرستاده است.", - "type": "stat", - "targets": [ - { - "expr": "sum(copykar_admin_actions_total{action=\"approved\"}) or vector(0)", - "legendFormat": "تایید شده", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } - } - } - }, - { - "collapsed": false, - "gridPos": { "h": 4, "w": 8, "x": 8, "y": 6 }, - "id": 5, - "title": "❌ پست‌های رد و بایگانی شده", - "description": "تعداد پست‌هایی که توسط ادمین در کانال بررسی رد شده‌اند.", - "type": "stat", - "targets": [ - { - "expr": "sum(copykar_admin_actions_total{action=\"rejected\"}) or vector(0)", - "legendFormat": "رد شده", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] } - } - } - }, - { - "collapsed": false, - "gridPos": { "h": 4, "w": 8, "x": 16, "y": 6 }, - "id": 6, - "title": "⚠️ مجموع خطاهای ثبت شده در سیستم", - "description": "تعداد کل استثناها و خطاهایی که در دیتابیس PostgreSQL و لاگ‌ها ثبت شده است.", - "type": "stat", - "targets": [ - { - "expr": "sum(copykar_errors_total) or vector(0)", - "legendFormat": "خطاها", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, + "unit": "short", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "purple", + "value": null + } ] } + }, + "overrides": [] + } + }, + { + "id": 3, + "type": "stat", + "title": "⚠️ خطاهای رفع‌نشده", + "description": "خطاهایی که هنوز توسط ادمین رفع‌شده علامت نخورده‌اند. با دکمه «⚠️ خطاهای سیستم» در ربات قابل مدیریت است.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "copykar_errors_open_total or vector(0)", + "refId": "A", + "instant": true } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] } }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, - "id": 102, - "title": "📡 فعالیت به تفکیک کانال‌های مبدا و مقصد (Channels Activity Breakdown)", - "type": "row" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 106, + "title": "📦 صف ارسال و تصمیمات ادمین (Queue & Review)", + "type": "row", + "panels": [] + }, + { + "id": 4, + "type": "timeseries", + "title": "📥 عمق صف هر کانال مقصد", + "description": "تعداد پست‌های منتظر ارسال در صف ردیس، به تفکیک کانال مقصد. مجموع این نمودار همان کل صف است.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 7 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "copykar_posts_queue_gauge", + "legendFormat": "{{status}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + } + } + }, + "overrides": [] + } + }, + { + "id": 5, + "type": "timeseries", + "title": "✅ تصمیمات ادمین در طول زمان", + "description": "نرخ تایید، رد و ارسال خودکار پست‌ها (تعداد در دقیقه).", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 7 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "sum by (action) (rate(copykar_admin_actions_total[5m]) * 60)", + "legendFormat": "{{action}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + }, + { + "id": 6, + "type": "stat", + "title": "🤖 پست‌های ارسال‌شده خودکار", + "description": "پست‌هایی که از طریق «ارسال خودکار از مبدا» بدون تایید دستی وارد صف شده‌اند.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 7 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "sum(copykar_auto_routed_posts_total) or vector(0)", + "refId": "A", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } }, { "collapsed": false, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 115, + "title": "📡 فعالیت کانال‌ها (Channels Activity)", + "type": "row", + "panels": [] + }, + { "id": 7, - "title": "📥 پست‌های دریافتی به تفکیک کانال مبدا", - "description": "نمودار زمانی و حجم پست‌های جمع‌آوری شده از هر یک از کانال‌های مبدا تحت مانیتور.", "type": "timeseries", + "title": "📥 دریافت به تفکیک کانال مبدا", + "description": "تعداد تجمعی پست‌های دریافت‌شده از هر کانال مبدا.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", "expr": "sum by (title) (copykar_source_activity_total)", - "legendFormat": "مبدا: {{title}}", - "refId": "A" + "legendFormat": "{{title}}" } - ] + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } }, { - "collapsed": false, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "id": 8, - "title": "🚀 پست‌های منتشر شده به تفکیک کانال مقصد", - "description": "نمودار زمانی تعداد پست‌های ارسال شده به هر یک از کانال‌های مقصد نهایی.", "type": "timeseries", + "title": "🚀 انتشار به تفکیک کانال مقصد", + "description": "تعداد تجمعی پست‌های ارسال‌شده به هر کانال مقصد.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", "expr": "sum by (title) (copykar_target_activity_total)", - "legendFormat": "مقصد: {{title}}", - "refId": "A" + "legendFormat": "{{title}}" } - ] + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } }, { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, - "id": 103, - "title": "🤖 عملیات هوش مصنوعی و تحلیل خطاها (AI Operations & Error Breakdown)", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "id": 9, - "title": "🤖 تعداد و وضعیت درخواست‌های هوش مصنوعی", - "description": "نرخ بازنویسی‌های ارسالی به هوش مصنوعی به تفکیک وضعیت پاسخ و نوع عملیات.", - "type": "timeseries", + "type": "stat", + "title": "♻️ پست‌های تکراری شناسایی‌شده", + "description": "پست‌هایی که هش محتوایشان با یک پست قبلی یکی بوده و به‌عنوان تکراری علامت خورده‌اند.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, "targets": [ { - "expr": "sum by (action, status) (rate(copykar_ai_requests_total[1m]) * 60)", - "legendFormat": "{{action}} ({{status}})", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "sum(copykar_duplicates_detected_total) or vector(0)", + "refId": "A", + "instant": true } - ] + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + } }, { "collapsed": false, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 124, + "title": "🤖 عملکرد هوش مصنوعی (AI Operations)", + "type": "row", + "panels": [] + }, + { "id": 10, - "title": "⚠️ خطاهای سیستم به تفکیک سرویس و نوع استثنا", - "description": "تحلیل آماری و نموداری خطاهای رخ داده در سیستم و ثبت شده در پایگاه داده.", "type": "timeseries", + "title": "🤖 نرخ درخواست‌های هوش مصنوعی", + "description": "تعداد درخواست‌های بازنویسی در دقیقه، به تفکیک وضعیت موفق/ناموفق.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 25 + }, "targets": [ { - "expr": "sum by (service, error_type) (copykar_errors_total)", - "legendFormat": "{{service}}: {{error_type}}", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "sum by (status) (rate(copykar_ai_requests_total[5m]) * 60)", + "legendFormat": "{{status}}" } - ] + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } }, { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, - "id": 104, - "title": "⚡ مانیتورینگ منابع سرور و کانتینر (Container Telemetry)", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 }, "id": 11, - "title": "⚡ درصد مصرف پردازنده (CPU %)", - "description": "میزان مصرف پردازنده توسط سرویس کپی‌کار.", "type": "timeseries", + "title": "⏱ زمان پاسخ هوش مصنوعی", + "description": "صدک ۹۵ و میانگین زمان پاسخ‌دهی مدل. اگر بالا باشد یعنی مدل اصلی محدود شده و fallback فعال است.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 25 + }, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"copykar\"}[1m]) * 100", - "legendFormat": "مصرف CPU %", - "refId": "A" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 }, - "id": 12, - "title": "💾 میزان مصرف حافظه رم (RAM MB)", - "description": "حافظه رم اشغال شده توسط پردازش برنامه به مگابایت.", - "type": "timeseries", - "targets": [ - { - "expr": "process_resident_memory_bytes{job=\"copykar\"} / 1024 / 1024", - "legendFormat": "حافظه رم (MB)", - "refId": "A" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 }, - "id": 13, - "title": "⏱ مدت زمان پاسخ‌دهی هوش مصنوعی (ثانیه)", - "description": "میانگین زمان و صدک ۹۵ ام برای بازنویسی پست‌ها متناسب با شخصیت هر کانال.", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(copykar_ai_latency_seconds_bucket[5m])) by (le, action))", - "legendFormat": "صدک ۹۵ام (p95): {{action}}", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(copykar_ai_latency_seconds_bucket[5m])))", + "legendFormat": "صدک ۹۵" }, { - "expr": "rate(copykar_ai_latency_seconds_sum[5m]) / rate(copykar_ai_latency_seconds_count[5m])", - "legendFormat": "میانگین (Avg): {{action}}", - "refId": "B" + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "B", + "expr": "sum(rate(copykar_ai_latency_seconds_sum[5m])) / sum(rate(copykar_ai_latency_seconds_count[5m]))", + "legendFormat": "میانگین" } - ] + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + }, + { + "id": 12, + "type": "stat", + "title": "🤖 مجموع درخواست‌های موفق", + "description": "تعداد کل بازنویسی‌های موفق هوش مصنوعی.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 25 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "sum(copykar_ai_requests_total{status=\"success\"}) or vector(0)", + "refId": "A", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 133, + "title": "⚠️ سلامت و خطاهای سیستم (Error Health)", + "type": "row", + "panels": [] + }, + { + "id": 13, + "type": "bargauge", + "title": "⚠️ خطاهای باز به تفکیک سرویس", + "description": "خطاهای رفع‌نشده، گروه‌بندی‌شده بر اساس سرویس و نوع استثنا.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 34 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "copykar_errors_open", + "legendFormat": "{{service}} / {{error_type}}", + "refId": "A", + "instant": true + } + ], + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "continuous-GrYlRd" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 14, + "type": "timeseries", + "title": "📈 روند بروز خطا", + "description": "نرخ بروز خطاهای جدید در دقیقه، به تفکیک سرویس.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 34 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "sum by (service) (rate(copykar_errors_total[5m]) * 60)", + "legendFormat": "{{service}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + }, + { + "id": 15, + "type": "stat", + "title": "✅ خطاهای رفع‌شده", + "description": "تعداد خطاهایی که ادمین آن‌ها را رفع‌شده علامت زده است.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 34 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "expr": "sum(copykar_errors_resolved_total) or vector(0)", + "refId": "A", + "instant": true + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 142, + "title": "⚡ منابع سرور (Container Telemetry)", + "type": "row", + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "⚡ مصرف پردازنده", + "description": "درصد مصرف CPU توسط کانتینر کپی‌کار.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "rate(process_cpu_seconds_total{job=\"copykar\"}[5m]) * 100", + "legendFormat": "CPU %" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + }, + { + "id": 17, + "type": "timeseries", + "title": "💾 مصرف حافظه", + "description": "میزان حافظه RAM اشغال‌شده توسط کانتینر کپی‌کار.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "process_resident_memory_bytes{job=\"copykar\"}", + "legendFormat": "RAM" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } + }, + { + "id": 18, + "type": "bargauge", + "title": "💽 فضای خالی هر درایو و Mount Point (Free Space)", + "description": "نمایش حجم و فضای خالی تفکیکی هر یک از درایوها و پارتیشن‌های مونت‌شده سیستم‌عامل به صورت خوانا (Human-Readable).", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "copykar_disk_free_bytes", + "legendFormat": "{{mountpoint}} ({{device}})", + "instant": true + } + ], + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "showUnfilled": true, + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1073741824 + }, + { + "color": "green", + "value": 5368709120 + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 19, + "type": "gauge", + "title": "📈 درصد فضای آزاد هر پارتیشن (Free % Gauges)", + "description": "درصد فضای خالی هر پارتیشن مونت‌شده سیستم‌عامل.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "copykar_disk_free_percent", + "legendFormat": "{{mountpoint}}", + "instant": true + } + ], + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 15 + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + "overrides": [] + } + }, + { + "id": 20, + "type": "timeseries", + "title": "💽 روند زمانی فضای آزاد هر درایو (Free Space History)", + "description": "روند زمانی فضای آزاد هر یک از درایوهای مونت‌شده سیستم‌عامل.", + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 59 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "copykar-prometheus" + }, + "refId": "A", + "expr": "copykar_disk_free_bytes", + "legendFormat": "{{mountpoint}} ({{device}})" + } + ], + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull", + "min", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none", + "group": "A" + } + } + }, + "overrides": [] + } } ], - "refresh": "5s", - "schemaVersion": 38, - "style": "dark", - "tags": ["copykar", "telegram", "ai", "telemetry", "persian", "errors"], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "copykar" + ], + "templating": { + "list": [] + }, "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", - "title": "داشبورد مدیریت ناوگان کپی‌کار (Copykar Executive Dashboard)", + "title": "داشبورد مدیریت کپی‌کار (Copykar Dashboard)", "uid": "copykar-executive-dashboard", - "version": 6 -} + "version": 7, + "weekStart": "" +} \ No newline at end of file diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml index bb009bb..d05133f 100644 --- a/monitoring/grafana/provisioning/datasources/prometheus.yml +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -2,6 +2,7 @@ apiVersion: 1 datasources: - name: Prometheus + uid: copykar-prometheus type: prometheus access: proxy url: http://prometheus:9090 diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index a2c9bfb..00ef85f 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -5,4 +5,5 @@ global: scrape_configs: - job_name: "copykar" static_configs: - - targets: ["copykar:8000"] + - targets: ["copykar:8008"] + diff --git a/services/metrics_reporter.py b/services/metrics_reporter.py new file mode 100644 index 0000000..f4c7e9d --- /dev/null +++ b/services/metrics_reporter.py @@ -0,0 +1,306 @@ +import os +import io +import time +import httpx +import logging +from typing import Dict, Any, List, Optional, Tuple + +logger = logging.getLogger(__name__) + +PROMETHEUS_URL = os.getenv( + "PROMETHEUS_URL", + "http://prometheus:9090" if os.path.exists("/.dockerenv") else "http://localhost:9090" +) + +# Colors and style configuration for dark-mode graphs +BG_COLOR = "#12171f" +PANEL_BG = "#1a2230" +GRID_COLOR = "#2c384d" +TEXT_COLOR = "#e2e8f0" +CYAN = "#38bdf8" +GREEN = "#4ade80" +PURPLE = "#a855f7" +YELLOW = "#facc15" +RED = "#f87171" +ORANGE = "#fb923c" + + +async def _query_instant(query: str) -> Optional[float]: + url = f"{PROMETHEUS_URL.rstrip('/')}/api/v1/query" + try: + async with httpx.AsyncClient(timeout=4.0) as client: + resp = await client.get(url, params={"query": query}) + if resp.status_code == 200: + data = resp.json() + results = data.get("data", {}).get("result", []) + if results: + val = results[0].get("value", [0, "0"])[1] + return float(val) + except Exception as e: + logger.debug(f"Prometheus instant query failed ({query}): {e}") + return None + + +async def _query_vector(query: str) -> List[Dict[str, Any]]: + url = f"{PROMETHEUS_URL.rstrip('/')}/api/v1/query" + try: + async with httpx.AsyncClient(timeout=4.0) as client: + resp = await client.get(url, params={"query": query}) + if resp.status_code == 200: + data = resp.json() + return data.get("data", {}).get("result", []) + except Exception as e: + logger.debug(f"Prometheus vector query failed ({query}): {e}") + return [] + + +async def _query_range(query: str, start: float, end: float, step: str) -> List[Tuple[float, float]]: + url = f"{PROMETHEUS_URL.rstrip('/')}/api/v1/query_range" + try: + async with httpx.AsyncClient(timeout=8.0) as client: + resp = await client.get(url, params={"query": query, "start": start, "end": end, "step": step}) + if resp.status_code == 200: + data = resp.json() + results = data.get("data", {}).get("result", []) + if results: + values = results[0].get("values", []) + return [(float(t), float(v)) for t, v in values] + except Exception as e: + logger.debug(f"Prometheus range query failed ({query}): {e}") + return [] + + +async def get_instant_metrics_report() -> str: + """Fetch current real-time metrics from Prometheus / Grafana data source and format a comprehensive report.""" + # 1. Ingest metrics + total_ingested = await _query_instant("sum(copykar_source_activity_total)") or 0 + ingest_rate = await _query_instant("sum(rate(copykar_source_activity_total[5m])) * 60") or 0.0 + + # 2. Publish metrics + total_published = await _query_instant("sum(copykar_target_activity_total)") or 0 + publish_rate = await _query_instant("sum(rate(copykar_target_activity_total[5m])) * 60") or 0.0 + + # 3. AI metrics + ai_success = await _query_instant('sum(copykar_ai_requests_total{status="success"})') or 0 + ai_error = await _query_instant('sum(copykar_ai_requests_total{status="error"})') or 0 + ai_total = ai_success + ai_error + ai_latency_sum = await _query_instant("sum(copykar_ai_latency_seconds_sum)") or 0.0 + ai_latency_cnt = await _query_instant("sum(copykar_ai_latency_seconds_count)") or 0.0 + avg_latency = (ai_latency_sum / ai_latency_cnt) if ai_latency_cnt > 0 else 0.0 + ai_rate = await _query_instant("sum(rate(copykar_ai_requests_total[5m])) * 60") or 0.0 + + # 4. Queue breakdown + queue_data = await _query_vector("copykar_posts_queue_gauge") + queue_breakdown: Dict[str, int] = {} + for item in queue_data: + st = item.get("metric", {}).get("status", "unknown") + val = int(float(item.get("value", [0, 0])[1])) + queue_breakdown[st] = val + total_queue = sum(queue_breakdown.values()) + + # 5. Duplicates & Errors + duplicates = await _query_instant("sum(copykar_duplicates_detected_total)") or 0 + total_errors = await _query_instant("sum(copykar_errors_total)") or 0 + open_errors = await _query_instant("copykar_errors_open_total") or 0 + resolved_errors = await _query_instant("sum(copykar_errors_resolved_total)") or 0 + + # 6. Admin Actions + admin_actions = await _query_instant("sum(copykar_admin_actions_total)") or 0 + + # 7. Disk Space per mount point + disk_free_data = await _query_vector("copykar_disk_free_bytes") + disk_total_data = await _query_vector("copykar_disk_total_bytes") + disk_pct_data = await _query_vector("copykar_disk_free_percent") + + mount_stats: Dict[str, Dict[str, Any]] = {} + for item in disk_free_data: + m = item.get("metric", {}).get("mountpoint", "/") + val = float(item.get("value", [0, 0])[1]) + mount_stats.setdefault(m, {})["free_gb"] = val / (1024 ** 3) + + for item in disk_total_data: + m = item.get("metric", {}).get("mountpoint", "/") + val = float(item.get("value", [0, 0])[1]) + mount_stats.setdefault(m, {})["total_gb"] = val / (1024 ** 3) + + for item in disk_pct_data: + m = item.get("metric", {}).get("mountpoint", "/") + val = float(item.get("value", [0, 0])[1]) + mount_stats.setdefault(m, {})["pct"] = val + + timestamp_str = time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime()) + + report = ( + "📊 گزارش وضعیت و متریک‌های لحظه‌ای سیستم (Grafana Metrics):\n" + f"🕒 زمان گزارش: {timestamp_str}\n\n" + "📥 ورودی از کانال‌های مبدا (Ingest):\n" + f"• کل پست‌های دریافت شده: {int(total_ingested):,}\n" + f"• نرخ ورودی لحظه‌ای: {ingest_rate:.2f} پست در دقیقه\n\n" + "🚀 انتشار در کانال‌های مقصد (Published):\n" + f"• کل پست‌های منتشر شده: {int(total_published):,}\n" + f"• نرخ انتشار لحظه‌ای: {publish_rate:.2f} پست در دقیقه\n\n" + "🧠 پردازش هوش مصنوعی (AI Engine):\n" + f"• کل درخواست‌ها: {int(ai_total):,} (✅ {int(ai_success)} موفق | ❌ {int(ai_error)} خطا)\n" + f"• میانگین تاخیر پاسخ: {avg_latency:.2f}s\n" + f"• نرخ درخواست: {ai_rate:.2f} req/min\n\n" + "📬 وضعیت صف انتشار (Paced Queue):\n" + f"• کل پیام‌ها در صف: {total_queue}\n" + ) + + if queue_breakdown: + details = " | ".join([f"{k}: {v}" for k, v in queue_breakdown.items()]) + report += f" ({details})\n\n" + else: + report += " (صف خالی است)\n\n" + + report += ( + "🛡 پایش و سلامت سیستم:\n" + f"• پست‌های تکراری شناسایی‌شده: {int(duplicates):,}\n" + f"• خطاهای ثبت‌شده: {int(total_errors):,} (⚠️ {int(open_errors)} باز | ✅ {int(resolved_errors)} رفع‌شده)\n" + f"• اقدامات ادمین: {int(admin_actions):,}\n\n" + ) + + if mount_stats: + report += "💽 فضای ذخیره‌سازی تفکیکی درایوها (Mount Points Storage):\n" + for mnt, data in sorted(mount_stats.items()): + free_gb = data.get("free_gb", 0.0) + tot_gb = data.get("total_gb", 0.0) + pct = data.get("pct", 0.0) + if tot_gb > 0 and tot_gb < 1.0: + free_mb = free_gb * 1024 + tot_mb = tot_gb * 1024 + report += f"• {mnt}: {free_mb:.0f} MB آزاد از {tot_mb:.0f} MB ({pct:.1f}% آزاد)\n" + else: + report += f"• {mnt}: {free_gb:.2f} GB آزاد از {tot_gb:.2f} GB ({pct:.1f}% آزاد)\n" + report += "\n" + + report += "👇 برای دریافت نمودار تصویری متریک‌ها روی بازه زمانی مورد نظر بزنید:" + return report + + +def _generate_chart_image( + time_range_label: str, + ingest_pts: List[Tuple[float, float]], + publish_pts: List[Tuple[float, float]], + ai_pts: List[Tuple[float, float]], + queue_pts: List[Tuple[float, float]], + error_pts: List[Tuple[float, float]], +) -> bytes: + """Generate dark-mode multi-panel metrics graph using matplotlib in memory.""" + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + from datetime import datetime + + plt.style.use("dark_background") + fig, axes = plt.subplots(2, 2, figsize=(12, 7.5), dpi=140) + fig.patch.set_facecolor(BG_COLOR) + fig.suptitle(f"Copykar System Metrics Dashboard ({time_range_label})", fontsize=15, color=TEXT_COLOR, fontweight="bold", y=0.98) + + for ax in axes.flat: + ax.set_facecolor(PANEL_BG) + ax.tick_params(colors=TEXT_COLOR, labelsize=8) + ax.grid(True, linestyle="--", alpha=0.3, color=GRID_COLOR) + for spine in ax.spines.values(): + spine.set_color(GRID_COLOR) + + # 1. Ingest & Publish Rates + ax1 = axes[0, 0] + ax1.set_title("Ingest vs Publish Rate (posts/min)", fontsize=10, color=CYAN, fontweight="bold") + if ingest_pts: + t1 = [datetime.fromtimestamp(p[0]) for p in ingest_pts] + v1 = [p[1] for p in ingest_pts] + ax1.plot(t1, v1, label="Ingested (posts/m)", color=CYAN, linewidth=1.8) + if publish_pts: + t2 = [datetime.fromtimestamp(p[0]) for p in publish_pts] + v2 = [p[1] for p in publish_pts] + ax1.plot(t2, v2, label="Published (posts/m)", color=GREEN, linewidth=1.8) + if ingest_pts or publish_pts: + ax1.legend(loc="upper left", fontsize=8, facecolor=PANEL_BG, edgecolor=GRID_COLOR) + + # 2. AI Request Rate + ax2 = axes[0, 1] + ax2.set_title("AI Request Rate (req/min)", fontsize=10, color=PURPLE, fontweight="bold") + if ai_pts: + t_ai = [datetime.fromtimestamp(p[0]) for p in ai_pts] + v_ai = [p[1] for p in ai_pts] + ax2.plot(t_ai, v_ai, label="AI Calls / min", color=PURPLE, linewidth=1.8) + ax2.fill_between(t_ai, v_ai, color=PURPLE, alpha=0.2) + ax2.legend(loc="upper left", fontsize=8, facecolor=PANEL_BG, edgecolor=GRID_COLOR) + + # 3. Queue Depth + ax3 = axes[1, 0] + ax3.set_title("Queue Depth (Active Posts)", fontsize=10, color=YELLOW, fontweight="bold") + if queue_pts: + t_q = [datetime.fromtimestamp(p[0]) for p in queue_pts] + v_q = [p[1] for p in queue_pts] + ax3.plot(t_q, v_q, label="Queue Size", color=YELLOW, linewidth=1.8) + ax3.fill_between(t_q, v_q, color=YELLOW, alpha=0.2) + ax3.legend(loc="upper left", fontsize=8, facecolor=PANEL_BG, edgecolor=GRID_COLOR) + + # 4. Error Rate + ax4 = axes[1, 1] + ax4.set_title("Error Rate (errors/min)", fontsize=10, color=RED, fontweight="bold") + if error_pts: + t_err = [datetime.fromtimestamp(p[0]) for p in error_pts] + v_err = [p[1] for p in error_pts] + ax4.plot(t_err, v_err, label="Errors / min", color=RED, linewidth=1.8) + ax4.fill_between(t_err, v_err, color=RED, alpha=0.2) + ax4.legend(loc="upper left", fontsize=8, facecolor=PANEL_BG, edgecolor=GRID_COLOR) + + + # Formatting date axes + for ax in axes.flat: + ax.xaxis.set_major_formatter(mdates.DateFormatter("%H:%M")) + fig.autofmt_xdate(rotation=25) + + plt.tight_layout(rect=[0, 0.03, 1, 0.95]) + buf = io.BytesIO() + plt.savefig(buf, format="jpg", facecolor=BG_COLOR, edgecolor="none", bbox_inches="tight", pil_kwargs={"quality": 95}) + plt.close(fig) + buf.seek(0) + return buf.getvalue() + + +async def generate_metrics_graph(time_range: str = "15m") -> Tuple[bytes, str]: + """Fetch time-series range metrics and return a JPG chart image along with its exact timestamped filename.""" + now = time.time() + now_dt = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(now)) + filename = f"copykar_metrics_{time_range}_{now_dt}.jpg" + + if time_range == "15m": + start = now - 15 * 60 + step = "15s" + label = "Last 15 Minutes" + elif time_range == "3h": + start = now - 3 * 3600 + step = "1m" + label = "Last 3 Hours" + elif time_range == "24h": + start = now - 24 * 3600 + step = "5m" + label = "Last 24 Hours" + else: + start = now - 15 * 60 + step = "15s" + label = "Last 15 Minutes" + + rate_window = "1m" if time_range in ("15m", "3h") else "5m" + + ingest_pts = await _query_range(f"sum(rate(copykar_source_activity_total[{rate_window}])) * 60", start, now, step) + publish_pts = await _query_range(f"sum(rate(copykar_target_activity_total[{rate_window}])) * 60", start, now, step) + ai_pts = await _query_range(f"sum(rate(copykar_ai_requests_total[{rate_window}])) * 60", start, now, step) + queue_pts = await _query_range("sum(copykar_posts_queue_gauge)", start, now, step) + error_pts = await _query_range(f"sum(rate(copykar_errors_total[{rate_window}])) * 60", start, now, step) + + chart_bytes = _generate_chart_image( + time_range_label=label, + ingest_pts=ingest_pts, + publish_pts=publish_pts, + ai_pts=ai_pts, + queue_pts=queue_pts, + error_pts=error_pts + ) + return chart_bytes, filename + diff --git a/tests/test_vision_and_disk_metrics.py b/tests/test_vision_and_disk_metrics.py new file mode 100644 index 0000000..badccf5 --- /dev/null +++ b/tests/test_vision_and_disk_metrics.py @@ -0,0 +1,122 @@ +import asyncio +import tempfile +import os +import shutil +from unittest.mock import AsyncMock, patch, MagicMock +from db.models import TargetChannel, AIProviderProfile +from core.llm import LLMClient +from services.ai_processor import AIProcessor +from core.metrics import ( + DISK_TOTAL_BYTES, + DISK_USED_BYTES, + DISK_FREE_BYTES, + DISK_FREE_PERCENT, + update_disk_metrics +) +from services.metrics_reporter import get_instant_metrics_report + + +def test_disk_metrics_gauge(): + update_disk_metrics() + sample = DISK_TOTAL_BYTES.collect()[0].samples + assert len(sample) > 0 + for s in sample: + assert s.value > 0 + + +async def test_metrics_report_includes_disk(): + fake_vector_free = [ + {"metric": {"mountpoint": "/"}, "value": [0, str(20.0 * 1024 ** 3)]}, + {"metric": {"mountpoint": "/projects"}, "value": [0, str(35.0 * 1024 ** 3)]} + ] + fake_vector_total = [ + {"metric": {"mountpoint": "/"}, "value": [0, str(200.0 * 1024 ** 3)]}, + {"metric": {"mountpoint": "/projects"}, "value": [0, str(40.0 * 1024 ** 3)]} + ] + fake_vector_pct = [ + {"metric": {"mountpoint": "/"}, "value": [0, "10.0"]}, + {"metric": {"mountpoint": "/projects"}, "value": [0, "87.5"]} + ] + + async def fake_query_vector(query): + if "copykar_disk_free_bytes" in query: + return fake_vector_free + if "copykar_disk_total_bytes" in query: + return fake_vector_total + if "copykar_disk_free_percent" in query: + return fake_vector_pct + return [] + + with patch("services.metrics_reporter._query_instant", return_value=0.0), \ + patch("services.metrics_reporter._query_vector", side_effect=fake_query_vector): + + report = await get_instant_metrics_report() + assert "فضای ذخیره‌سازی تفکیکی درایوها (Mount Points Storage)" in report + assert "/" in report + assert "/projects" in report + assert "20.00 GB" in report + assert "35.00 GB" in report + assert "10.0%" in report + assert "87.5%" in report + + +async def test_multimodal_vision_image_payload(): + with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp: + tmp.write(b"fake-image-binary-data") + tmp_path = tmp.name + + try: + profile_openai = AIProviderProfile( + id=1, + name="OpenAI Vision", + provider_type="openai", + model="gpt-4o", + is_active=True + ) + repo_mock = AsyncMock() + repo_mock.get_active_provider_profile.return_value = profile_openai + repo_mock.get_provider_profiles.return_value = [profile_openai] + repo_mock.get_setting.return_value = "false" + repo_mock.record_ai_log = AsyncMock() + + client = LLMClient(repo=repo_mock) + + # 1. Test OpenAI vision call formatting + captured_messages = [] + async def fake_post(url, headers=None, json=None): + captured_messages.extend(json.get("messages", [])) + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.json.return_value = { + "choices": [{"message": {"content": '{"decision": "accept", "rewritten_text": "Image saw a cat"}'}}] + } + return mock_resp + + with patch("httpx.AsyncClient.post", side_effect=fake_post): + target = TargetChannel(id=1, channel_id=-100123456, title="Vision Channel", username="vision_ch", language="fa", personality="طنز") + processor = AIProcessor(repo=repo_mock, llm=client) + res = await processor.rewrite_for_target("عکس را ببین", target, has_media=True, image_path=tmp_path) + assert res.is_rejected is False + assert "Image saw a cat" in str(res) + + user_msg = [m for m in captured_messages if m.get("role") == "user"][0] + assert isinstance(user_msg["content"], list) + types = [part["type"] for part in user_msg["content"]] + assert "text" in types + assert "image_url" in types + img_url = [part["image_url"]["url"] for part in user_msg["content"] if part["type"] == "image_url"][0] + assert img_url.startswith("data:image/jpeg;base64,") + + finally: + if os.path.exists(tmp_path): + os.remove(tmp_path) + + +async def main(): + test_disk_metrics_gauge() + await test_metrics_report_includes_disk() + await test_multimodal_vision_image_payload() + print("All disk metrics and multimodal vision image passing tests passed successfully!") + +if __name__ == "__main__": + asyncio.run(main())