from prometheus_client import Counter, Histogram, Gauge, start_http_server import logging from typing import Optional logger = logging.getLogger(__name__) # Counters # Ingest is counted once, by SOURCE_ACTIVITY_TOTAL, which also carries the channel title. SOURCE_ACTIVITY_TOTAL = Counter( "copykar_source_activity_total", "Total posts ingested per source channel", ["channel_id", "title"] ) TARGET_ACTIVITY_TOTAL = Counter( "copykar_target_activity_total", "Total posts published per target channel", ["channel_id", "title"] ) AI_REQUESTS_TOTAL = Counter( "copykar_ai_requests_total", "Total AI API calls made", ["action", "status"] ) DUPLICATES_DETECTED_TOTAL = Counter( "copykar_duplicates_detected_total", "Total duplicate posts detected", ["method"] ) ADMIN_ACTIONS_TOTAL = Counter( "copykar_admin_actions_total", "Total review decisions by admins", ["action"] ) # Delivery is counted once, by TARGET_ACTIVITY_TOTAL. AUTO_ROUTED_POSTS_TOTAL = Counter( "copykar_auto_routed_posts_total", "Posts queued automatically by a source-to-target route", ["source_channel_id", "target_title"] ) ERRORS_TOTAL = Counter( "copykar_errors_total", "Total exceptions and errors caught across services", ["service", "error_type"] ) ERRORS_RESOLVED_TOTAL = Counter( "copykar_errors_resolved_total", "Errors an admin has marked as fixed", ["service", "error_type"] ) # Histograms AI_LATENCY_SECONDS = Histogram( "copykar_ai_latency_seconds", "Time taken for AI API operations", ["action"] ) # Gauges QUEUE_POSTS_GAUGE = Gauge( "copykar_posts_queue_gauge", "Number of posts currently in various queue states", ["status"] ) ERRORS_OPEN_GAUGE = Gauge( "copykar_errors_open", "Unresolved errors currently recorded, by service and exception type", ["service", "error_type"] ) ERRORS_OPEN_TOTAL_GAUGE = Gauge( "copykar_errors_open_total", "Total unresolved errors across all services" ) import shutil import asyncio # Disk Space Gauges per Mount Point DISK_TOTAL_BYTES = Gauge( "copykar_disk_total_bytes", "Total disk capacity in bytes", ["mountpoint", "device"] ) DISK_USED_BYTES = Gauge( "copykar_disk_used_bytes", "Used disk space in bytes", ["mountpoint", "device"] ) DISK_FREE_BYTES = Gauge( "copykar_disk_free_bytes", "Free/available disk space in bytes", ["mountpoint", "device"] ) DISK_FREE_PERCENT = Gauge( "copykar_disk_free_percent", "Percentage of free disk space", ["mountpoint", "device"] ) import os VOLUME_DEFINITIONS = [ { "mountpoint": "/", "device": "/dev/sda1", "container_paths": ["/host_os/home", "/hostfs/host_mnt/home", "/home", "/host_os_disk", "/hostfs", "/"] }, { "mountpoint": "/projects", "device": "/dev/sda2", "container_paths": ["/host_os/projects", "/projects"] }, { "mountpoint": "/boot", "device": "/dev/sda4", "container_paths": ["/host_os/boot", "/boot"] }, { "mountpoint": "/boot/efi", "device": "/dev/sda3", "container_paths": ["/host_os/boot_efi", "/boot/efi"] }, { "mountpoint": "/run/media/mamad/WIN11_25H2_", "device": "/dev/sdc1", "container_paths": ["/host_os/media/mamad/WIN11_25H2_", "/run/media/mamad/WIN11_25H2_"] } ] def update_disk_metrics(): """Update Prometheus gauges with real host OS filesystem disk usage for each mounted volume.""" try: recorded_mounts = set() for v in VOLUME_DEFINITIONS: mnt = v["mountpoint"] dev = v["device"] for path in v["container_paths"]: if os.path.exists(path) and os.path.isdir(path): try: total, used, free = shutil.disk_usage(path) DISK_TOTAL_BYTES.labels(mountpoint=mnt, device=dev).set(total) DISK_USED_BYTES.labels(mountpoint=mnt, device=dev).set(used) DISK_FREE_BYTES.labels(mountpoint=mnt, device=dev).set(free) free_pct = (free / total * 100.0) if total > 0 else 0.0 DISK_FREE_PERCENT.labels(mountpoint=mnt, device=dev).set(free_pct) recorded_mounts.add(mnt) break except Exception as err: logger.debug(f"Error measuring disk usage for {path}: {err}") # If none of the specific mounts matched, fallback to root if not recorded_mounts: total, used, free = shutil.disk_usage("/") DISK_TOTAL_BYTES.labels(mountpoint="/", device="default").set(total) DISK_USED_BYTES.labels(mountpoint="/", device="default").set(used) DISK_FREE_BYTES.labels(mountpoint="/", device="default").set(free) free_pct = (free / total * 100.0) if total > 0 else 0.0 DISK_FREE_PERCENT.labels(mountpoint="/", device="default").set(free_pct) except Exception as e: logger.debug(f"Error updating host disk metrics: {e}") async def _disk_metrics_loop(interval: int = 15): """Background task to keep disk metrics updated.""" while True: try: update_disk_metrics() except Exception as e: logger.debug(f"Disk metrics loop error: {e}") await asyncio.sleep(interval) # The overall queue depth is sum(copykar_posts_queue_gauge) - no separate total series, # which previously made the dashboard report the queue twice. def start_metrics_server(port: int = 8008): try: start_http_server(port) update_disk_metrics() try: loop = asyncio.get_event_loop() if loop.is_running(): asyncio.create_task(_disk_metrics_loop()) except Exception: pass logger.info(f"Prometheus metrics server running on port {port}") except Exception as e: logger.error(f"Failed to start Prometheus metrics server: {e}")