# Prometheus configuration for the homelab.
# Deployed to /opt/prometheus/config/prometheus.yml on Dankclaw.

global:
  scrape_interval: 15s
  scrape_timeout: 10s
  evaluation_interval: 15s

# Alerting and recording rules for the LLM fleet. They evaluate and expose state
# through /alerts, the ALERTS metric and the Grafana dashboard; nothing is
# notified, because this stack has no Alertmanager (see the rules file header).
rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  # Prometheus itself
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

  # Host metrics — both hosts
  - job_name: "node-dankclaw"
    static_configs:
      - targets: ["192.168.0.143:9100"]

  - job_name: "node-dankserver"
    static_configs:
      - targets: ["192.168.0.235:9100"]

  # Gitea metrics (quadlet on an isolated network; use the published LAN port)
  - job_name: "gitea"
    static_configs:
      - targets: ["192.168.0.143:3000"]
    metrics_path: "/metrics"

  # Grafana metrics
  - job_name: "grafana"
    static_configs:
      - targets: ["grafana:3000"]
    metrics_path: "/metrics"

  # Loki metrics
  - job_name: "loki"
    static_configs:
      - targets: ["loki:3100"]
    metrics_path: "/metrics"

  # ComfyUI (no native /metrics; probe the UI)
  - job_name: "comfyui"
    metrics_path: "/"
    static_configs:
      - targets: ["comfyui:8188"]

  # Blackbox-style probes for service up/down (published LAN ports;
  # gitea and comfyui are covered by their own jobs above)
  - job_name: "services"
    metrics_path: "/"
    static_configs:
      - targets:
          - "192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
          - "192.168.0.143:3005"   # OpenWebUI

  # LLM fleet. Scraped on the LAN addresses, not the guarded vhosts: these are
  # machine clients and Prometheus cannot solve an Anubis challenge.
  # frank-medium is vLLM, and its series are the ones that explain behaviour
  # under load — vllm:kv_cache_usage_perc (how full the per-card pool is),
  # vllm:num_requests_waiting (queue depth), the request-latency histograms,
  # and the prefix-cache pair vllm:prefix_cache_hits_total over
  # vllm:prefix_cache_queries_total (their rates give the hit rate, which says
  # whether pinned conversations are actually being reused).
  # frank-large is deliberately NOT scraped: the halogen engine that serves it
  # publishes /health, /cache and /v1/* and no Prometheus text endpoint, so a
  # target here would be red by construction and would fire LlmEndpointDown
  # forever. Its liveness is on the homepage (a loopback probe) and in the
  # engine's own boot ledger. The remedy, when the fleet wants it, is the
  # blackbox exporter already wanted for the comfyui/services probes.
  #
  # frank-small is llama.cpp (its serve script passes --metrics) and does
  # export text-format counters under llamacpp: on the port its unit publishes.
  - job_name: "frank-medium"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["192.168.0.235:7770"]
        labels:
          service: frank-medium

  - job_name: "frank-small"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["192.168.0.143:7771"]
        labels:
          service: frank-small