# Prometheus configuration for the homelab.
# Deployed to /opt/prometheus/config/prometheus.yml on Dankclaw.

global:
  scrape_interval: 15s
  scrape_timeout: 10s
  evaluation_interval: 15s

# Alerting and recording rules for the LLM fleet. They evaluate, expose
# state through /alerts and the ALERTS metric, and — since 2026-09-17 —
# deliver to Signal through Alertmanager (see the alerting section below).
rule_files:
  - /etc/prometheus/rules/*.yml

# The delivery half. Alertmanager runs on the same dankclaw_default network,
# so it is addressed by container name, like grafana and loki below. Its
# single receiver webhooks the host-side signal-relay (192.168.0.143:18093,
# reached from the container as host.containers.internal).
alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  # Prometheus itself
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

  # Host metrics — both hosts. node-dankclaw is scraped over
  # host.containers.internal, not its own LAN address: this Prometheus is a
  # bridge container, and on this netavark host a container can only reach the
  # host's LAN IP at a PUBLISHED port — node-exporter's net=host bind on
  # 192.168.0.143:9100 is refused by construction (the same rule that sends
  # frank-large and frank-small to the gateway below). node-dankserver is a
  # different machine, so its LAN address works.
  - job_name: "node-dankclaw"
    static_configs:
      - targets: ["host.containers.internal:9100"]

  - job_name: "node-dankserver"
    static_configs:
      - targets: ["192.168.0.235:9100"]

  # Gitea metrics (quadlet on an isolated network; use the published LAN port)
  - job_name: "gitea"
    static_configs:
      - targets: ["192.168.0.143:3000"]
    metrics_path: "/metrics"

  # Grafana metrics
  - job_name: "grafana"
    static_configs:
      - targets: ["grafana:3000"]
    metrics_path: "/metrics"

  # Loki metrics
  - job_name: "loki"
    static_configs:
      - targets: ["loki:3100"]
    metrics_path: "/metrics"

  # ComfyUI (no native /metrics; probe the UI)
  - job_name: "comfyui"
    metrics_path: "/"
    static_configs:
      - targets: ["comfyui:8188"]

  # Blackbox-style probes for service up/down (published LAN ports;
  # gitea and comfyui are covered by their own jobs above)
  - job_name: "services"
    metrics_path: "/"
    static_configs:
      - targets:
          - "192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
          - "192.168.0.143:3005"   # OpenWebUI

  # LLM fleet. Scraped on the LAN addresses where the endpoint is reachable from
  # this container, not the guarded vhosts: these are machine clients and
  # Prometheus cannot solve an Anubis challenge.
  #
  # Two metric namespaces, because the fleet runs two runtimes:
  #
  #   * vllm:* — frank-medium (vllm-radiance TP=2 since 2026-09-20): prompt and
  #     generation token counters, num_requests_waiting plus its reason label,
  #     num_preemptions_total, the kv_cache_usage_perc fill gauge, the
  #     prefix_cache hits/queries pair, the spec_decode drafts/accepted triple,
  #     and real latency histograms (time_to_first_token, inter_token,
  #     e2e_request_latency).
  #   * llamacpp:* — frank-large and frank-small. Both take --metrics and both
  #     were verified 2026-09-20 to export requests_deferred (the queue depth,
  #     the only form llama.cpp shows the capacity wall in),
  #     spec_decode_num_drafts_total, tokens_predicted_total, and the
  #     prompt_tokens_cached_total / prompt_tokens_total reuse pair.
  #
  # frank-large is scraped over host.containers.internal, not its own LAN address:
  # this Prometheus is a bridge-networked container (Network=dankclaw_default),
  # and on this netavark host a container can only reach the host's LAN IP at a
  # PUBLISHED port — a host-process bind like llama-server's 0.0.0.0:7770 is
  # refused at 192.168.0.143:7770 while it answers at 169.254.1.2:7770. That is
  # why frank-small was red here for days (up{job="frank-small"} == 0,
  # "dial tcp 192.168.0.143:7771") even though the endpoint was healthy and
  # exporting: the target address was unreachable by construction, not the
  # service down. Same host, same rule, so both local endpoints use the gateway.
  # frank-medium stays a LAN address — it is a different machine.
  - job_name: "frank-medium"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["192.168.0.235:7770"]
        labels:
          service: frank-medium
  
  - job_name: "frank-small"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["host.containers.internal:7771"]
        labels:
          service: frank-small
  
  - job_name: "frank-large"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["host.containers.internal:7770"]
        labels:
          service: frank-large