serverAppConfigs/shared/prometheus/prometheus.yml

# Prometheus configuration for the homelab.
# Deployed to /opt/prometheus/config/prometheus.yml on Dankclaw.

global:
  scrape_interval: 15s
  scrape_timeout: 10s
  evaluation_interval: 15s

# Alerting and recording rules for the LLM fleet. They evaluate, expose
# state through /alerts and the ALERTS metric, and — since 2026-09-17 —
# deliver to Signal through Alertmanager (see the alerting section below).
rule_files:
  - /etc/prometheus/rules/*.yml

# The delivery half. Alertmanager runs on the same dankclaw_default network,
# so it is addressed by container name, like grafana and loki below. Its
# single receiver webhooks the host-side signal-relay (192.168.0.143:18093,
# reached from the container as host.containers.internal).
alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  # Prometheus itself
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

  # Host metrics — both hosts. node-dankclaw is scraped over
  # host.containers.internal, not its own LAN address: this Prometheus is a
  # bridge container, and on this netavark host a container can only reach the
  # host's LAN IP at a PUBLISHED port — node-exporter's net=host bind on
  # 192.168.0.143:9100 is refused by construction (the same rule that sends
  # frank-dragoon and frank-zealot to the gateway below). node-dankserver is a
  # different machine, so its LAN address works.
  - job_name: "node-dankclaw"
    static_configs:
      - targets: ["host.containers.internal:9100"]

  - job_name: "node-dankserver"
    static_configs:
      - targets: ["192.168.0.235:9100"]

  # Gitea metrics (quadlet on an isolated network; use the published LAN port)
  - job_name: "gitea"
    static_configs:
      - targets: ["192.168.0.143:3000"]
    metrics_path: "/metrics"

  # Grafana metrics
  - job_name: "grafana"
    static_configs:
      - targets: ["grafana:3000"]
    metrics_path: "/metrics"

  # Loki metrics
  - job_name: "loki"
    static_configs:
      - targets: ["loki:3100"]
    metrics_path: "/metrics"

  # ComfyUI (no native /metrics; probe the UI)
  - job_name: "comfyui"
    metrics_path: "/"
    static_configs:
      - targets: ["comfyui:8188"]

  # Blackbox-style probes for service up/down (published LAN ports;
  # gitea and comfyui are covered by their own jobs above)
  - job_name: "services"
    metrics_path: "/"
    static_configs:
      - targets:
          - "192.168.0.143:8081"   # SearXNG (via the Gluetun tunnel)
          - "192.168.0.143:3005"   # OpenWebUI

  # LLM fleet. Scraped on the LAN addresses where the endpoint is reachable from
  # this container, not the guarded vhosts: these are machine clients and
  # Prometheus cannot solve an Anubis challenge.
  #
  # One metric namespace: llamacpp:* — frank-dragoon, frank-zealot and
  # frank-archon, all llama.cpp with /metrics (the family verified from
  # /metrics on 2026-09-16 and 2026-09-20) and all exporting
  # requests_deferred (the queue depth, the only form llama.cpp shows the
  # capacity wall in), spec_decode_num_drafts_total, tokens_predicted_total,
  # and the prompt_tokens_cached_total / prompt_tokens_total reuse pair.
  #
  # frank-dragoon and frank-zealot are scraped over host.containers.internal,
  # not their own LAN addresses: this Prometheus is a bridge-networked
  # container (Network=dankclaw_default), and on this netavark host a
  # container can only reach the host's LAN IP at a PUBLISHED port — a
  # host-process bind like llama-server's 0.0.0.0:7770 is refused at
  # 192.168.0.143:7770 while it answers at 169.254.1.2:7770. That is
  # why frank-zealot was red here for days (up{job="frank-zealot"} == 0,
  # "dial tcp 192.168.0.143:7771") even though the endpoint was healthy and
  # exporting: the target address was unreachable by construction, not the
  # service down. Same host, same rule, so both local endpoints use the
  # gateway.
  #
  # (2026-10-05) frank-archon LEFT the llamacpp family with FreeToken
  # (the PR587 ROCm engine then serving it had NO /metrics — the job
  # scraped 404s forever, a structural never-green) and RE-JOINED the same
  # day on the radiance engine: its exporter speaks vLLM-compatible names
  # plus radiance:* families, a different namespace — hence the separate
  # job below, not a re-add to the llamacpp pair. The netavark quirk above
  # is same-host only; plain LAN routing REACHES 192.168.0.235:7770.
  - job_name: "frank-zealot"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["host.containers.internal:7771"]
        labels:
          service: frank-zealot

  - job_name: "frank-dragoon"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["host.containers.internal:7770"]
        labels:
          service: frank-dragoon

  # frank-archon (dankserver) — the radiance engine since 2026-10-05.
  # A DIFFERENT host, so plain LAN routing is correct here (the
  # host.containers.internal trick is same-host only): health answers at
  # 192.168.0.235:7770 from this bridge container. Radiance exports
  # vLLM-compatible names plus radiance:* families — NOT the llamacpp:*
  # namespace, so llm-fleet rules must not be pointed at this target.
  - job_name: "frank-archon"
    metrics_path: "/metrics"
    static_configs:
      - targets: ["192.168.0.235:7770"]
        labels:
          service: frank-archon

  # Per-container metrics from cadvisor (a rootless quadlet on each host).
  # cadvisor keys every series by the cgroup path; for a quadlet container that
  # path is …/app.slice/<name>.service[/libpod-payload-<hex>], so the unit NAME
  # lives in it. The relabel below pulls that name out into a `container` label
  # and keeps only the per-unit aggregate — cadvisor's raw host root,
  # system.slice daemons and machine.slice scopes carry no such name and are
  # dropped, as is the per-CPU-core split (only cpu="total" survives), so the
  # scrape stays small. cadvisor cannot bind per-interface, so it is reached at
  # the host's PUBLISHED LAN port: unlike node-exporter's host-net bind (which
  # is why node-dankclaw scrapes via host.containers.internal), a published port
  # IS reachable from this bridge container at the host LAN address.
  - job_name: "cadvisor-dankclaw"
    static_configs:
      - targets: ["192.168.0.143:8080"]
        labels:
          host: dankclaw
    metric_relabel_configs:
      - source_labels: [id]
        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
        target_label: container
        replacement: '$1'
      # Keep only the unit-level cgroup; drop its payload/control children so a
      # container is not counted twice.
      - source_labels: [id]
        regex: '.*/\.service/.*'
        action: drop
      # Drop everything that is not a named unit (no `container` was extracted).
      - source_labels: [container]
        regex: '.+'
        action: keep
      # cpu="total" for cpu series; metrics with no cpu label (memory, network,
      # filesystem) are kept.
      - source_labels: [cpu]
        regex: '^$|total'
        action: keep
      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
        action: labeldrop

  - job_name: "cadvisor-dankserver"
    static_configs:
      - targets: ["192.168.0.235:8080"]
        labels:
          host: dankserver
    metric_relabel_configs:
      - source_labels: [id]
        regex: '.*/app\.slice/([^.]+)\.service(?:/.*)?$'
        target_label: container
        replacement: '$1'
      - source_labels: [id]
        regex: '.*/\.service/.*'
        action: drop
      - source_labels: [container]
        regex: '.+'
        action: keep
      - source_labels: [cpu]
        regex: '^$|total'
        action: keep
      - regex: 'id|name|image|container_label.*|machine_id|boot_id'
        action: labeldrop