Frank
8f747a2104
A socket bound to 127.0.0.1 refuses container traffic that arrives via the podman bridge: the first live webhook failed with 'dial tcp 169.254.1.2:18093: connection refused'. The relay now binds the host-gateway endpoint — link-local to the podman bridge, off the LAN — and the deployer's health probe follows it there.
103 lines
3.6 KiB
YAML
103 lines
3.6 KiB
YAML
# Prometheus configuration for the homelab.
|
|
# Deployed to /opt/prometheus/config/prometheus.yml on Dankclaw.
|
|
|
|
global:
|
|
scrape_interval: 15s
|
|
scrape_timeout: 10s
|
|
evaluation_interval: 15s
|
|
|
|
# Alerting and recording rules for the LLM fleet. They evaluate, expose
|
|
# state through /alerts and the ALERTS metric, and — since 2026-09-17 —
|
|
# deliver to Signal through Alertmanager (see the alerting section below).
|
|
rule_files:
|
|
- /etc/prometheus/rules/*.yml
|
|
|
|
# The delivery half. Alertmanager runs on the same dankclaw_default network,
|
|
# so it is addressed by container name, like grafana and loki below. Its
|
|
# single receiver webhooks the host-side signal-relay (169.254.1.2:18093,
|
|
# the host.containers.internal endpoint).
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs:
|
|
- targets: ["alertmanager:9093"]
|
|
|
|
scrape_configs:
|
|
# Prometheus itself
|
|
- job_name: "prometheus"
|
|
static_configs:
|
|
- targets: ["localhost:9090"]
|
|
|
|
# Host metrics — both hosts
|
|
- job_name: "node-dankclaw"
|
|
static_configs:
|
|
- targets: ["192.168.0.143:9100"]
|
|
|
|
- job_name: "node-dankserver"
|
|
static_configs:
|
|
- targets: ["192.168.0.235:9100"]
|
|
|
|
# Gitea metrics (quadlet on an isolated network; use the published LAN port)
|
|
- job_name: "gitea"
|
|
static_configs:
|
|
- targets: ["192.168.0.143:3000"]
|
|
metrics_path: "/metrics"
|
|
|
|
# Grafana metrics
|
|
- job_name: "grafana"
|
|
static_configs:
|
|
- targets: ["grafana:3000"]
|
|
metrics_path: "/metrics"
|
|
|
|
# Loki metrics
|
|
- job_name: "loki"
|
|
static_configs:
|
|
- targets: ["loki:3100"]
|
|
metrics_path: "/metrics"
|
|
|
|
# ComfyUI (no native /metrics; probe the UI)
|
|
- job_name: "comfyui"
|
|
metrics_path: "/"
|
|
static_configs:
|
|
- targets: ["comfyui:8188"]
|
|
|
|
# Blackbox-style probes for service up/down (published LAN ports;
|
|
# gitea and comfyui are covered by their own jobs above)
|
|
- job_name: "services"
|
|
metrics_path: "/"
|
|
static_configs:
|
|
- targets:
|
|
- "192.168.0.143:8081" # SearXNG (via the Gluetun tunnel)
|
|
- "192.168.0.143:3005" # OpenWebUI
|
|
|
|
# LLM fleet. Scraped on the LAN addresses, not the guarded vhosts: these are
|
|
# machine clients and Prometheus cannot solve an Anubis challenge.
|
|
# Both LLM endpoints are llama.cpp, so both export the same llamacpp:
|
|
# counter family: prompt/decode throughput, requests_deferred (queue depth
|
|
# — the only form llama.cpp exposes the capacity wall in), the
|
|
# spec_decode_num_drafts / spec_decode_num_accepted pair (proof the drafter
|
|
# is running), and the prefix-cache pair prompt_tokens_cached_total over
|
|
# prompt_tokens_total (their rates are the reuse rate, which says whether
|
|
# pinned conversations are actually being reused).
|
|
# frank-large is deliberately NOT scraped: the halogen engine that serves it
|
|
# publishes /health, /cache and /v1/* and no Prometheus text endpoint, so a
|
|
# target here would be red by construction and would fire LlmEndpointDown
|
|
# forever. Its liveness is on the homepage (a loopback probe) and in the
|
|
# engine's own boot ledger. The remedy, when the fleet wants it, is the
|
|
# blackbox exporter already wanted for the comfyui/services probes.
|
|
#
|
|
# frank-small is llama.cpp (its serve script passes --metrics) and does
|
|
# export text-format counters under llamacpp: on the port its unit publishes.
|
|
- job_name: "frank-medium"
|
|
metrics_path: "/metrics"
|
|
static_configs:
|
|
- targets: ["192.168.0.235:7770"]
|
|
labels:
|
|
service: frank-medium
|
|
|
|
- job_name: "frank-small"
|
|
metrics_path: "/metrics"
|
|
static_configs:
|
|
- targets: ["192.168.0.143:7771"]
|
|
labels:
|
|
service: frank-small
|