alerting: Alertmanager + Signal relay deliver firing rules to Frank's phone

- dankclaw/observability/quadlet/alertmanager.container: loopback :9093,
  config from shared/prometheus/alertmanager.yml, family conventions
- shared/prometheus/alertmanager.yml: single signal receiver, group_by
  alertname, 30s/5m/4h intervals, webhook to host.containers.internal:18093
- scripts/signal-relay.py + .service: host-side user unit, loopback :18093,
  15-min dedupe, sends via signal-cli account 318311
- prometheus.yml: alerting block to alertmanager:9093; stale comments fixed
- llm-fleet.yml: delivery header; LlmSpecDecodeOff annotation now drafter-agnostic
- deploy.sh/deploy-remote.sh/deploy-plan.py: alertmanager service, config
  copy, relay install, planner routing for config + relay files
- homepage: alertmanager status card (loopback probe)
- tests: alertmanager in UNITS/loopback/health/PAIRS/planner + five new
  delivery-path seams (webhook<->relay contract, prometheus target, relay
  unit, signal-cli account, planner routing)
- docs: Alertmanager.md component page, Observability.md, Prometheus.md,
  index.md, AGENTS.md delivery line
This commit is contained in:
Frank
2026-09-17 13:09:59 -07:00
parent 17450dd172
commit 590853c133
17 changed files with 513 additions and 51 deletions
+33 -3
View File
@@ -86,7 +86,7 @@ service_is_allowed() {
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
dankclaw:comfyui|dankclaw:hermes|\
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
dankclaw:authentik-postgres|dankclaw:authentik-server|\
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
@@ -210,6 +210,31 @@ copy_prometheus_config() {
/opt/prometheus/config/rules/llm-fleet.yml
}
# Alertmanager config ships beside the Prometheus config: the two halves of
# the alerting path change together, and a receiver with no route (or a
# route with no receiver) fails silently.
copy_alertmanager_config() {
run_privileged mkdir -p /opt/alertmanager/config /opt/alertmanager/data
run_privileged cp shared/prometheus/alertmanager.yml /opt/alertmanager/config/alertmanager.yml
}
# The Signal relay is Alertmanager's only webhook consumer and the only
# thing on this host that talks to signal-cli. It is a plain systemd USER
# unit (not a quadlet): it runs python3 from the checkout, so a deploy
# updates it without a copy step, and — unlike generated quadlet units,
# which this host's podman 5.x v1 generator rejects — a hand-written user
# unit can be enabled, and Linger=yes for dank starts it at boot.
install_signal_relay() {
local unit_dir="$HOME/.config/systemd/user"
mkdir -p "$unit_dir"
cp scripts/signal-relay.service "$unit_dir/signal-relay.service"
systemctl --user daemon-reload
systemctl --user enable signal-relay.service
systemctl --user restart signal-relay.service
curl -fsS -m 5 http://127.0.0.1:18093/health >/dev/null \
|| { echo "signal-relay is not answering on 127.0.0.1:18093" >&2; return 1; }
}
copy_loki_config() {
run_privileged mkdir -p /opt/loki/config
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
@@ -426,13 +451,17 @@ seed_grafana_env() {
deploy_observability() {
local service="$1"
# Observability units live under <target>/observability/quadlet: Dankclaw
# has five, Dankserver one (node-exporter, since 2026-09-16). The config
# has six, Dankserver one (node-exporter, since 2026-09-16). The config
# copies below are Dankclaw-owned state and run only there.
local unit_dir="$TARGET/observability/quadlet"
case "$service" in
prometheus)
copy_prometheus_config
;;
alertmanager)
copy_alertmanager_config
install_signal_relay
;;
loki)
copy_loki_config
build_compose_image loki
@@ -456,6 +485,7 @@ deploy_observability() {
systemctl --user restart "$service.service"
case "$service" in
prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;;
alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
@@ -1378,7 +1408,7 @@ deploy_service() {
copyparty) deploy_copyparty ;;
plex) deploy_plex ;;
tautulli) deploy_tautulli ;;
prometheus|grafana|loki|alloy|node-exporter) deploy_observability "$service" ;;
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
comfyui) deploy_comfyui ;;
*) deploy_compose_service "$service" ;;
esac