alerting: Alertmanager + Signal relay deliver firing rules to Frank's phone
- dankclaw/observability/quadlet/alertmanager.container: loopback :9093, config from shared/prometheus/alertmanager.yml, family conventions - shared/prometheus/alertmanager.yml: single signal receiver, group_by alertname, 30s/5m/4h intervals, webhook to host.containers.internal:18093 - scripts/signal-relay.py + .service: host-side user unit, loopback :18093, 15-min dedupe, sends via signal-cli account 318311 - prometheus.yml: alerting block to alertmanager:9093; stale comments fixed - llm-fleet.yml: delivery header; LlmSpecDecodeOff annotation now drafter-agnostic - deploy.sh/deploy-remote.sh/deploy-plan.py: alertmanager service, config copy, relay install, planner routing for config + relay files - homepage: alertmanager status card (loopback probe) - tests: alertmanager in UNITS/loopback/health/PAIRS/planner + five new delivery-path seams (webhook<->relay contract, prometheus target, relay unit, signal-cli account, planner routing) - docs: Alertmanager.md component page, Observability.md, Prometheus.md, index.md, AGENTS.md delivery line
This commit is contained in:
@@ -86,7 +86,7 @@ service_is_allowed() {
|
||||
dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
|
||||
dankclaw:comfyui|dankclaw:hermes|\
|
||||
dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
|
||||
dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|dankclaw:alertmanager|\
|
||||
dankclaw:authentik-postgres|dankclaw:authentik-server|\
|
||||
dankclaw:authentik-worker|dankclaw:authentik-outpost|dankclaw:authentik-outpost-ldap|dankclaw:anubis|\
|
||||
dankclaw:caddy|dankclaw:homepage|dankclaw:seerr|dankclaw:marreta|\
|
||||
@@ -210,6 +210,31 @@ copy_prometheus_config() {
|
||||
/opt/prometheus/config/rules/llm-fleet.yml
|
||||
}
|
||||
|
||||
# Alertmanager config ships beside the Prometheus config: the two halves of
|
||||
# the alerting path change together, and a receiver with no route (or a
|
||||
# route with no receiver) fails silently.
|
||||
copy_alertmanager_config() {
|
||||
run_privileged mkdir -p /opt/alertmanager/config /opt/alertmanager/data
|
||||
run_privileged cp shared/prometheus/alertmanager.yml /opt/alertmanager/config/alertmanager.yml
|
||||
}
|
||||
|
||||
# The Signal relay is Alertmanager's only webhook consumer and the only
|
||||
# thing on this host that talks to signal-cli. It is a plain systemd USER
|
||||
# unit (not a quadlet): it runs python3 from the checkout, so a deploy
|
||||
# updates it without a copy step, and — unlike generated quadlet units,
|
||||
# which this host's podman 5.x v1 generator rejects — a hand-written user
|
||||
# unit can be enabled, and Linger=yes for dank starts it at boot.
|
||||
install_signal_relay() {
|
||||
local unit_dir="$HOME/.config/systemd/user"
|
||||
mkdir -p "$unit_dir"
|
||||
cp scripts/signal-relay.service "$unit_dir/signal-relay.service"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable signal-relay.service
|
||||
systemctl --user restart signal-relay.service
|
||||
curl -fsS -m 5 http://127.0.0.1:18093/health >/dev/null \
|
||||
|| { echo "signal-relay is not answering on 127.0.0.1:18093" >&2; return 1; }
|
||||
}
|
||||
|
||||
copy_loki_config() {
|
||||
run_privileged mkdir -p /opt/loki/config
|
||||
run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
|
||||
@@ -426,13 +451,17 @@ seed_grafana_env() {
|
||||
deploy_observability() {
|
||||
local service="$1"
|
||||
# Observability units live under <target>/observability/quadlet: Dankclaw
|
||||
# has five, Dankserver one (node-exporter, since 2026-09-16). The config
|
||||
# has six, Dankserver one (node-exporter, since 2026-09-16). The config
|
||||
# copies below are Dankclaw-owned state and run only there.
|
||||
local unit_dir="$TARGET/observability/quadlet"
|
||||
case "$service" in
|
||||
prometheus)
|
||||
copy_prometheus_config
|
||||
;;
|
||||
alertmanager)
|
||||
copy_alertmanager_config
|
||||
install_signal_relay
|
||||
;;
|
||||
loki)
|
||||
copy_loki_config
|
||||
build_compose_image loki
|
||||
@@ -456,6 +485,7 @@ deploy_observability() {
|
||||
systemctl --user restart "$service.service"
|
||||
case "$service" in
|
||||
prometheus) wait_for_health prometheus http://127.0.0.1:9090/-/healthy 60 ;;
|
||||
alertmanager) wait_for_health alertmanager http://127.0.0.1:9093/-/healthy 60 ;;
|
||||
grafana) wait_for_health grafana http://127.0.0.1:3002/api/health 90 ;;
|
||||
loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;;
|
||||
alloy) wait_for_health alloy http://127.0.0.1:12345/-/healthy 60 ;;
|
||||
@@ -1378,7 +1408,7 @@ deploy_service() {
|
||||
copyparty) deploy_copyparty ;;
|
||||
plex) deploy_plex ;;
|
||||
tautulli) deploy_tautulli ;;
|
||||
prometheus|grafana|loki|alloy|node-exporter) deploy_observability "$service" ;;
|
||||
prometheus|grafana|loki|alloy|node-exporter|alertmanager) deploy_observability "$service" ;;
|
||||
comfyui) deploy_comfyui ;;
|
||||
*) deploy_compose_service "$service" ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user