alerting: Alertmanager + Signal relay deliver firing rules to Frank's phone
- dankclaw/observability/quadlet/alertmanager.container: loopback :9093, config from shared/prometheus/alertmanager.yml, family conventions - shared/prometheus/alertmanager.yml: single signal receiver, group_by alertname, 30s/5m/4h intervals, webhook to host.containers.internal:18093 - scripts/signal-relay.py + .service: host-side user unit, loopback :18093, 15-min dedupe, sends via signal-cli account 318311 - prometheus.yml: alerting block to alertmanager:9093; stale comments fixed - llm-fleet.yml: delivery header; LlmSpecDecodeOff annotation now drafter-agnostic - deploy.sh/deploy-remote.sh/deploy-plan.py: alertmanager service, config copy, relay install, planner routing for config + relay files - homepage: alertmanager status card (loopback probe) - tests: alertmanager in UNITS/loopback/health/PAIRS/planner + five new delivery-path seams (webhook<->relay contract, prometheus target, relay unit, signal-cli account, planner routing) - docs: Alertmanager.md component page, Observability.md, Prometheus.md, index.md, AGENTS.md delivery line
This commit is contained in:
@@ -21,7 +21,7 @@ load helpers
|
||||
|
||||
UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
UNITS=(prometheus grafana loki alloy node-exporter)
|
||||
UNITS=(prometheus grafana loki alloy node-exporter alertmanager)
|
||||
|
||||
# container_section <unit> — the [Container] key/value block only
|
||||
container_section() {
|
||||
@@ -33,7 +33,7 @@ service_section() {
|
||||
awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
|
||||
}
|
||||
|
||||
@test "all five observability units exist under dankclaw/observability/quadlet/" {
|
||||
@test "all six observability units exist under dankclaw/observability/quadlet/" {
|
||||
local svc
|
||||
for svc in "${UNITS[@]}"; do
|
||||
[ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
|
||||
@@ -72,10 +72,11 @@ service_section() {
|
||||
! grep -qE '^PublishPort=' "$UNIT_DIR/alloy.container" || true
|
||||
}
|
||||
|
||||
@test "the three HTTP UIs publish on loopback only" {
|
||||
@test "the four HTTP UIs publish on loopback only" {
|
||||
grep -q '^PublishPort=127.0.0.1:9090:9090$' "$UNIT_DIR/prometheus.container"
|
||||
grep -q '^PublishPort=127.0.0.1:3002:3000$' "$UNIT_DIR/grafana.container"
|
||||
grep -q '^PublishPort=127.0.0.1:3100:3100$' "$UNIT_DIR/loki.container"
|
||||
grep -q '^PublishPort=127.0.0.1:9093:9093$' "$UNIT_DIR/alertmanager.container"
|
||||
}
|
||||
|
||||
@test "the homepage status timer's probe targets are the published loopback ports" {
|
||||
@@ -88,6 +89,7 @@ service_section() {
|
||||
grep -q '"probe": "127.0.0.1:3100"' "$registry"
|
||||
grep -q '"probe": "127.0.0.1:9100"' "$registry"
|
||||
grep -q '"probe": "127.0.0.1:12345"' "$registry"
|
||||
grep -q '"probe": "127.0.0.1:9093"' "$registry"
|
||||
}
|
||||
|
||||
@test "node-exporter binds the host namespace and both of its consumers" {
|
||||
@@ -142,7 +144,7 @@ service_section() {
|
||||
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
||||
# /bin/sh. Its health command ships in the image instead.
|
||||
local svc
|
||||
for svc in prometheus grafana alloy node-exporter; do
|
||||
for svc in prometheus grafana alloy node-exporter alertmanager; do
|
||||
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc has no HealthCmd"; return 1; }
|
||||
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
||||
@@ -248,6 +250,7 @@ grafana http://127.0.0.1:3002/api/health
|
||||
loki http://127.0.0.1:3100/ready
|
||||
alloy http://127.0.0.1:12345/-/healthy
|
||||
node-exporter http://127.0.0.1:9100/metrics
|
||||
alertmanager http://127.0.0.1:9093/-/healthy
|
||||
PAIRS
|
||||
}
|
||||
|
||||
@@ -416,3 +419,67 @@ assert any('ALERTS' in e for e in exprs), exprs
|
||||
grep -q 'local unit_dir="$TARGET/observability/quadlet"' "$handler"
|
||||
grep -q 'install_quadlet observability "$unit_dir" "$service"' "$handler"
|
||||
}
|
||||
|
||||
@test "alertmanager delivers to the Signal relay through the podman host gateway" {
|
||||
# The relay listens on the HOST's loopback. From the container that
|
||||
# address is only reachable as host.containers.internal (netavark DNATs
|
||||
# published ports only — the LAN IP and the gateway are refused), so the
|
||||
# webhook URL and the relay's listen port are a cross-file contract: if
|
||||
# either half changes alone, every alert is dropped with no error
|
||||
# anywhere.
|
||||
local am="$REPO_ROOT/shared/prometheus/alertmanager.yml"
|
||||
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
||||
[ -f "$am" ] && [ -f "$relay" ]
|
||||
grep -q 'host.containers.internal:18093/alertmanager' "$am"
|
||||
# The relay must listen exactly where the webhook points.
|
||||
grep -q '18093' "$relay"
|
||||
grep -q '"/alertmanager"' "$relay"
|
||||
}
|
||||
|
||||
@test "prometheus ships alerts to alertmanager on the shared network" {
|
||||
# Container-name address, like grafana and loki: the published port is
|
||||
# loopback-only and would be wrong from inside Prometheus.
|
||||
local cfg="$REPO_ROOT/shared/prometheus/prometheus.yml"
|
||||
grep -q 'alertmanagers:' "$cfg"
|
||||
grep -q '"alertmanager:9093"' "$cfg"
|
||||
}
|
||||
|
||||
@test "the signal relay is a boot-persistent user unit the deployer installs" {
|
||||
# Plain user unit, not a quadlet: it runs python3 from the checkout and
|
||||
# can be enabled (generated quadlet units cannot on this host). Linger is
|
||||
# on for dank, so [Install] + enable is the whole boot story.
|
||||
local unit="$REPO_ROOT/scripts/signal-relay.service"
|
||||
local handler="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
[ -f "$unit" ]
|
||||
grep -q '^WantedBy=default.target$' "$unit"
|
||||
grep -q '^Restart=always$' "$unit"
|
||||
grep -q '^ExecStart=/usr/bin/python3 /home/dank/serverAppConfigs/scripts/signal-relay.py$' "$unit"
|
||||
grep -q '^install_signal_relay() {' "$handler"
|
||||
grep -q 'systemctl --user enable signal-relay.service' "$handler"
|
||||
grep -q 'systemctl --user restart signal-relay.service' "$handler"
|
||||
# The alertmanager deploy is what reaches the relay on the host.
|
||||
grep -q 'install_signal_relay' <(sed -n '/^deploy_observability()/,/^}/p' "$handler")
|
||||
}
|
||||
|
||||
@test "the relay sends through the existing signal-cli account, loopback only" {
|
||||
# The credential is the account state under ~/.local/share/signal-cli;
|
||||
# the number and account name are numbers, not secrets. The listener must
|
||||
# stay on loopback — an unauthenticated message-sending endpoint on a
|
||||
# host interface is the failure this whole unit family exists to avoid.
|
||||
local relay="$REPO_ROOT/scripts/signal-relay.py"
|
||||
grep -q 'signal-cli' "$relay"
|
||||
grep -q 'SIGNAL_CLI = "/usr/local/bin/signal-cli"' "$relay"
|
||||
grep -q 'SIGNAL_ACCOUNT = "318311"' "$relay"
|
||||
grep -q 'LISTEN_ADDR = ("127.0.0.1", 18093)' "$relay"
|
||||
}
|
||||
|
||||
@test "the planner selects alertmanager for its config and relay files" {
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
||||
--paths shared/prometheus/alertmanager.yml
|
||||
[ "$status" -eq 0 ]
|
||||
echo "$output" | grep -qx 'alertmanager'
|
||||
run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
|
||||
--paths scripts/signal-relay.py
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$output" = "alertmanager" ]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user