alerting: Alertmanager + Signal relay deliver firing rules to Frank's phone

- dankclaw/observability/quadlet/alertmanager.container: loopback :9093,
  config from shared/prometheus/alertmanager.yml, family conventions
- shared/prometheus/alertmanager.yml: single signal receiver, group_by
  alertname, 30s/5m/4h intervals, webhook to host.containers.internal:18093
- scripts/signal-relay.py + .service: host-side user unit, loopback :18093,
  15-min dedupe, sends via signal-cli account 318311
- prometheus.yml: alerting block to alertmanager:9093; stale comments fixed
- llm-fleet.yml: delivery header; LlmSpecDecodeOff annotation now drafter-agnostic
- deploy.sh/deploy-remote.sh/deploy-plan.py: alertmanager service, config
  copy, relay install, planner routing for config + relay files
- homepage: alertmanager status card (loopback probe)
- tests: alertmanager in UNITS/loopback/health/PAIRS/planner + five new
  delivery-path seams (webhook<->relay contract, prometheus target, relay
  unit, signal-cli account, planner routing)
- docs: Alertmanager.md component page, Observability.md, Prometheus.md,
  index.md, AGENTS.md delivery line

shared/prometheus/alertmanager.yml

@@ -0,0 +1,28 @@
+# Alertmanager configuration for the homelab.
+# Deployed to /opt/alertmanager/config/alertmanager.yml on Dankclaw.
+#
+# The one delivery path is Signal. The webhook lands on the host-side
+# signal-relay (a systemd user unit listening on 127.0.0.1:18093), which
+# formats the group and hands it to signal-cli under Frank's existing
+# account. There is deliberately no second receiver: a duplicate channel is
+# how an alert starts arriving twice and getting muted everywhere.
+#
+# From this container the host's loopback is reachable only as
+# host.containers.internal (169.254.1.2): netavark DNATs published ports
+# only, so the host's LAN IP and the podman gateway are both refused for an
+# unpublished port — the same quirk that 502'd the hermes upstream in the
+# Caddyfile on 2026-09-16.
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: signal
+  group_by: ['alertname']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: signal
+    webhook_configs:
+      - url: http://host.containers.internal:18093/alertmanager

shared/prometheus/prometheus.yml

@@ -6,12 +6,21 @@ global:
   scrape_timeout: 10s
   evaluation_interval: 15s
 
-# Alerting and recording rules for the LLM fleet. They evaluate and expose state
-# through /alerts, the ALERTS metric and the Grafana dashboard; nothing is
-# notified, because this stack has no Alertmanager (see the rules file header).
+# Alerting and recording rules for the LLM fleet. They evaluate, expose
+# state through /alerts and the ALERTS metric, and — since 2026-09-17 —
+# deliver to Signal through Alertmanager (see the alerting section below).
 rule_files:
   - /etc/prometheus/rules/*.yml
 
+# The delivery half. Alertmanager runs on the same dankclaw_default network,
+# so it is addressed by container name, like grafana and loki below. Its
+# single receiver webhooks the host-side signal-relay (loopback :18093,
+# reachable from the container only as host.containers.internal).
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
 scrape_configs:
   # Prometheus itself
   - job_name: "prometheus"
@@ -62,12 +71,13 @@ scrape_configs:
 
   # LLM fleet. Scraped on the LAN addresses, not the guarded vhosts: these are
   # machine clients and Prometheus cannot solve an Anubis challenge.
-  # frank-medium is vLLM, and its series are the ones that explain behaviour
-  # under load — vllm:kv_cache_usage_perc (how full the per-card pool is),
-  # vllm:num_requests_waiting (queue depth), the request-latency histograms,
-  # and the prefix-cache pair vllm:prefix_cache_hits_total over
-  # vllm:prefix_cache_queries_total (their rates give the hit rate, which says
-  # whether pinned conversations are actually being reused).
+  # Both LLM endpoints are llama.cpp, so both export the same llamacpp:
+  # counter family: prompt/decode throughput, requests_deferred (queue depth
+  # — the only form llama.cpp exposes the capacity wall in), the
+  # spec_decode_num_drafts / spec_decode_num_accepted pair (proof the drafter
+  # is running), and the prefix-cache pair prompt_tokens_cached_total over
+  # prompt_tokens_total (their rates are the reuse rate, which says whether
+  # pinned conversations are actually being reused).
   # frank-large is deliberately NOT scraped: the halogen engine that serves it
   # publishes /health, /cache and /v1/* and no Prometheus text endpoint, so a
   # target here would be red by construction and would fire LlmEndpointDown

shared/prometheus/rules/llm-fleet.yml

@@ -14,12 +14,13 @@
 #     LlmSpecDecodeOff takes its place: the failure an operator actually
 #     cannot see from the outside on this deployment.
 #
-# IMPORTANT: nothing is notified. There is no Alertmanager in this stack, so a
-# firing alert appears on Prometheus' /alerts page, in the ALERTS metric (which
-# the LLM fleet dashboard graphs) and nowhere else. Turning these into something
-# that reaches a person means adding an Alertmanager or using Grafana's own
-# alerting with a contact point — a decision about how Frank wants to be paged,
-# not a missing line of configuration.
+# IMPORTANT: delivery. Since 2026-09-17 a firing alert reaches Frank on
+# Signal: Prometheus pushes to Alertmanager (dankclaw/observability/quadlet/
+# alertmanager.container), whose single receiver webhooks the host-side
+# signal-relay user unit, which hands the text to signal-cli under the
+# existing account on this host. Before that date a firing alert appeared on
+# Prometheus' /alerts page, in the ALERTS metric (which the LLM fleet
+# dashboard graphs) and nowhere else.
 groups:
   - name: llm-fleet
     interval: 30s
@@ -77,9 +78,11 @@ groups:
           summary: "{{ $labels.job }}: serving without speculative decoding"
           description: >-
             Tokens are being produced but no draft has been proposed for 15
-            minutes. On frank-medium that means the DFlash2 drafter is not
-            running (check the unit's log for common_speculative_impl_draft_dflash),
-            and the endpoint is decoding autoregressively.
+            minutes. On the llama.cpp endpoints that means the drafter is not
+            running (check the unit's log for the
+            "adding speculative implementation" line — draft-mtp on both
+            endpoints as of 2026-09-17), and the endpoint is decoding
+            autoregressively.
 
       # The endpoint, not a replica: one process serves each port now, so a
       # restart of the unit is a scrape gap, not a failover.