alerting: Alertmanager + Signal relay deliver firing rules to Frank's phone

- dankclaw/observability/quadlet/alertmanager.container: loopback :9093,
  config from shared/prometheus/alertmanager.yml, family conventions
- shared/prometheus/alertmanager.yml: single signal receiver, group_by
  alertname, 30s/5m/4h intervals, webhook to host.containers.internal:18093
- scripts/signal-relay.py + .service: host-side user unit, loopback :18093,
  15-min dedupe, sends via signal-cli account 318311
- prometheus.yml: alerting block to alertmanager:9093; stale comments fixed
- llm-fleet.yml: delivery header; LlmSpecDecodeOff annotation now drafter-agnostic
- deploy.sh/deploy-remote.sh/deploy-plan.py: alertmanager service, config
  copy, relay install, planner routing for config + relay files
- homepage: alertmanager status card (loopback probe)
- tests: alertmanager in UNITS/loopback/health/PAIRS/planner + five new
  delivery-path seams (webhook<->relay contract, prometheus target, relay
  unit, signal-cli account, planner routing)
- docs: Alertmanager.md component page, Observability.md, Prometheus.md,
  index.md, AGENTS.md delivery line

scripts/signal-relay.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Alertmanager-to-Signal relay for the homelab (Dankclaw).
+
+Listens on 127.0.0.1:18093, receives Alertmanager's webhook payload, formats
+each alert group into a short message and hands it to signal-cli under the
+existing account on this host. Runs as a systemd user unit
+(scripts/signal-relay.service); logs go to the journal.
+
+The phone number and account name are numbers, not credentials — the
+credential is the signal-cli account state under
+~/.local/share/signal-cli, which never leaves the host.
+
+Stdlib only: this box's user units run the system python3.
+"""
+
+import json
+import shutil
+import subprocess
+import sys
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+LISTEN_ADDR = ("127.0.0.1", 18093)
+SIGNAL_CLI = "/usr/local/bin/signal-cli"
+SIGNAL_ACCOUNT = "318311"
+SIGNAL_TO = "+12062476336"
+
+# The relay is a safety net, not the rate limiter: Alertmanager already
+# repeats on repeat_interval. This suppresses exact duplicates that arrive
+# inside the window (config reloads, double-fires) so a misconfiguration
+# cannot fill the phone.
+DEDUPE_SECONDS = 15 * 60
+_last_sent = {}
+
+# Cap on the instances listed per group; the count is always stated.
+MAX_INSTANCES = 3
+
+
+def log(msg):
+    print(f"signal-relay: {msg}", file=sys.stderr, flush=True)
+
+
+def format_group(alerts):
+    """One message per webhook delivery (Alertmanager groups by alertname)."""
+    if not alerts:
+        return None
+    first = alerts[0]
+    name = first.get("labels", {}).get("alertname", "unknown")
+    state = "firing" if first.get("status") == "firing" else "resolved"
+    severity = first.get("labels", {}).get("severity", "info").upper()
+    lines = [f"[{severity}] {name} ({len(alerts)} {state})"]
+    for a in alerts[:MAX_INSTANCES]:
+        labels = a.get("labels", {})
+        ident = labels.get("instance") or labels.get("job") or "?"
+        summary = (a.get("annotations") or {}).get("summary")
+        line = f"  {ident}"
+        if summary:
+            line += f": {summary}"
+        lines.append(line)
+    if len(alerts) > MAX_INSTANCES:
+        lines.append(f"  … and {len(alerts) - MAX_INSTANCES} more")
+    return "\n".join(lines)
+
+
+def dedupe_key(alerts):
+    first = alerts[0]
+    labels = tuple(sorted(first.get("labels", {}).items()))
+    return (first.get("status"),) + labels
+
+
+def send_signal(text):
+    if shutil.which(SIGNAL_CLI) is None:
+        log(f"signal-cli not found at {SIGNAL_CLI}")
+        return False
+    try:
+        proc = subprocess.run(
+            [SIGNAL_CLI, "-a", SIGNAL_ACCOUNT, "-o", "json", "send", SIGNAL_TO, text],
+            capture_output=True, text=True, timeout=30,
+        )
+    except subprocess.TimeoutExpired:
+        log("signal-cli timed out after 30s")
+        return False
+    if proc.returncode != 0:
+        log(f"signal-cli failed rc={proc.returncode}: {proc.stderr.strip()[:300]}")
+        return False
+    return True
+
+
+def handle_alertmanager(payload):
+    alerts = payload.get("alerts", [])
+    text = format_group(alerts)
+    if text is None:
+        log("empty alert group, nothing sent")
+        return
+    now = time.monotonic()
+    key = dedupe_key(alerts)
+    _last_sent = {k: v for k, v in _last_sent.items() if now - v < DEDUPE_SECONDS}
+    if key in _last_sent:
+        log(f"suppressed duplicate: {key[0]}")
+        return
+    if send_signal(text):
+        _last_sent[key] = now
+        log(f"sent to {SIGNAL_TO}: {text.splitlines()[0]}")
+    else:
+        # Do not mark a failed send: the next repeat should retry.
+        log("send failed; will retry on the next delivery")
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/health":
+            self.send_response(200)
+            self.end_headers()
+            self.wfile.write(b"ok")
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def do_POST(self):
+        if self.path != "/alertmanager":
+            self.send_response(404)
+            self.end_headers()
+            return
+        length = int(self.headers.get("Content-Length", 0))
+        try:
+            payload = json.loads(self.rfile.read(length))
+        except (ValueError, UnicodeDecodeError) as e:
+            log(f"bad payload: {e}")
+            self.send_response(400)
+            self.end_headers()
+            return
+        try:
+            handle_alertmanager(payload)
+        except Exception as e:  # a relay bug must not 500 Alertmanager
+            log(f"handler error: {e!r}")
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"ok")
+
+    def log_message(self, fmt, *args):
+        # Default access log is per-request noise; the journal keeps the
+        # send/suppress lines, which are the ones that matter.
+        pass
+
+
+def main():
+    server = ThreadingHTTPServer(LISTEN_ADDR, Handler)
+    log(f"listening on {'.'.join(map(str, LISTEN_ADDR))}, account {SIGNAL_ACCOUNT}")
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()