test(observability): copier sandboxes, probe-target seam exception, cop gates

- caddyfile seam learns the documented exception: URL-shaped targets
  are legal only inside a prober job (state is per-job; the first cut
  leaked it and passed a planted URL).
- observability seams: blackbox unit + planner entries, both copiers
  executed for real in sandboxes, alloy engine-validation refusal
  (image-gated), dashboard panel invariants.
- AGENTS.md: structural target downs retired.

AGENTS.md

@@ -176,11 +176,12 @@ alertmanager.container`, loopback :9093) → host-side Signal relay
 (`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN
 IP, the one bind a container can reach as `host.containers.internal`;
 iptables admits loopback, the host and the podman bridge subnet only) → signal-cli account 318311.
-Prometheus targets read down for structural reasons (HTML-scraped "probes",
-Gitea metrics off); the remedy is a blackbox exporter. `frank-zealot` was down for
-a different reason until 2026-10-18: llama.cpp will not serve `/metrics` at all
-without `--metrics`. See
-`serverAppConfigs-docs/Infrastructure/Observability.md`.
+The former structural target downs (HTML-scraped "probes", Gitea metrics off)
+are retired as of 2026-10-21: HTTP liveness rides a blackbox exporter
+(`shared/blackbox/blackbox.yml`), Gitea serves `/metrics` from its quadlet, and
+every Prometheus job reports up. `frank-zealot` was down for a different reason
+until 2026-10-18: llama.cpp will not serve `/metrics` at all without
+`--metrics`. See `serverAppConfigs-docs/Infrastructure/Observability.md`.
 
 ## Testing
 

tests/caddyfile.bats

@@ -193,7 +193,18 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
 }
 
 @test "prometheus targets are host:port (no URL schemes)" {
-    ! grep -nE '^\s*-\s*"?http' "$REPO_ROOT/shared/prometheus/prometheus.yml"
+    # Plain scrape targets are host:port; a URL target is only meaningful for
+    # the blackbox prober job, whose relabel_configs copy it to __param_target
+    # and point __address__ at the prober. Anywhere else a URL target is the
+    # old HTML-probe defect wearing a new hat.
+    awk '
+        /job_name:/ { job=$0; sub(/.*"/, "", job); sub(/".*/, "", job); probe = 0 }
+        /metrics_path: \/probe/ { probe = 1 }
+        /^[[:space:]]*-[[:space:]]*"http/ && !probe {
+            print FILENAME ":" FNR ": URL target outside a prober job: " $0; bad = 1
+        }
+        END { exit bad }
+    ' "$REPO_ROOT/shared/prometheus/prometheus.yml"
 }
 
 @test "observability containers run as container root (rootless = host user)" {

tests/observability_quadlet.bats

@@ -21,7 +21,7 @@ load helpers
 
 UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet"
 REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
-UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager)
+UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager blackbox-exporter)
 
 # container_section <unit> — the [Container] key/value block only
 container_section() {
@@ -33,7 +33,7 @@ service_section() {
     awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1"
 }
 
-@test "all seven observability units exist under dankclaw/observability/quadlet/" {
+@test "all eight observability units exist under dankclaw/observability/quadlet/" {
     local svc
     for svc in "${UNITS[@]}"; do
         [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; }
@@ -668,3 +668,89 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names),
     [ -f "$sandbox/opt/grafana/provisioning/dashboards/provider.yml" ]
     [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ]
 }
+
+@test "changes under shared/blackbox/ plan the prober" {
+    run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \
+        --paths shared/blackbox/blackbox.yml
+    [ "$status" -eq 0 ]
+    [ "$output" = "blackbox-exporter" ]
+}
+
+@test "the copier runs the same code path the deployer runs" {
+    # Textually pinned so a refactor cannot quietly route the deployer around
+    # the copy_* functions the seams execute.
+    grep -q '        prometheus)' "$REMOTE_DEPLOY"
+    grep -q 'copy_prometheus_config' "$REMOTE_DEPLOY"
+    grep -q 'copy_alertmanager_config' "$REMOTE_DEPLOY"
+    grep -q 'copy_loki_config' "$REMOTE_DEPLOY"
+    grep -q '        blackbox-exporter)' "$REMOTE_DEPLOY"
+    grep -q 'copy_blackbox_config' "$REMOTE_DEPLOY"
+}
+
+@test "the alloy copier installs the host-labeled config, token-free" {
+    # Executes copy_alloy_config for real against a sandbox (stubbed
+    # run_privileged). A surviving GRAFANA_HOST_LABEL token would ship a
+    # config whose host label is a literal placeholder — the exact class of
+    # silent-label defect this tranche retired the board for.
+    local sandbox="$BATS_TEST_TMPDIR/opt"
+    (
+        cd "$REPO_ROOT"
+        eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)"
+        run_privileged() { "$@"; }
+        # NOTE: VAR=x func does not bind for shell functions — export.
+        export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox"
+        copy_alloy_config
+    )
+    [ -f "$sandbox/config/config.alloy" ]
+    grep -q 'host = "dankclaw",' "$sandbox/config/config.alloy"
+    ! grep -q "GRAFANA_HOST_LABEL" "$sandbox/config/config.alloy"
+}
+
+@test "the blackbox copier installs the prober config" {
+    local sandbox="$BATS_TEST_TMPDIR/blackbox"
+    (
+        cd "$REPO_ROOT"
+        eval "$(sed -n '/^copy_blackbox_config()/,/^}/p' scripts/deploy-remote.sh)"
+        run_privileged() { "$@"; }
+        export BLACKBOX_CONF_ROOT="$sandbox"
+        copy_blackbox_config
+    )
+    [ -f "$sandbox/blackbox.yml" ]
+    grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml"
+}
+
+@test "the alloy copier refuses to install a config the engine rejects" {
+    # The syntax gate: alloy attributes are position-sensitive and a
+    # misplaced one only explodes at run time (crash-loop behind a green
+    # deploy). Prove the copier refuses a config the pinned image won't
+    # validate, and installs nothing. Needs the alloy image; skipped
+    # where it is absent (the copier's gate is likewise image-gated).
+    if ! podman image exists localhost/franksdank/alloy:latest; then
+        skip "franksdank/alloy image not present on this box"
+    fi
+    local sandbox="$BATS_TEST_TMPDIR/opt" cfg="$BATS_TEST_TMPDIR/broken.alloy"
+    sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg"
+    printf 'bogus_attr = "x"\n' >> "$cfg"
+    ! (
+        cd "$REPO_ROOT"
+        eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)"
+        run_privileged() { "$@"; }
+        cp() { if [ "$1" = "$cfg" ]; then return 1; fi; command cp "$@"; }
+        export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox"
+        copy_alloy_config
+    )
+    [ ! -f "$sandbox/config/config.alloy" ]
+}
+
+@test "the installed alloy config is engine-valid" {
+    # Runs the pinned image's own validator over the shipped config. The
+    # 2026-10-21 defect (`on_error` on the labels stage, which alloy 1.21
+    # does not define) passed every textual seam and only failed at boot.
+    if ! podman image exists localhost/franksdank/alloy:latest; then
+        skip "franksdank/alloy image not present on this box"
+    fi
+    local cfg="$BATS_TEST_TMPDIR/rendered.alloy"
+    sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg"
+    podman run --rm -v "$cfg:/c.alloy:ro" --entrypoint /usr/bin/alloy \
+        localhost/franksdank/alloy:latest validate /c.alloy
+}