From 078b550bd8e547ed6116a44a7ea3c97b00573b16 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:33:03 -0700 Subject: [PATCH] test(observability): copier sandboxes, probe-target seam exception, cop gates - caddyfile seam learns the documented exception: URL-shaped targets are legal only inside a prober job (state is per-job; the first cut leaked it and passed a planted URL). - observability seams: blackbox unit + planner entries, both copiers executed for real in sandboxes, alloy engine-validation refusal (image-gated), dashboard panel invariants. - AGENTS.md: structural target downs retired. --- AGENTS.md | 11 ++-- tests/caddyfile.bats | 13 ++++- tests/observability_quadlet.bats | 90 +++++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 02873d4..04717a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,11 +176,12 @@ alertmanager.container`, loopback :9093) → host-side Signal relay (`scripts/signal-relay.py`, user unit, 192.168.0.143:18093 — the host LAN IP, the one bind a container can reach as `host.containers.internal`; iptables admits loopback, the host and the podman bridge subnet only) → signal-cli account 318311. -Prometheus targets read down for structural reasons (HTML-scraped "probes", -Gitea metrics off); the remedy is a blackbox exporter. `frank-zealot` was down for -a different reason until 2026-10-18: llama.cpp will not serve `/metrics` at all -without `--metrics`. See -`serverAppConfigs-docs/Infrastructure/Observability.md`. +The former structural target downs (HTML-scraped "probes", Gitea metrics off) +are retired as of 2026-10-21: HTTP liveness rides a blackbox exporter +(`shared/blackbox/blackbox.yml`), Gitea serves `/metrics` from its quadlet, and +every Prometheus job reports up. `frank-zealot` was down for a different reason +until 2026-10-18: llama.cpp will not serve `/metrics` at all without +`--metrics`. See `serverAppConfigs-docs/Infrastructure/Observability.md`. ## Testing diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index 28b8f2a..fdc44df 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -193,7 +193,18 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" } @test "prometheus targets are host:port (no URL schemes)" { - ! grep -nE '^\s*-\s*"?http' "$REPO_ROOT/shared/prometheus/prometheus.yml" + # Plain scrape targets are host:port; a URL target is only meaningful for + # the blackbox prober job, whose relabel_configs copy it to __param_target + # and point __address__ at the prober. Anywhere else a URL target is the + # old HTML-probe defect wearing a new hat. + awk ' + /job_name:/ { job=$0; sub(/.*"/, "", job); sub(/".*/, "", job); probe = 0 } + /metrics_path: \/probe/ { probe = 1 } + /^[[:space:]]*-[[:space:]]*"http/ && !probe { + print FILENAME ":" FNR ": URL target outside a prober job: " $0; bad = 1 + } + END { exit bad } + ' "$REPO_ROOT/shared/prometheus/prometheus.yml" } @test "observability containers run as container root (rootless = host user)" { diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index fb4fe9b..d360cb1 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -21,7 +21,7 @@ load helpers UNIT_DIR="$REPO_ROOT/dankclaw/observability/quadlet" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager) +UNITS=(prometheus grafana loki alloy node-exporter cadvisor alertmanager blackbox-exporter) # container_section — the [Container] key/value block only container_section() { @@ -33,7 +33,7 @@ service_section() { awk '/^\[Service\]/{f=1;next} /^\[/{f=0} f' "$1" } -@test "all seven observability units exist under dankclaw/observability/quadlet/" { +@test "all eight observability units exist under dankclaw/observability/quadlet/" { local svc for svc in "${UNITS[@]}"; do [ -f "$UNIT_DIR/$svc.container" ] || { echo "missing $svc.container"; return 1; } @@ -668,3 +668,89 @@ assert not any('container_network' in e or 'container_fs_' in e for e in names), [ -f "$sandbox/opt/grafana/provisioning/dashboards/provider.yml" ] [ -f "$sandbox/opt/grafana/provisioning/datasources/prometheus.yml" ] } + +@test "changes under shared/blackbox/ plan the prober" { + run python3 "$REPO_ROOT/scripts/deploy-plan.py" --target dankclaw \ + --paths shared/blackbox/blackbox.yml + [ "$status" -eq 0 ] + [ "$output" = "blackbox-exporter" ] +} + +@test "the copier runs the same code path the deployer runs" { + # Textually pinned so a refactor cannot quietly route the deployer around + # the copy_* functions the seams execute. + grep -q ' prometheus)' "$REMOTE_DEPLOY" + grep -q 'copy_prometheus_config' "$REMOTE_DEPLOY" + grep -q 'copy_alertmanager_config' "$REMOTE_DEPLOY" + grep -q 'copy_loki_config' "$REMOTE_DEPLOY" + grep -q ' blackbox-exporter)' "$REMOTE_DEPLOY" + grep -q 'copy_blackbox_config' "$REMOTE_DEPLOY" +} + +@test "the alloy copier installs the host-labeled config, token-free" { + # Executes copy_alloy_config for real against a sandbox (stubbed + # run_privileged). A surviving GRAFANA_HOST_LABEL token would ship a + # config whose host label is a literal placeholder — the exact class of + # silent-label defect this tranche retired the board for. + local sandbox="$BATS_TEST_TMPDIR/opt" + ( + cd "$REPO_ROOT" + eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + # NOTE: VAR=x func does not bind for shell functions — export. + export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox" + copy_alloy_config + ) + [ -f "$sandbox/config/config.alloy" ] + grep -q 'host = "dankclaw",' "$sandbox/config/config.alloy" + ! grep -q "GRAFANA_HOST_LABEL" "$sandbox/config/config.alloy" +} + +@test "the blackbox copier installs the prober config" { + local sandbox="$BATS_TEST_TMPDIR/blackbox" + ( + cd "$REPO_ROOT" + eval "$(sed -n '/^copy_blackbox_config()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + export BLACKBOX_CONF_ROOT="$sandbox" + copy_blackbox_config + ) + [ -f "$sandbox/blackbox.yml" ] + grep -q "listen_address: 127.0.0.1:9115" "$sandbox/blackbox.yml" +} + +@test "the alloy copier refuses to install a config the engine rejects" { + # The syntax gate: alloy attributes are position-sensitive and a + # misplaced one only explodes at run time (crash-loop behind a green + # deploy). Prove the copier refuses a config the pinned image won't + # validate, and installs nothing. Needs the alloy image; skipped + # where it is absent (the copier's gate is likewise image-gated). + if ! podman image exists localhost/franksdank/alloy:latest; then + skip "franksdank/alloy image not present on this box" + fi + local sandbox="$BATS_TEST_TMPDIR/opt" cfg="$BATS_TEST_TMPDIR/broken.alloy" + sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg" + printf 'bogus_attr = "x"\n' >> "$cfg" + ! ( + cd "$REPO_ROOT" + eval "$(sed -n '/^copy_alloy_config()/,/^}/p' scripts/deploy-remote.sh)" + run_privileged() { "$@"; } + cp() { if [ "$1" = "$cfg" ]; then return 1; fi; command cp "$@"; } + export TARGET=dankclaw ALLOY_CONF_ROOT="$sandbox" + copy_alloy_config + ) + [ ! -f "$sandbox/config/config.alloy" ] +} + +@test "the installed alloy config is engine-valid" { + # Runs the pinned image's own validator over the shipped config. The + # 2026-10-21 defect (`on_error` on the labels stage, which alloy 1.21 + # does not define) passed every textual seam and only failed at boot. + if ! podman image exists localhost/franksdank/alloy:latest; then + skip "franksdank/alloy image not present on this box" + fi + local cfg="$BATS_TEST_TMPDIR/rendered.alloy" + sed 's/GRAFANA_HOST_LABEL/dankclaw/g' shared/alloy/config.alloy > "$cfg" + podman run --rm -v "$cfg:/c.alloy:ro" --entrypoint /usr/bin/alloy \ + localhost/franksdank/alloy:latest validate /c.alloy +}