From 765a8cb7de1c3c6b5f422469aba1a6eafaf35113 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 21 Oct 2026 13:33:03 -0700 Subject: [PATCH] feat(deploy): blackbox copier, alloy engine-validation gate, planner wiring copy_alloy_config now validates the rendered config with the very alloy image that will run it, before installing: a misplaced attribute crash-loops the unit behind a green deploy, and no textual seam can see it. copy_blackbox_config ships the prober config; the planner maps the new unit and shared/blackbox/** to the blackbox-exporter service, ordered before prometheus. --- scripts/deploy-plan.py | 9 ++++++-- scripts/deploy-remote.sh | 45 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/scripts/deploy-plan.py b/scripts/deploy-plan.py index a03231c..009fb72 100755 --- a/scripts/deploy-plan.py +++ b/scripts/deploy-plan.py @@ -34,7 +34,6 @@ SERVICE_ORDER = ( "hermes", "hermes-dashboard", "hermes-webui", - "prometheus", "grafana", "loki", "alloy", @@ -42,6 +41,9 @@ SERVICE_ORDER = ( "cadvisor", "gpu-exporter", "alertmanager", + # The prober must answer before prometheus re-runs the services job. + "blackbox-exporter", + "prometheus", "authentik-postgres", "authentik-server", "authentik-worker", @@ -97,6 +99,7 @@ DANKCLAW_SERVICES = { "cadvisor", "gpu-exporter", "alertmanager", + "blackbox-exporter", "authentik-postgres", "authentik-server", "authentik-worker", @@ -277,6 +280,8 @@ def services_for_path(path: str, target: str) -> set[str]: # Provisioned datasources and dashboards: git is the source of # truth, so a change here must restart the service that reads them. _add(services, "grafana") + elif group == "blackbox": + _add(services, "blackbox-exporter") elif group == "homepage": # Homepage is a Dankclaw-owned deployment. Shared homepage files # are intentionally not copied during a Dankserver-only run. @@ -380,7 +385,7 @@ def services_for_path(path: str, target: str) -> set[str]: # consumer (prometheus before grafana, loki before alloy). if len(parts) >= 4 and parts[2] == "quadlet": name = parts[3].removesuffix(".container") - if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager"}: + if name in {"prometheus", "grafana", "loki", "alloy", "node-exporter", "cadvisor", "alertmanager", "blackbox-exporter"}: _add(services, name) return services if len(parts) >= 2 and parts[1] == "gitea": diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index c715ba6..560b765 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -311,8 +311,45 @@ copy_loki_config() { } copy_alloy_config() { - run_privileged mkdir -p /opt/alloy/config - run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy + # The repo config carries the token GRAFANA_HOST_LABEL; the installed copy + # carries the host's name, so Loki streams say WHICH machine's journal a + # line came from without a second config file per host. + # *_CONF_ROOT are test seams: the bats cases execute these copiers for + # real against a sandbox with a stubbed run_privileged. + local root="${ALLOY_CONF_ROOT:-/opt/alloy}" tmp + tmp="$(mktemp)" + sed "s/GRAFANA_HOST_LABEL/$TARGET/g" shared/alloy/config.alloy > "$tmp" + if ! grep -q "host = \"$TARGET\"," "$tmp" || grep -q "GRAFANA_HOST_LABEL" "$tmp"; then + echo "alloy config: host-label substitution failed" >&2 + rm -f "$tmp" + return 1 + fi + # Syntax gate: alloy's config language has no schema — a misplaced + # attribute (e.g. `on_error` outside the json stage) parses only at RUN + # time, and the unit then crash-loops with a healthy-looking deploy. + # Validate against the very image that will run it, before installing. + # Skipped only in the bats sandbox (no image there). + if [ "$root" = "/opt/alloy" ] || podman image exists localhost/franksdank/alloy:latest 2>/dev/null; then + if ! podman run --rm -v "$tmp:/c.alloy:ro" \ + --entrypoint /usr/bin/alloy localhost/franksdank/alloy:latest \ + validate /c.alloy; then + echo "alloy config: validate FAILED against the pinned image" >&2 + rm -f "$tmp" + return 1 + fi + fi + run_privileged mkdir -p "$root/config" + run_privileged cp "$tmp" "$root/config/config.alloy" + rm -f "$tmp" +} + +copy_blackbox_config() { + # Single-file mount target: the unit binds this exact path into + # /etc/blackbox_exporter/config.yml. cp (not cp -r) keeps it one file. + # BLACKBOX_CONF_ROOT is the same test seam. + local root="${BLACKBOX_CONF_ROOT:-/opt/blackbox}" + run_privileged mkdir -p "$root" + run_privileged cp shared/blackbox/blackbox.yml "$root/blackbox.yml" } copy_grafana_provisioning() { @@ -563,6 +600,9 @@ deploy_observability() { ;; cadvisor) ;; + blackbox-exporter) + copy_blackbox_config + ;; *) echo "unknown observability service: $service" >&2 return 2 @@ -577,6 +617,7 @@ deploy_observability() { loki) wait_for_health loki http://127.0.0.1:3100/ready 60 ;; node-exporter) wait_for_health node-exporter http://127.0.0.1:9100/metrics 60 ;; cadvisor) wait_for_health cadvisor http://127.0.0.1:8080/healthz 90 ;; + blackbox-exporter) wait_for_health blackbox-exporter http://127.0.0.1:9115/-/healthy 60 ;; esac }