fix(observability): give loki an exec-form healthcheck in its image

The loki unit came up 'unhealthy' after the quadlet cutover. Cause: a unit's
HealthCmd reaches podman as CMD-SHELL, i.e. /bin/sh -c, and the loki image is
distroless — no shell exists in it. The busybox command itself works (probed
by hand: /ready answers 'ready'); it was the shell wrapper that could not run.
podman does not accept an exec-form command through --health-cmd either: it
stores a JSON array as one literal argument.

The command therefore moves into dankclaw/loki/Containerfile as a
HEALTHCHECK ... CMD [...] exec form, which is what that image exists for — it
already ships the static busybox solely so a healthcheck has something to run.
podman reads the image's healthcheck when the unit declares none.

dankclaw/loki/Containerfile

@@ -1,9 +1,13 @@
 # Loki + a static busybox.
 #
-# grafana/loki is distroless: no shell, no wget, so the compose healthcheck
+# grafana/loki is distroless: no shell, no wget, so nothing can run inside the
-# cannot run anything inside the container. This image drops a static
+# container to check it. This image drops a static busybox into it for the
-# busybox into it for the healthcheck command only. All Loki behavior is
+# healthcheck. All Loki behavior is unchanged.
-# unchanged.
+#
+# The healthcheck lives HERE, in exec form, and not in the Quadlet unit: a
+# unit's HealthCmd reaches podman as CMD-SHELL, which needs /bin/sh — the one
+# thing this image does not have. Specifying it here is also what lets the
+# container report its own health instead of looking merely "Up" while wedged.
 #
 # Build on dankclaw:
 #   podman build -t franksdank/loki:latest -f dankclaw/loki/Containerfile .
@@ -15,4 +19,7 @@ FROM docker.io/library/alpine:latest AS tools
 RUN apk add --no-cache busybox-static
 
 FROM docker.io/grafana/loki:latest
-COPY --from=tools /bin/busybox.static /usr/local/bin/busybox
+COPY --from=tools /bin/busybox.static /usr/local/bin/busybox
+# Exec form (JSON array): no shell is involved.
+HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
+    CMD ["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"]

dankclaw/observability/quadlet/loki.container

@@ -43,11 +43,11 @@ Exec=-config.file=/etc/loki/loki-config.yaml
 PublishPort=127.0.0.1:3100:3100
 Volume=/opt/loki/config:/etc/loki
 Volume=/opt/loki/data:/loki
-HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:3100/ready
+# No HealthCmd here on purpose. A unit's HealthCmd reaches podman as
-HealthInterval=30s
+# CMD-SHELL, which means `/bin/sh -c <command>` — and this image is
-HealthTimeout=10s
+# distroless, with no shell at all. The healthcheck therefore ships in the
-HealthRetries=3
+# image (dankclaw/loki/Containerfile) in exec form, where it also carries its
-HealthStartPeriod=30s
+# interval, timeout, start period and retry count.
 
 [Service]
 Restart=always

scripts/test-changed.sh

@@ -209,6 +209,14 @@ for path in "${CHANGED[@]}"; do
             add_test tests/docs_completeness.bats
             add_test tests/deployment_scoping.bats
             ;;
+        # loki and alloy images are built from this repo's Containerfiles; the
+        # loki one carries the healthcheck command (see its comment), so both
+        # seams read it.
+        dankclaw/loki/*|dankclaw/alloy/*)
+            add_test tests/observability_quadlet.bats
+            add_test tests/caddyfile.bats
+            add_test tests/deployment_scoping.bats
+            ;;
         dankclaw/udev/*)
             # Host-level udev rules are applied by manual sudo install + udevadm
             # trigger, not dispatched as a service; nothing here is bats-testable.

tests/caddyfile.bats

@@ -221,15 +221,16 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
     done
 }
 
-@test "loki and alloy healthchecks use the busybox wget" {
+@test "loki and alloy healthchecks run the busybox they ship" {
-    # Both images are distroless-ish, so the healthcheck only works because
+    # Both images are distroless-ish, so a healthcheck only works because the
-    # the local Containerfile drops a static busybox into the image.
+    # local Containerfile drops a static busybox into the image. loki cannot
-    local n=0 unit
+    # take the command from the unit (podman would wrap it in CMD-SHELL and
-    for unit in "$REPO_ROOT"/dankclaw/observability/quadlet/loki.container \
+    # the image has no shell), so it comes from the image; alloy's image does
-                "$REPO_ROOT"/dankclaw/observability/quadlet/alloy.container; do
+    # have a shell, so its command lives in the unit.
-        grep -q '^HealthCmd=/usr/local/bin/busybox wget' "$unit" && n=$((n + 1))
+    grep -q '^HealthCmd=/usr/local/bin/busybox wget' \
-    done
+        "$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
-    [ "$n" -eq 2 ] || { echo "expected 2 busybox healthchecks, found $n"; return 1; }
+    grep -q 'CMD \["/usr/local/bin/busybox", "wget"' \
+        "$REPO_ROOT/dankclaw/loki/Containerfile"
 }
 
 @test "alloy's UI address comes from the unit, not the config file" {

tests/observability_quadlet.bats

@@ -135,15 +135,30 @@ service_section() {
 }
 
 @test "every observability unit has a health check" {
-    # A unit without one shows as plainly "Up" in podman ps while its process
+    # A container without one shows as plainly "Up" in podman ps while its
-    # is wedged, and the deploy has nothing to wait on.
+    # process is wedged, and the deploy has nothing to wait on.
+    #
+    # Four units declare it inline. loki cannot: its image is distroless, so a
+    # unit HealthCmd would reach podman as CMD-SHELL and fail for want of
+    # /bin/sh. Its health command ships in the image instead.
     local svc
-    for svc in "${UNITS[@]}"; do
+    for svc in prometheus grafana alloy node-exporter; do
         grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
             || { echo "$svc has no HealthCmd"; return 1; }
         grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
             || { echo "$svc has no HealthRetries"; return 1; }
     done
+    ! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
+}
+
+@test "loki's health command is exec form in the image, not a shell string" {
+    # The whole reason the loki image exists: distroless has no shell, so the
+    # command must be a JSON array that podman execs directly.
+    local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
+    grep -q '^HEALTHCHECK ' "$cf"
+    grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
+    # ... and the busybox it execs has to be in the image.
+    grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
 }
 
 @test "grafana gets its OAuth client secret without getting a whole env file" {