diff --git a/dankclaw/loki/Containerfile b/dankclaw/loki/Containerfile index cec8b0c..b14626c 100644 --- a/dankclaw/loki/Containerfile +++ b/dankclaw/loki/Containerfile @@ -1,9 +1,13 @@ # Loki + a static busybox. # -# grafana/loki is distroless: no shell, no wget, so the compose healthcheck -# cannot run anything inside the container. This image drops a static -# busybox into it for the healthcheck command only. All Loki behavior is -# unchanged. +# grafana/loki is distroless: no shell, no wget, so nothing can run inside the +# container to check it. This image drops a static busybox into it for the +# healthcheck. All Loki behavior is unchanged. +# +# The healthcheck lives HERE, in exec form, and not in the Quadlet unit: a +# unit's HealthCmd reaches podman as CMD-SHELL, which needs /bin/sh — the one +# thing this image does not have. Specifying it here is also what lets the +# container report its own health instead of looking merely "Up" while wedged. # # Build on dankclaw: # podman build -t franksdank/loki:latest -f dankclaw/loki/Containerfile . @@ -15,4 +19,7 @@ FROM docker.io/library/alpine:latest AS tools RUN apk add --no-cache busybox-static FROM docker.io/grafana/loki:latest -COPY --from=tools /bin/busybox.static /usr/local/bin/busybox \ No newline at end of file +COPY --from=tools /bin/busybox.static /usr/local/bin/busybox +# Exec form (JSON array): no shell is involved. +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD ["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"] \ No newline at end of file diff --git a/dankclaw/observability/quadlet/loki.container b/dankclaw/observability/quadlet/loki.container index bccb7ea..e3bc31d 100644 --- a/dankclaw/observability/quadlet/loki.container +++ b/dankclaw/observability/quadlet/loki.container @@ -43,11 +43,11 @@ Exec=-config.file=/etc/loki/loki-config.yaml PublishPort=127.0.0.1:3100:3100 Volume=/opt/loki/config:/etc/loki Volume=/opt/loki/data:/loki -HealthCmd=/usr/local/bin/busybox wget --spider -q http://localhost:3100/ready -HealthInterval=30s -HealthTimeout=10s -HealthRetries=3 -HealthStartPeriod=30s +# No HealthCmd here on purpose. A unit's HealthCmd reaches podman as +# CMD-SHELL, which means `/bin/sh -c ` — and this image is +# distroless, with no shell at all. The healthcheck therefore ships in the +# image (dankclaw/loki/Containerfile) in exec form, where it also carries its +# interval, timeout, start period and retry count. [Service] Restart=always diff --git a/scripts/test-changed.sh b/scripts/test-changed.sh index 42341c5..e24c38c 100755 --- a/scripts/test-changed.sh +++ b/scripts/test-changed.sh @@ -209,6 +209,14 @@ for path in "${CHANGED[@]}"; do add_test tests/docs_completeness.bats add_test tests/deployment_scoping.bats ;; + # loki and alloy images are built from this repo's Containerfiles; the + # loki one carries the healthcheck command (see its comment), so both + # seams read it. + dankclaw/loki/*|dankclaw/alloy/*) + add_test tests/observability_quadlet.bats + add_test tests/caddyfile.bats + add_test tests/deployment_scoping.bats + ;; dankclaw/udev/*) # Host-level udev rules are applied by manual sudo install + udevadm # trigger, not dispatched as a service; nothing here is bats-testable. diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index cb2a7de..9040a32 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -221,15 +221,16 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" done } -@test "loki and alloy healthchecks use the busybox wget" { - # Both images are distroless-ish, so the healthcheck only works because - # the local Containerfile drops a static busybox into the image. - local n=0 unit - for unit in "$REPO_ROOT"/dankclaw/observability/quadlet/loki.container \ - "$REPO_ROOT"/dankclaw/observability/quadlet/alloy.container; do - grep -q '^HealthCmd=/usr/local/bin/busybox wget' "$unit" && n=$((n + 1)) - done - [ "$n" -eq 2 ] || { echo "expected 2 busybox healthchecks, found $n"; return 1; } +@test "loki and alloy healthchecks run the busybox they ship" { + # Both images are distroless-ish, so a healthcheck only works because the + # local Containerfile drops a static busybox into the image. loki cannot + # take the command from the unit (podman would wrap it in CMD-SHELL and + # the image has no shell), so it comes from the image; alloy's image does + # have a shell, so its command lives in the unit. + grep -q '^HealthCmd=/usr/local/bin/busybox wget' \ + "$REPO_ROOT/dankclaw/observability/quadlet/alloy.container" + grep -q 'CMD \["/usr/local/bin/busybox", "wget"' \ + "$REPO_ROOT/dankclaw/loki/Containerfile" } @test "alloy's UI address comes from the unit, not the config file" { diff --git a/tests/observability_quadlet.bats b/tests/observability_quadlet.bats index e20b592..aeb0a5f 100755 --- a/tests/observability_quadlet.bats +++ b/tests/observability_quadlet.bats @@ -135,15 +135,30 @@ service_section() { } @test "every observability unit has a health check" { - # A unit without one shows as plainly "Up" in podman ps while its process - # is wedged, and the deploy has nothing to wait on. + # A container without one shows as plainly "Up" in podman ps while its + # process is wedged, and the deploy has nothing to wait on. + # + # Four units declare it inline. loki cannot: its image is distroless, so a + # unit HealthCmd would reach podman as CMD-SHELL and fail for want of + # /bin/sh. Its health command ships in the image instead. local svc - for svc in "${UNITS[@]}"; do + for svc in prometheus grafana alloy node-exporter; do grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \ || { echo "$svc has no HealthCmd"; return 1; } grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \ || { echo "$svc has no HealthRetries"; return 1; } done + ! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container" +} + +@test "loki's health command is exec form in the image, not a shell string" { + # The whole reason the loki image exists: distroless has no shell, so the + # command must be a JSON array that podman execs directly. + local cf="$REPO_ROOT/dankclaw/loki/Containerfile" + grep -q '^HEALTHCHECK ' "$cf" + grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf" + # ... and the busybox it execs has to be in the image. + grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf" } @test "grafana gets its OAuth client secret without getting a whole env file" {