fix(observability): give loki an exec-form healthcheck in its image
The loki unit came up 'unhealthy' after the quadlet cutover. Cause: a unit's HealthCmd reaches podman as CMD-SHELL, i.e. /bin/sh -c, and the loki image is distroless — no shell exists in it. The busybox command itself works (probed by hand: /ready answers 'ready'); it was the shell wrapper that could not run. podman does not accept an exec-form command through --health-cmd either: it stores a JSON array as one literal argument. The command therefore moves into dankclaw/loki/Containerfile as a HEALTHCHECK ... CMD [...] exec form, which is what that image exists for — it already ships the static busybox solely so a healthcheck has something to run. podman reads the image's healthcheck when the unit declares none.
This commit is contained in:
1 parent
fe4eeaa03a
commit
b96359e628
5 files changed
+53
-22
No files matched your search
+10
-9
@@ -221,15 +221,16 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
|
||||
done
|
||||
}
|
||||
|
||||
@test "loki and alloy healthchecks use the busybox wget" {
|
||||
# Both images are distroless-ish, so the healthcheck only works because
|
||||
# the local Containerfile drops a static busybox into the image.
|
||||
local n=0 unit
|
||||
for unit in "$REPO_ROOT"/dankclaw/observability/quadlet/loki.container \
|
||||
"$REPO_ROOT"/dankclaw/observability/quadlet/alloy.container; do
|
||||
grep -q '^HealthCmd=/usr/local/bin/busybox wget' "$unit" && n=$((n + 1))
|
||||
done
|
||||
[ "$n" -eq 2 ] || { echo "expected 2 busybox healthchecks, found $n"; return 1; }
|
||||
@test "loki and alloy healthchecks run the busybox they ship" {
|
||||
# Both images are distroless-ish, so a healthcheck only works because the
|
||||
# local Containerfile drops a static busybox into the image. loki cannot
|
||||
# take the command from the unit (podman would wrap it in CMD-SHELL and
|
||||
# the image has no shell), so it comes from the image; alloy's image does
|
||||
# have a shell, so its command lives in the unit.
|
||||
grep -q '^HealthCmd=/usr/local/bin/busybox wget' \
|
||||
"$REPO_ROOT/dankclaw/observability/quadlet/alloy.container"
|
||||
grep -q 'CMD \["/usr/local/bin/busybox", "wget"' \
|
||||
"$REPO_ROOT/dankclaw/loki/Containerfile"
|
||||
}
|
||||
|
||||
@test "alloy's UI address comes from the unit, not the config file" {
|
||||
|
||||
@@ -135,15 +135,30 @@ service_section() {
|
||||
}
|
||||
|
||||
@test "every observability unit has a health check" {
|
||||
# A unit without one shows as plainly "Up" in podman ps while its process
|
||||
# is wedged, and the deploy has nothing to wait on.
|
||||
# A container without one shows as plainly "Up" in podman ps while its
|
||||
# process is wedged, and the deploy has nothing to wait on.
|
||||
#
|
||||
# Four units declare it inline. loki cannot: its image is distroless, so a
|
||||
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
||||
# /bin/sh. Its health command ships in the image instead.
|
||||
local svc
|
||||
for svc in "${UNITS[@]}"; do
|
||||
for svc in prometheus grafana alloy node-exporter; do
|
||||
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc has no HealthCmd"; return 1; }
|
||||
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc has no HealthRetries"; return 1; }
|
||||
done
|
||||
! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
|
||||
}
|
||||
|
||||
@test "loki's health command is exec form in the image, not a shell string" {
|
||||
# The whole reason the loki image exists: distroless has no shell, so the
|
||||
# command must be a JSON array that podman execs directly.
|
||||
local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
|
||||
grep -q '^HEALTHCHECK ' "$cf"
|
||||
grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
|
||||
# ... and the busybox it execs has to be in the image.
|
||||
grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
|
||||
}
|
||||
|
||||
@test "grafana gets its OAuth client secret without getting a whole env file" {
|
||||
|
||||
Reference in new issue
Block a user