fix(observability): give loki an exec-form healthcheck in its image
The loki unit came up 'unhealthy' after the quadlet cutover. Cause: a unit's HealthCmd reaches podman as CMD-SHELL, i.e. /bin/sh -c, and the loki image is distroless — no shell exists in it. The busybox command itself works (probed by hand: /ready answers 'ready'); it was the shell wrapper that could not run. podman does not accept an exec-form command through --health-cmd either: it stores a JSON array as one literal argument. The command therefore moves into dankclaw/loki/Containerfile as a HEALTHCHECK ... CMD [...] exec form, which is what that image exists for — it already ships the static busybox solely so a healthcheck has something to run. podman reads the image's healthcheck when the unit declares none.
This commit is contained in:
1 parent
fe4eeaa03a
commit
b96359e628
5 files changed
+53
-22
No files matched your search
@@ -135,15 +135,30 @@ service_section() {
|
||||
}
|
||||
|
||||
@test "every observability unit has a health check" {
|
||||
# A unit without one shows as plainly "Up" in podman ps while its process
|
||||
# is wedged, and the deploy has nothing to wait on.
|
||||
# A container without one shows as plainly "Up" in podman ps while its
|
||||
# process is wedged, and the deploy has nothing to wait on.
|
||||
#
|
||||
# Four units declare it inline. loki cannot: its image is distroless, so a
|
||||
# unit HealthCmd would reach podman as CMD-SHELL and fail for want of
|
||||
# /bin/sh. Its health command ships in the image instead.
|
||||
local svc
|
||||
for svc in "${UNITS[@]}"; do
|
||||
for svc in prometheus grafana alloy node-exporter; do
|
||||
grep -q '^HealthCmd=' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc has no HealthCmd"; return 1; }
|
||||
grep -q '^HealthRetries=' "$UNIT_DIR/$svc.container" \
|
||||
|| { echo "$svc has no HealthRetries"; return 1; }
|
||||
done
|
||||
! grep -q '^HealthCmd=' "$UNIT_DIR/loki.container"
|
||||
}
|
||||
|
||||
@test "loki's health command is exec form in the image, not a shell string" {
|
||||
# The whole reason the loki image exists: distroless has no shell, so the
|
||||
# command must be a JSON array that podman execs directly.
|
||||
local cf="$REPO_ROOT/dankclaw/loki/Containerfile"
|
||||
grep -q '^HEALTHCHECK ' "$cf"
|
||||
grep -q 'CMD \["/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"\]' "$cf"
|
||||
# ... and the busybox it execs has to be in the image.
|
||||
grep -q 'COPY --from=tools /bin/busybox.static /usr/local/bin/busybox' "$cf"
|
||||
}
|
||||
|
||||
@test "grafana gets its OAuth client secret without getting a whole env file" {
|
||||
|
||||
Reference in new issue
Block a user