From 933420fd29ac04f03107bd82e21fe624137f5a8c Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 19 Oct 2026 16:07:28 -0700 Subject: [PATCH] fix(open-notebook): the database probe has to be exec-form JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit podman wraps a bare HealthCmd as CMD-SHELL (measured on Dankclaw's podman 5.7: --health-cmd "/surreal is-ready" stores {"Test":["CMD-SHELL","/surreal is-ready"]}), and the pinned SurrealDB v2 image is distroless — no /bin/sh, not even `ls`. So the probe could never run: every 30 s interval exited 1 in ~30 ms with empty output while the database was listening and answering, and open-notebook's first deploy died at the gate ("did not become healthy in 90s"). The hand-check `podman exec … /surreal is-ready` printed OK, which is why the old comment believed the probe worked. The JSON array survives the Quadlet generator (--health-cmd "[\"/surreal\",\"is-ready\"]") and stores ["CMD","/surreal","is-ready"], which execs directly: a throwaway instance of the pinned digest went healthy on that form. The seam is re-pinned to the shell-free contract; the old assertion forbade quotes in any HealthCmd, which is what pinned the broken shape. --- .../quadlet/open-notebook-surrealdb.container | 25 +++++++++++++------ tests/open_notebook.bats | 23 +++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/dankclaw/open-notebook/quadlet/open-notebook-surrealdb.container b/dankclaw/open-notebook/quadlet/open-notebook-surrealdb.container index acbcd94..11e9fb2 100644 --- a/dankclaw/open-notebook/quadlet/open-notebook-surrealdb.container +++ b/dankclaw/open-notebook/quadlet/open-notebook-surrealdb.container @@ -48,13 +48,22 @@ # without `podman unshare` — accepted for a datastore nobody edits by hand; # episode AUDIO is a host directory for the opposite reason (see the app unit). # -# HEALTH. The base image is distroless: no shell, no curl, no wget, so no -# CMD-SHELL probe and no wget/curl probe is possible here (same class of unit -# as loki, whose probe had to move into the image). SurrealDB ships its own -# probe instead — `surreal is-ready` connects to ws://localhost:8000 by default -# and needs no credentials — verified against this digest: it prints `OK` and -# exits 0 while the server is up. Argument-only HealthCmd, like every unit on -# this host, because the quadlet generator splits it on spaces. +# HEALTH. The base image is distroless — measured against this digest, not even +# `ls` exists (`exec: "ls": executable file not found in $PATH`), so there is no +# `/bin/sh`, no curl and no wget. SurrealDB ships its own probe instead: +# `surreal is-ready` connects to ws://localhost:8000 by default, needs no +# credentials, prints `OK` and exits 0 while the server is up. +# +# It must be handed over as a QUADLET-SIDE JSON ARRAY, not a bare command. +# podman wraps a bare `HealthCmd` string as `CMD-SHELL` (measured on this host's +# podman 5.7: `--health-cmd "/surreal is-ready"` stores +# `{"Test":["CMD-SHELL","/surreal is-ready"]}`), and with no `/bin/sh` in the +# image that probe fails in ~30 ms with empty output on every interval while the +# database answers perfectly — which is exactly how this service's first deploy +# died on 2026-10-19 (`did not become healthy in 90s`). The quoted array survives +# the generator (`--health-cmd "[\"/surreal\",\"is-ready\"]"`) and stores +# `["CMD","/surreal","is-ready"]`, which execs directly and goes healthy. Do not +# "simplify" it back to a bare command. [Unit] Description=SurrealDB (open-notebook datastore) @@ -70,7 +79,7 @@ EnvironmentFile=/home/dank/serverAppConfigs/scripts/open-notebook.env EnvironmentFile=/home/dank/Containers/open-notebook/env Exec=start --log info rocksdb:/mydata/mydatabase.db Volume=open-notebook-db:/mydata -HealthCmd=/surreal is-ready +HealthCmd=["/surreal","is-ready"] HealthInterval=30s HealthTimeout=10s HealthRetries=3 diff --git a/tests/open_notebook.bats b/tests/open_notebook.bats index 14897ed..f105ebc 100755 --- a/tests/open_notebook.bats +++ b/tests/open_notebook.bats @@ -170,18 +170,19 @@ DB_PIN="sha256:e5d7387e3727bbe2f7cd6d869e48cd34777272258572ead37c9656b25f745723" ! grep -qE '^Exec=.*--(pass|user)' "$DB" } -@test "both health probes are argument-only and name real paths" { - for unit in "$APP" "$DB"; do - cmd="$(sed -n 's/^HealthCmd=//p' "$unit")" - [ -n "$cmd" ] || { echo "$(basename "$unit") has no HealthCmd"; false; } - # The quadlet generator splits HealthCmd on spaces, so anything needing - # a shell (quotes, pipes, &&) would be silently mis-sent. - ! printf '%s' "$cmd" | grep -qE '["&;|`]' - done - # The API's own route, exempt from its password middleware upstream. +@test "the database probe can run in a shell-less image and the API probe names the health route" { + # podman wraps a BARE HealthCmd as CMD-SHELL (measured on this host's podman + # 5.7), and the SurrealDB image is distroless — no /bin/sh — so a bare + # command there is a probe that can only ever fail: the database answers + # perfectly while the health status stays `unhealthy` and the deploy gate + # times out. Only the exec-form JSON array runs in that image. + db_cmd="$(sed -n 's/^HealthCmd=//p' "$DB")" + [ "$db_cmd" = '["/surreal","is-ready"]' ] || { + echo "database probe is not exec-form JSON: $db_cmd"; false; } + # The API image ships curl and a shell, so a bare command is fine there; it + # must name the route that is exempt from the password middleware, on the + # CONTAINER port (the probe runs inside the netns). grep -qxF 'HealthCmd=curl -fsS http://127.0.0.1:5055/health' "$APP" - # The distroless database image has no shell and no curl; it ships a probe. - grep -qxF 'HealthCmd=/surreal is-ready' "$DB" } @test "the deployer gates open-notebook on the health path that actually answers" {