fix: basicauth double-prefix render; open-webui off slirp4netns

basicauth (2026-08-22 incident):
- the Caddyfile template carried its own $2a$14$ prefix and deploy.sh
  substituted the full hash (prefix included), rendering an invalid
  double-prefixed hash. Every basicauth vhost rejected every password
  with 401 and Caddy logged nothing.
- template is now a bare REPLACE_WITH_HASH token; tests pin the render
  contract (no prefix in template; rendered line is exactly 'dank <hash>').

open-webui (2026-08-22 incident):
- its slirp4netns helper process died, leaving the container running
  with only 'lo' in its netns: port 3005 accepted TCP and reset every
  request, so chat served empty responses for ~2 days, unlogged.
- move to a pinned bridge network (systemd-open-webui, 10.89.3.0/24,
  gw 10.89.3.1) — no per-container helper to die.
- LLM endpoint reached via the gateway (host.containers.internal is
  slirp-only): OPENAI_API_BASE_URL=http://10.89.3.1:52625.
- add a quadlet healthcheck: /health (app) + non-loopback interface
  (network liveness; the dead-tap state leaves only 'lo').
- deploy script now copies the .network file too.
- docs updated (OpenWebUI History section, Caddy basic-auth contract).
This commit is contained in:
Frank
2026-08-22 13:07:43 -07:00
parent 5c9ade4e7a
commit da68b5ce1a
9 changed files with 129 additions and 27 deletions
+13
View File
@@ -49,6 +49,19 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
grep -q 'REPLACE_WITH_HASH' "$CADDYFILE"
}
@test "the basicauth placeholder is a bare token (no bcrypt prefix in the template)" {
# Regression (2026-08-22): the template line was
# dank $2a$14$REPLACE_WITH_HASH
# and deploy.sh substitutes the FULL hash (prefix included), so the
# rendered line became an invalid double-prefixed hash and every
# basicauth vhost rejected every password with 401.
if grep -qE '\$2[abxy]\$[0-9]+\$REPLACE_WITH_HASH' "$CADDYFILE"; then
echo "Caddyfile: bcrypt prefix before REPLACE_WITH_HASH (double-prefix regression)"
return 1
fi
grep -qE 'dank REPLACE_WITH_HASH' "$CADDYFILE"
}
@test "compose no longer defines the quadlet-replaced services" {
# gitea, its postgres, and open-webui run as boot-persistent quadlets on
# dankclaw (see dankclaw/gitea/quadlet, dankclaw/open-webui/quadlet).
+5
View File
@@ -44,4 +44,9 @@ load helpers
[ -n "$rendered" ]
[[ "$rendered" == *"${hash}"* ]]
! grep -q 'REPLACE_WITH_HASH' <<< "$rendered"
# The rendered line must be exactly "dank <hash>". A template that keeps
# its own $2a$14$ prefix renders an invalid double-prefixed hash and
# every basicauth vhost returns 401 (2026-08-22 incident).
grep -qF "dank ${hash}" <<< "$rendered"
! grep -qF '$2a$14$$2a$14$' <<< "$rendered"
}
+32 -6
View File
@@ -24,9 +24,33 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet"
return 0
}
@test "open-webui is isolated on slirp4netns" {
grep -qE '^Network=slirp4netns' "$QDIR/open-webui.container" \
|| { echo "open-webui.container must set Network=slirp4netns"; return 1; }
@test "open-webui is on a private bridge network (not slirp4netns)" {
# slirp4netns runs a user-space helper per container. When the helper
# dies, the container keeps running but loses its tap interface: a
# zombie that accepts TCP and resets every request (2026-08-22: chat
# returned empty responses for ~2 days). A bridge network has no
# per-container helper, so the failure mode cannot occur.
grep -qE '^Network=open-webui\.network' "$QDIR/open-webui.container" \
|| { echo "open-webui.container must set Network=open-webui.network"; return 1; }
! grep -qE '^Network=slirp4netns' "$QDIR/open-webui.container"
[ -f "$QDIR/open-webui.network" ]
}
@test "open-webui network pins a deterministic subnet and gateway" {
# The LLM endpoint is reached via the bridge gateway, so the subnet
# must be pinned (an auto-assigned subnet is not a promise).
grep -qE '^Subnet=10\.89\.3\.0/24' "$QDIR/open-webui.network"
grep -qE '^Gateway=10\.89\.3\.1' "$QDIR/open-webui.network"
}
@test "open-webui has a healthcheck that sees the app AND the network" {
# The /health probe catches app hangs; the non-loopback interface check
# catches the dead-tap zombie state (only "lo" left in the netns).
grep -qE '^HealthCmd=CMD-SHELL .*/health.*' "$QDIR/open-webui.container"
grep -qE '^HealthCmd=.*sys/class/net.*' "$QDIR/open-webui.container"
grep -qE '^HealthInterval=30s' "$QDIR/open-webui.container"
grep -qE '^HealthRetries=3' "$QDIR/open-webui.container"
grep -qE '^HealthStartPeriod=' "$QDIR/open-webui.container"
}
@test "open-webui publishes 3005:8080 (a free port; 3000 is Gitea)" {
@@ -34,10 +58,12 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet"
|| { echo "open-webui.container must publish 3005:8080"; return 1; }
}
@test "open-webui points its LLM at the live vLLM backend (52625)" {
grep -qE '^Environment=OPENAI_API_BASE_URL=http://host\.containers\.internal:52625' \
@test "open-webui points its LLM at the live vLLM backend (52625) via the bridge gateway" {
# host.containers.internal is a slirp4netns-only name. On the bridge,
# the host's 0.0.0.0-bound services are reached via the gateway IP.
grep -qE '^Environment=OPENAI_API_BASE_URL=http://10\.89\.3\.1:52625' \
"$QDIR/open-webui.container" \
|| { echo "open-webui.container must set OPENAI_API_BASE_URL to the vLLM on 52625"; return 1; }
|| { echo "open-webui.container must set OPENAI_API_BASE_URL to the vLLM on the bridge gateway 10.89.3.1:52625"; return 1; }
}
@test "open-webui persists its data under /home/dank/Containers/open-webui/data" {