fix: basicauth double-prefix render; open-webui off slirp4netns
basicauth (2026-08-22 incident): - the Caddyfile template carried its own $2a$14$ prefix and deploy.sh substituted the full hash (prefix included), rendering an invalid double-prefixed hash. Every basicauth vhost rejected every password with 401 and Caddy logged nothing. - template is now a bare REPLACE_WITH_HASH token; tests pin the render contract (no prefix in template; rendered line is exactly 'dank <hash>'). open-webui (2026-08-22 incident): - its slirp4netns helper process died, leaving the container running with only 'lo' in its netns: port 3005 accepted TCP and reset every request, so chat served empty responses for ~2 days, unlogged. - move to a pinned bridge network (systemd-open-webui, 10.89.3.0/24, gw 10.89.3.1) — no per-container helper to die. - LLM endpoint reached via the gateway (host.containers.internal is slirp-only): OPENAI_API_BASE_URL=http://10.89.3.1:52625. - add a quadlet healthcheck: /health (app) + non-loopback interface (network liveness; the dead-tap state leaves only 'lo'). - deploy script now copies the .network file too. - docs updated (OpenWebUI History section, Caddy basic-auth contract).
This commit is contained in:
Frank
+8
-8
@@ -43,7 +43,7 @@
|
||||
# --- Homepage (password protected) ---
|
||||
dankclaw.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
root * /srv/homepage
|
||||
file_server
|
||||
@@ -61,7 +61,7 @@ dankserver.franksdank.cloud {
|
||||
# --- SearXNG (quadlet; behind the Gluetun tunnel, published 8081) ---
|
||||
search.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy 192.168.0.143:8081
|
||||
}
|
||||
@@ -74,7 +74,7 @@ comfyui.franksdank.cloud {
|
||||
# --- OpenWebUI (quadlet on slirp4netns; published 3005) ---
|
||||
chat.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy 192.168.0.143:3005
|
||||
}
|
||||
@@ -82,7 +82,7 @@ chat.franksdank.cloud {
|
||||
# --- Hermes ---
|
||||
hermes.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy hermes:8090
|
||||
}
|
||||
@@ -95,7 +95,7 @@ gitea.franksdank.cloud {
|
||||
# --- Grafana ---
|
||||
grafana.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy grafana:3000
|
||||
}
|
||||
@@ -103,7 +103,7 @@ grafana.franksdank.cloud {
|
||||
# --- Prometheus ---
|
||||
prometheus.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy prometheus:9090
|
||||
}
|
||||
@@ -138,7 +138,7 @@ books.franksdank.cloud {
|
||||
# --- RomM ---
|
||||
romm.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy 192.168.0.235:8085
|
||||
}
|
||||
@@ -146,7 +146,7 @@ romm.franksdank.cloud {
|
||||
# --- Copyparty ---
|
||||
files.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$REPLACE_WITH_HASH
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy 192.168.0.235:3923
|
||||
}
|
||||
|
||||
@@ -5,10 +5,18 @@
|
||||
#
|
||||
# Chat UI for the LLM fleet. It runs on the CPU image (dankclaw is CPU-only, so
|
||||
# the :cuda image would waste a large pull for a GPU it cannot use) and points
|
||||
# its OpenAI connection at the live vLLM backend on this host
|
||||
# (host.containers.internal:52625, model frank-default-large). It is isolated on
|
||||
# slirp4netns and publishes 3005 (3000 is Gitea's HTTP port). Data persists under
|
||||
# its OpenAI connection at the live vLLM backend on this host via the bridge
|
||||
# gateway (10.89.3.1:52625, model frank-default-large). It publishes 3005
|
||||
# (3000 is Gitea's HTTP port). Data persists under
|
||||
# /home/dank/Containers/open-webui/data. See docs/Dankclaw/OpenWebUI.md.
|
||||
#
|
||||
# Network history (2026-08-22): this unit used Network=slirp4netns. When the
|
||||
# slirp4netns helper process died, the container kept running but lost its tap
|
||||
# interface: a zombie that accepted TCP and reset every request, and chat
|
||||
# served empty responses for ~2 days with no error anywhere. It now runs on a
|
||||
# pinned bridge network (open-webui.network), which has no per-container
|
||||
# helper to die. The healthcheck probes /health (app liveness) and requires a
|
||||
# non-loopback interface (network liveness).
|
||||
|
||||
[Unit]
|
||||
Description=open-webui chat UI container
|
||||
@@ -16,14 +24,22 @@ After=network.target
|
||||
|
||||
[Container]
|
||||
ContainerName=open-webui
|
||||
Network=slirp4netns
|
||||
Network=open-webui.network
|
||||
Image=ghcr.io/open-webui/open-webui:main
|
||||
PublishPort=3005:8080
|
||||
Environment=OPENAI_API_BASE_URL=http://host.containers.internal:52625
|
||||
Environment=OPENAI_API_BASE_URL=http://10.89.3.1:52625
|
||||
Environment=ANONYMIZED_TELEMETRY=False
|
||||
Environment=DO_NOT_TRACK=True
|
||||
Volume=/home/dank/Containers/open-webui/data:/app/backend/data
|
||||
|
||||
# App liveness (/health) AND network liveness (a non-loopback interface must
|
||||
# exist; a dead tap leaves only "lo" behind).
|
||||
HealthCmd=CMD-SHELL python -c 'import os,urllib.request,sys; urllib.request.urlopen("http://127.0.0.1:8080/health",timeout=5).read(); sys.exit(0 if any(i!="lo" for i in os.listdir("/sys/class/net")) else 1)'
|
||||
HealthInterval=30s
|
||||
HealthTimeout=10s
|
||||
HealthRetries=3
|
||||
HealthStartPeriod=300s
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=900
|
||||
Restart=always
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# Quadlet network: private bridge for open-webui (rootless).
|
||||
#
|
||||
# Install path on dankclaw: ~/.config/containers/systemd/open-webui.network
|
||||
# Unit name: open-webui-network.service (podman network name: systemd-open-webui)
|
||||
#
|
||||
# Why a bridge instead of slirp4netns (2026-08-22 incident):
|
||||
# slirp4netns runs a user-space helper process per container. When that
|
||||
# helper died, the container kept running but lost its tap interface,
|
||||
# becoming a zombie that accepted TCP and reset every request. chat
|
||||
# served empty responses for ~2 days and nothing logged an error.
|
||||
# A kernel bridge has no per-container helper, so the failure mode
|
||||
# cannot occur.
|
||||
#
|
||||
# The subnet is pinned so the gateway IP is deterministic: the LLM
|
||||
# backend (llama-server :52625, bound 0.0.0.0 on the host) is reached
|
||||
# from the container via the gateway. host.containers.internal is a
|
||||
# slirp4netns-only name and does not resolve on a bridge.
|
||||
#
|
||||
# Subnet registry on dankclaw (rootless):
|
||||
# 10.88.0.0/16 default "podman" bridge
|
||||
# 10.89.0.0/24 gitea (systemd-gitea)
|
||||
# 10.89.1.0/24 compose stack (dankclaw_default)
|
||||
# 10.89.2.0/24 searxng (systemd-searxng)
|
||||
# 10.89.3.0/24 open-webui (systemd-open-webui) <- this file
|
||||
|
||||
[Network]
|
||||
Driver=bridge
|
||||
Subnet=10.89.3.0/24
|
||||
Gateway=10.89.3.1
|
||||
@@ -11,8 +11,8 @@ OpenWebUI is a self-hosted chat interface for LLMs. It connects to OpenAI-compat
|
||||
| Image | `ghcr.io/open-webui/open-webui:main` (CPU build; Dankclaw is CPU-only) |
|
||||
| Port | 3005 (host) → 8080 (container). Port 3000 is Gitea's HTTP, so it is not usable. |
|
||||
| Data volume | `/home/dank/Containers/open-webui/data` → `/app/backend/data` |
|
||||
| Default backend | vLLM on Dankclaw `:52625` = `frank-default-large` (via `host.containers.internal:52625`) |
|
||||
| Network | `slirp4netns` (isolated) |
|
||||
| Default backend | vLLM on Dankclaw `:52625` = `frank-default-large` (via bridge gateway `10.89.3.1:52625`) |
|
||||
| Network | Pinned bridge `systemd-open-webui` (`10.89.3.0/24`, gateway `10.89.3.1`) — see [History](#history) |
|
||||
| Persistence | Quadlet unit `open-webui.service`; `[Install] WantedBy=default.target` |
|
||||
| Deploy | `scripts/deploy-open-webui-quadlet.sh dankclaw` |
|
||||
|
||||
@@ -43,7 +43,12 @@ The data dir holds the database, RAG indexes, and uploads. The deploy script cre
|
||||
|
||||
## Healthcheck
|
||||
|
||||
OpenWebUI responds on `/health` with 200. Probe it at `http://dankclaw:3005/health`.
|
||||
The quadlet carries a built-in healthcheck (30s interval, 3 retries, 300s start period). It checks two things from inside the container:
|
||||
|
||||
1. **App liveness** — `GET http://127.0.0.1:8080/health` returns 200 (`{"status":true}`).
|
||||
2. **Network liveness** — a non-loopback interface exists in `/sys/class/net`. The dead-tap zombie state (History) leaves only `lo` behind, and this is the check that catches it.
|
||||
|
||||
`podman ps` shows the container as `unhealthy` if either fails. Probe manually at `http://dankclaw:3005/health`.
|
||||
|
||||
## Retired instances
|
||||
|
||||
@@ -56,7 +61,11 @@ OpenWebUI responds on `/health` with 200. Probe it at `http://dankclaw:3005/heal
|
||||
- Use the non-CUDA image on this CPU-only host.
|
||||
- The default admin account is created on first login; secure it immediately.
|
||||
- If the backend is loading (3–8 minute model load), OpenWebUI shows connection errors until the model is ready.
|
||||
- `host.containers.internal` reaches only the podman host. To reach the Dankserver backend, use the LAN address `192.168.0.235:7770`.
|
||||
- The LLM is reached via the bridge gateway `10.89.3.1`, not `host.containers.internal` (that name is slirp4netns-only and does not resolve on a bridge). To reach the Dankserver backend, use the LAN address `192.168.0.235:7770`.
|
||||
|
||||
## History
|
||||
|
||||
- **2026-08-22 — slirp4netns → pinned bridge.** The unit ran on `Network=slirp4netns`. The slirp4netns helper process died (no OOM, no kill entry, no crash log found), and the container kept running with only `lo` in its netns: published port 3005 accepted TCP and reset every request, so chat served empty responses for ~2 days. Nothing logged an error. Moving to a kernel bridge removes the per-container helper entirely, and the healthcheck's interface check makes the zombie state visible if it ever returns.
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
@@ -50,17 +50,19 @@ The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/
|
||||
|
||||
## Basic auth
|
||||
|
||||
Generate a bcrypt hash, then reference it:
|
||||
The Caddyfile in git carries a **bare** placeholder; deploy.sh substitutes the **full** hash from the host `.env` (`CADDY_BASICAUTH_HASH`):
|
||||
|
||||
```
|
||||
chat.franksdank.cloud {
|
||||
basicauth {
|
||||
dank $2a$14$xxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
dank REPLACE_WITH_HASH
|
||||
}
|
||||
reverse_proxy 192.168.0.143:3005
|
||||
}
|
||||
```
|
||||
|
||||
After rendering, the line reads `dank $2a$14$xxxx…` (one complete hash). Do not put a `$2a$14$` prefix in the template: it would render a double-prefixed, invalid hash (2026-08-22 incident — every basicauth vhost returned 401 for every password). `tests/deployment_flow.bats` pins the render contract.
|
||||
|
||||
The homepage also uses basic auth; see [[Homepage]].
|
||||
|
||||
## Operations
|
||||
@@ -75,6 +77,7 @@ Certificates and ACME state live in the caddy data volume. Back it up with the s
|
||||
|
||||
## Gotchas
|
||||
|
||||
- The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged.
|
||||
- A new vhost needs three things: a dnsmasq `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]].
|
||||
- The G36 has no NAT loopback, so LAN clients must resolve the name to the LAN IP. That is the dnsmasq role in [[LAN-DNS]].
|
||||
- Caddy serves HTTPS on 443 and HTTP on 80 (redirects). The router needs no 80/443 port forward for LAN use.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# open-webui is boot-persistent via Quadlet (see AGENTS.md). This script:
|
||||
# 1. Pushes main to origin
|
||||
# 2. Pulls the repo on the target
|
||||
# 3. Creates the data dir and copies the unit into ~/.config/containers/systemd/
|
||||
# 3. Creates the data dir and copies the unit + network into ~/.config/containers/systemd/
|
||||
# 4. daemon-reloads and starts open-webui (first run pulls the image)
|
||||
#
|
||||
# It is idempotent: it stops any running unit and removes any container with the
|
||||
@@ -75,9 +75,10 @@ podman rm -f open-webui 2>/dev/null || true
|
||||
echo ' Creating the data dir (preserved across reruns)...'
|
||||
mkdir -p '${DATA_DIR}'
|
||||
|
||||
echo ' Copying the quadlet unit into ~/.config/containers/systemd/ ...'
|
||||
echo ' Copying the quadlet unit + network into ~/.config/containers/systemd/ ...'
|
||||
mkdir -p ~/.config/containers/systemd
|
||||
cp ${QDIR}/open-webui.container ~/.config/containers/systemd/
|
||||
cp ${QDIR}/open-webui.network ~/.config/containers/systemd/
|
||||
|
||||
echo ' daemon-reload (generates the unit)...'
|
||||
# Quadlet units are generator-managed: systemctl enable fails on them. The
|
||||
|
||||
@@ -49,6 +49,19 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
|
||||
grep -q 'REPLACE_WITH_HASH' "$CADDYFILE"
|
||||
}
|
||||
|
||||
@test "the basicauth placeholder is a bare token (no bcrypt prefix in the template)" {
|
||||
# Regression (2026-08-22): the template line was
|
||||
# dank $2a$14$REPLACE_WITH_HASH
|
||||
# and deploy.sh substitutes the FULL hash (prefix included), so the
|
||||
# rendered line became an invalid double-prefixed hash and every
|
||||
# basicauth vhost rejected every password with 401.
|
||||
if grep -qE '\$2[abxy]\$[0-9]+\$REPLACE_WITH_HASH' "$CADDYFILE"; then
|
||||
echo "Caddyfile: bcrypt prefix before REPLACE_WITH_HASH (double-prefix regression)"
|
||||
return 1
|
||||
fi
|
||||
grep -qE 'dank REPLACE_WITH_HASH' "$CADDYFILE"
|
||||
}
|
||||
|
||||
@test "compose no longer defines the quadlet-replaced services" {
|
||||
# gitea, its postgres, and open-webui run as boot-persistent quadlets on
|
||||
# dankclaw (see dankclaw/gitea/quadlet, dankclaw/open-webui/quadlet).
|
||||
|
||||
@@ -44,4 +44,9 @@ load helpers
|
||||
[ -n "$rendered" ]
|
||||
[[ "$rendered" == *"${hash}"* ]]
|
||||
! grep -q 'REPLACE_WITH_HASH' <<< "$rendered"
|
||||
# The rendered line must be exactly "dank <hash>". A template that keeps
|
||||
# its own $2a$14$ prefix renders an invalid double-prefixed hash and
|
||||
# every basicauth vhost returns 401 (2026-08-22 incident).
|
||||
grep -qF "dank ${hash}" <<< "$rendered"
|
||||
! grep -qF '$2a$14$$2a$14$' <<< "$rendered"
|
||||
}
|
||||
|
||||
@@ -24,9 +24,33 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet"
|
||||
return 0
|
||||
}
|
||||
|
||||
@test "open-webui is isolated on slirp4netns" {
|
||||
grep -qE '^Network=slirp4netns' "$QDIR/open-webui.container" \
|
||||
|| { echo "open-webui.container must set Network=slirp4netns"; return 1; }
|
||||
@test "open-webui is on a private bridge network (not slirp4netns)" {
|
||||
# slirp4netns runs a user-space helper per container. When the helper
|
||||
# dies, the container keeps running but loses its tap interface: a
|
||||
# zombie that accepts TCP and resets every request (2026-08-22: chat
|
||||
# returned empty responses for ~2 days). A bridge network has no
|
||||
# per-container helper, so the failure mode cannot occur.
|
||||
grep -qE '^Network=open-webui\.network' "$QDIR/open-webui.container" \
|
||||
|| { echo "open-webui.container must set Network=open-webui.network"; return 1; }
|
||||
! grep -qE '^Network=slirp4netns' "$QDIR/open-webui.container"
|
||||
[ -f "$QDIR/open-webui.network" ]
|
||||
}
|
||||
|
||||
@test "open-webui network pins a deterministic subnet and gateway" {
|
||||
# The LLM endpoint is reached via the bridge gateway, so the subnet
|
||||
# must be pinned (an auto-assigned subnet is not a promise).
|
||||
grep -qE '^Subnet=10\.89\.3\.0/24' "$QDIR/open-webui.network"
|
||||
grep -qE '^Gateway=10\.89\.3\.1' "$QDIR/open-webui.network"
|
||||
}
|
||||
|
||||
@test "open-webui has a healthcheck that sees the app AND the network" {
|
||||
# The /health probe catches app hangs; the non-loopback interface check
|
||||
# catches the dead-tap zombie state (only "lo" left in the netns).
|
||||
grep -qE '^HealthCmd=CMD-SHELL .*/health.*' "$QDIR/open-webui.container"
|
||||
grep -qE '^HealthCmd=.*sys/class/net.*' "$QDIR/open-webui.container"
|
||||
grep -qE '^HealthInterval=30s' "$QDIR/open-webui.container"
|
||||
grep -qE '^HealthRetries=3' "$QDIR/open-webui.container"
|
||||
grep -qE '^HealthStartPeriod=' "$QDIR/open-webui.container"
|
||||
}
|
||||
|
||||
@test "open-webui publishes 3005:8080 (a free port; 3000 is Gitea)" {
|
||||
@@ -34,10 +58,12 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet"
|
||||
|| { echo "open-webui.container must publish 3005:8080"; return 1; }
|
||||
}
|
||||
|
||||
@test "open-webui points its LLM at the live vLLM backend (52625)" {
|
||||
grep -qE '^Environment=OPENAI_API_BASE_URL=http://host\.containers\.internal:52625' \
|
||||
@test "open-webui points its LLM at the live vLLM backend (52625) via the bridge gateway" {
|
||||
# host.containers.internal is a slirp4netns-only name. On the bridge,
|
||||
# the host's 0.0.0.0-bound services are reached via the gateway IP.
|
||||
grep -qE '^Environment=OPENAI_API_BASE_URL=http://10\.89\.3\.1:52625' \
|
||||
"$QDIR/open-webui.container" \
|
||||
|| { echo "open-webui.container must set OPENAI_API_BASE_URL to the vLLM on 52625"; return 1; }
|
||||
|| { echo "open-webui.container must set OPENAI_API_BASE_URL to the vLLM on the bridge gateway 10.89.3.1:52625"; return 1; }
|
||||
}
|
||||
|
||||
@test "open-webui persists its data under /home/dank/Containers/open-webui/data" {
|
||||
|
||||
Reference in New Issue
Block a user