diff --git a/dankclaw/Caddyfile b/dankclaw/Caddyfile index 718184a..673e1de 100644 --- a/dankclaw/Caddyfile +++ b/dankclaw/Caddyfile @@ -43,7 +43,7 @@ # --- Homepage (password protected) --- dankclaw.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } root * /srv/homepage file_server @@ -61,7 +61,7 @@ dankserver.franksdank.cloud { # --- SearXNG (quadlet; behind the Gluetun tunnel, published 8081) --- search.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy 192.168.0.143:8081 } @@ -74,7 +74,7 @@ comfyui.franksdank.cloud { # --- OpenWebUI (quadlet on slirp4netns; published 3005) --- chat.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy 192.168.0.143:3005 } @@ -82,7 +82,7 @@ chat.franksdank.cloud { # --- Hermes --- hermes.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy hermes:8090 } @@ -95,7 +95,7 @@ gitea.franksdank.cloud { # --- Grafana --- grafana.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy grafana:3000 } @@ -103,7 +103,7 @@ grafana.franksdank.cloud { # --- Prometheus --- prometheus.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy prometheus:9090 } @@ -138,7 +138,7 @@ books.franksdank.cloud { # --- RomM --- romm.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy 192.168.0.235:8085 } @@ -146,7 +146,7 @@ romm.franksdank.cloud { # --- Copyparty --- files.franksdank.cloud { basicauth { - dank $2a$14$REPLACE_WITH_HASH + dank REPLACE_WITH_HASH } reverse_proxy 192.168.0.235:3923 } diff --git a/dankclaw/open-webui/quadlet/open-webui.container b/dankclaw/open-webui/quadlet/open-webui.container index f11f6dd..d401182 100644 --- a/dankclaw/open-webui/quadlet/open-webui.container +++ b/dankclaw/open-webui/quadlet/open-webui.container @@ -5,10 +5,18 @@ # # Chat UI for the LLM fleet. It runs on the CPU image (dankclaw is CPU-only, so # the :cuda image would waste a large pull for a GPU it cannot use) and points -# its OpenAI connection at the live vLLM backend on this host -# (host.containers.internal:52625, model frank-default-large). It is isolated on -# slirp4netns and publishes 3005 (3000 is Gitea's HTTP port). Data persists under +# its OpenAI connection at the live vLLM backend on this host via the bridge +# gateway (10.89.3.1:52625, model frank-default-large). It publishes 3005 +# (3000 is Gitea's HTTP port). Data persists under # /home/dank/Containers/open-webui/data. See docs/Dankclaw/OpenWebUI.md. +# +# Network history (2026-08-22): this unit used Network=slirp4netns. When the +# slirp4netns helper process died, the container kept running but lost its tap +# interface: a zombie that accepted TCP and reset every request, and chat +# served empty responses for ~2 days with no error anywhere. It now runs on a +# pinned bridge network (open-webui.network), which has no per-container +# helper to die. The healthcheck probes /health (app liveness) and requires a +# non-loopback interface (network liveness). [Unit] Description=open-webui chat UI container @@ -16,14 +24,22 @@ After=network.target [Container] ContainerName=open-webui -Network=slirp4netns +Network=open-webui.network Image=ghcr.io/open-webui/open-webui:main PublishPort=3005:8080 -Environment=OPENAI_API_BASE_URL=http://host.containers.internal:52625 +Environment=OPENAI_API_BASE_URL=http://10.89.3.1:52625 Environment=ANONYMIZED_TELEMETRY=False Environment=DO_NOT_TRACK=True Volume=/home/dank/Containers/open-webui/data:/app/backend/data +# App liveness (/health) AND network liveness (a non-loopback interface must +# exist; a dead tap leaves only "lo" behind). +HealthCmd=CMD-SHELL python -c 'import os,urllib.request,sys; urllib.request.urlopen("http://127.0.0.1:8080/health",timeout=5).read(); sys.exit(0 if any(i!="lo" for i in os.listdir("/sys/class/net")) else 1)' +HealthInterval=30s +HealthTimeout=10s +HealthRetries=3 +HealthStartPeriod=300s + [Service] TimeoutStartSec=900 Restart=always diff --git a/dankclaw/open-webui/quadlet/open-webui.network b/dankclaw/open-webui/quadlet/open-webui.network new file mode 100644 index 0000000..3e272ad --- /dev/null +++ b/dankclaw/open-webui/quadlet/open-webui.network @@ -0,0 +1,29 @@ +# Quadlet network: private bridge for open-webui (rootless). +# +# Install path on dankclaw: ~/.config/containers/systemd/open-webui.network +# Unit name: open-webui-network.service (podman network name: systemd-open-webui) +# +# Why a bridge instead of slirp4netns (2026-08-22 incident): +# slirp4netns runs a user-space helper process per container. When that +# helper died, the container kept running but lost its tap interface, +# becoming a zombie that accepted TCP and reset every request. chat +# served empty responses for ~2 days and nothing logged an error. +# A kernel bridge has no per-container helper, so the failure mode +# cannot occur. +# +# The subnet is pinned so the gateway IP is deterministic: the LLM +# backend (llama-server :52625, bound 0.0.0.0 on the host) is reached +# from the container via the gateway. host.containers.internal is a +# slirp4netns-only name and does not resolve on a bridge. +# +# Subnet registry on dankclaw (rootless): +# 10.88.0.0/16 default "podman" bridge +# 10.89.0.0/24 gitea (systemd-gitea) +# 10.89.1.0/24 compose stack (dankclaw_default) +# 10.89.2.0/24 searxng (systemd-searxng) +# 10.89.3.0/24 open-webui (systemd-open-webui) <- this file + +[Network] +Driver=bridge +Subnet=10.89.3.0/24 +Gateway=10.89.3.1 \ No newline at end of file diff --git a/docs/Dankclaw/OpenWebUI.md b/docs/Dankclaw/OpenWebUI.md index b2c1003..83aed1e 100644 --- a/docs/Dankclaw/OpenWebUI.md +++ b/docs/Dankclaw/OpenWebUI.md @@ -11,8 +11,8 @@ OpenWebUI is a self-hosted chat interface for LLMs. It connects to OpenAI-compat | Image | `ghcr.io/open-webui/open-webui:main` (CPU build; Dankclaw is CPU-only) | | Port | 3005 (host) → 8080 (container). Port 3000 is Gitea's HTTP, so it is not usable. | | Data volume | `/home/dank/Containers/open-webui/data` → `/app/backend/data` | -| Default backend | vLLM on Dankclaw `:52625` = `frank-default-large` (via `host.containers.internal:52625`) | -| Network | `slirp4netns` (isolated) | +| Default backend | vLLM on Dankclaw `:52625` = `frank-default-large` (via bridge gateway `10.89.3.1:52625`) | +| Network | Pinned bridge `systemd-open-webui` (`10.89.3.0/24`, gateway `10.89.3.1`) — see [History](#history) | | Persistence | Quadlet unit `open-webui.service`; `[Install] WantedBy=default.target` | | Deploy | `scripts/deploy-open-webui-quadlet.sh dankclaw` | @@ -43,7 +43,12 @@ The data dir holds the database, RAG indexes, and uploads. The deploy script cre ## Healthcheck -OpenWebUI responds on `/health` with 200. Probe it at `http://dankclaw:3005/health`. +The quadlet carries a built-in healthcheck (30s interval, 3 retries, 300s start period). It checks two things from inside the container: + +1. **App liveness** — `GET http://127.0.0.1:8080/health` returns 200 (`{"status":true}`). +2. **Network liveness** — a non-loopback interface exists in `/sys/class/net`. The dead-tap zombie state (History) leaves only `lo` behind, and this is the check that catches it. + +`podman ps` shows the container as `unhealthy` if either fails. Probe manually at `http://dankclaw:3005/health`. ## Retired instances @@ -56,7 +61,11 @@ OpenWebUI responds on `/health` with 200. Probe it at `http://dankclaw:3005/heal - Use the non-CUDA image on this CPU-only host. - The default admin account is created on first login; secure it immediately. - If the backend is loading (3–8 minute model load), OpenWebUI shows connection errors until the model is ready. -- `host.containers.internal` reaches only the podman host. To reach the Dankserver backend, use the LAN address `192.168.0.235:7770`. +- The LLM is reached via the bridge gateway `10.89.3.1`, not `host.containers.internal` (that name is slirp4netns-only and does not resolve on a bridge). To reach the Dankserver backend, use the LAN address `192.168.0.235:7770`. + +## History + +- **2026-08-22 — slirp4netns → pinned bridge.** The unit ran on `Network=slirp4netns`. The slirp4netns helper process died (no OOM, no kill entry, no crash log found), and the container kept running with only `lo` in its netns: published port 3005 accepted TCP and reset every request, so chat served empty responses for ~2 days. Nothing logged an error. Moving to a kernel bridge removes the per-container helper entirely, and the healthcheck's interface check makes the zombie state visible if it ever returns. ## See Also diff --git a/docs/Infrastructure/Caddy.md b/docs/Infrastructure/Caddy.md index 57199c8..87feebb 100644 --- a/docs/Infrastructure/Caddy.md +++ b/docs/Infrastructure/Caddy.md @@ -50,17 +50,19 @@ The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/ ## Basic auth -Generate a bcrypt hash, then reference it: +The Caddyfile in git carries a **bare** placeholder; deploy.sh substitutes the **full** hash from the host `.env` (`CADDY_BASICAUTH_HASH`): ``` chat.franksdank.cloud { basicauth { - dank $2a$14$xxxxxxxxxxxxxxxxxxxxxxxxxxxxx + dank REPLACE_WITH_HASH } reverse_proxy 192.168.0.143:3005 } ``` +After rendering, the line reads `dank $2a$14$xxxx…` (one complete hash). Do not put a `$2a$14$` prefix in the template: it would render a double-prefixed, invalid hash (2026-08-22 incident — every basicauth vhost returned 401 for every password). `tests/deployment_flow.bats` pins the render contract. + The homepage also uses basic auth; see [[Homepage]]. ## Operations @@ -75,6 +77,7 @@ Certificates and ACME state live in the caddy data volume. Back it up with the s ## Gotchas +- The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged. - A new vhost needs three things: a dnsmasq `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]]. - The G36 has no NAT loopback, so LAN clients must resolve the name to the LAN IP. That is the dnsmasq role in [[LAN-DNS]]. - Caddy serves HTTPS on 443 and HTTP on 80 (redirects). The router needs no 80/443 port forward for LAN use. diff --git a/scripts/deploy-open-webui-quadlet.sh b/scripts/deploy-open-webui-quadlet.sh index 794c6a3..d430f78 100755 --- a/scripts/deploy-open-webui-quadlet.sh +++ b/scripts/deploy-open-webui-quadlet.sh @@ -4,7 +4,7 @@ # open-webui is boot-persistent via Quadlet (see AGENTS.md). This script: # 1. Pushes main to origin # 2. Pulls the repo on the target -# 3. Creates the data dir and copies the unit into ~/.config/containers/systemd/ +# 3. Creates the data dir and copies the unit + network into ~/.config/containers/systemd/ # 4. daemon-reloads and starts open-webui (first run pulls the image) # # It is idempotent: it stops any running unit and removes any container with the @@ -75,9 +75,10 @@ podman rm -f open-webui 2>/dev/null || true echo ' Creating the data dir (preserved across reruns)...' mkdir -p '${DATA_DIR}' -echo ' Copying the quadlet unit into ~/.config/containers/systemd/ ...' +echo ' Copying the quadlet unit + network into ~/.config/containers/systemd/ ...' mkdir -p ~/.config/containers/systemd cp ${QDIR}/open-webui.container ~/.config/containers/systemd/ +cp ${QDIR}/open-webui.network ~/.config/containers/systemd/ echo ' daemon-reload (generates the unit)...' # Quadlet units are generator-managed: systemctl enable fails on them. The diff --git a/tests/caddyfile.bats b/tests/caddyfile.bats index a206cf3..f33e909 100644 --- a/tests/caddyfile.bats +++ b/tests/caddyfile.bats @@ -49,6 +49,19 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile" grep -q 'REPLACE_WITH_HASH' "$CADDYFILE" } +@test "the basicauth placeholder is a bare token (no bcrypt prefix in the template)" { + # Regression (2026-08-22): the template line was + # dank $2a$14$REPLACE_WITH_HASH + # and deploy.sh substitutes the FULL hash (prefix included), so the + # rendered line became an invalid double-prefixed hash and every + # basicauth vhost rejected every password with 401. + if grep -qE '\$2[abxy]\$[0-9]+\$REPLACE_WITH_HASH' "$CADDYFILE"; then + echo "Caddyfile: bcrypt prefix before REPLACE_WITH_HASH (double-prefix regression)" + return 1 + fi + grep -qE 'dank REPLACE_WITH_HASH' "$CADDYFILE" +} + @test "compose no longer defines the quadlet-replaced services" { # gitea, its postgres, and open-webui run as boot-persistent quadlets on # dankclaw (see dankclaw/gitea/quadlet, dankclaw/open-webui/quadlet). diff --git a/tests/deployment_flow.bats b/tests/deployment_flow.bats index 96f429c..21c6822 100644 --- a/tests/deployment_flow.bats +++ b/tests/deployment_flow.bats @@ -44,4 +44,9 @@ load helpers [ -n "$rendered" ] [[ "$rendered" == *"${hash}"* ]] ! grep -q 'REPLACE_WITH_HASH' <<< "$rendered" + # The rendered line must be exactly "dank ". A template that keeps + # its own $2a$14$ prefix renders an invalid double-prefixed hash and + # every basicauth vhost returns 401 (2026-08-22 incident). + grep -qF "dank ${hash}" <<< "$rendered" + ! grep -qF '$2a$14$$2a$14$' <<< "$rendered" } diff --git a/tests/open_webui_quadlet.bats b/tests/open_webui_quadlet.bats index 9df3f9f..ceb22d8 100644 --- a/tests/open_webui_quadlet.bats +++ b/tests/open_webui_quadlet.bats @@ -24,9 +24,33 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet" return 0 } -@test "open-webui is isolated on slirp4netns" { - grep -qE '^Network=slirp4netns' "$QDIR/open-webui.container" \ - || { echo "open-webui.container must set Network=slirp4netns"; return 1; } +@test "open-webui is on a private bridge network (not slirp4netns)" { + # slirp4netns runs a user-space helper per container. When the helper + # dies, the container keeps running but loses its tap interface: a + # zombie that accepts TCP and resets every request (2026-08-22: chat + # returned empty responses for ~2 days). A bridge network has no + # per-container helper, so the failure mode cannot occur. + grep -qE '^Network=open-webui\.network' "$QDIR/open-webui.container" \ + || { echo "open-webui.container must set Network=open-webui.network"; return 1; } + ! grep -qE '^Network=slirp4netns' "$QDIR/open-webui.container" + [ -f "$QDIR/open-webui.network" ] +} + +@test "open-webui network pins a deterministic subnet and gateway" { + # The LLM endpoint is reached via the bridge gateway, so the subnet + # must be pinned (an auto-assigned subnet is not a promise). + grep -qE '^Subnet=10\.89\.3\.0/24' "$QDIR/open-webui.network" + grep -qE '^Gateway=10\.89\.3\.1' "$QDIR/open-webui.network" +} + +@test "open-webui has a healthcheck that sees the app AND the network" { + # The /health probe catches app hangs; the non-loopback interface check + # catches the dead-tap zombie state (only "lo" left in the netns). + grep -qE '^HealthCmd=CMD-SHELL .*/health.*' "$QDIR/open-webui.container" + grep -qE '^HealthCmd=.*sys/class/net.*' "$QDIR/open-webui.container" + grep -qE '^HealthInterval=30s' "$QDIR/open-webui.container" + grep -qE '^HealthRetries=3' "$QDIR/open-webui.container" + grep -qE '^HealthStartPeriod=' "$QDIR/open-webui.container" } @test "open-webui publishes 3005:8080 (a free port; 3000 is Gitea)" { @@ -34,10 +58,12 @@ QDIR="$REPO_ROOT/dankclaw/open-webui/quadlet" || { echo "open-webui.container must publish 3005:8080"; return 1; } } -@test "open-webui points its LLM at the live vLLM backend (52625)" { - grep -qE '^Environment=OPENAI_API_BASE_URL=http://host\.containers\.internal:52625' \ +@test "open-webui points its LLM at the live vLLM backend (52625) via the bridge gateway" { + # host.containers.internal is a slirp4netns-only name. On the bridge, + # the host's 0.0.0.0-bound services are reached via the gateway IP. + grep -qE '^Environment=OPENAI_API_BASE_URL=http://10\.89\.3\.1:52625' \ "$QDIR/open-webui.container" \ - || { echo "open-webui.container must set OPENAI_API_BASE_URL to the vLLM on 52625"; return 1; } + || { echo "open-webui.container must set OPENAI_API_BASE_URL to the vLLM on the bridge gateway 10.89.3.1:52625"; return 1; } } @test "open-webui persists its data under /home/dank/Containers/open-webui/data" {