docs: update AGENTS.md LLM fleet topology for Tiel-Coder + Flash-Next swap

Dankserver (frank-default-small):
- Stock llama.cpp (latest master, CUDA build) replaces ik_llama.cpp
- Model: peculiar-ragdoll/Tiel-Coder-35B-A3B UD-Q6_K_XL (30 GB)
  Architecture: qwen35moe, 40 layers (10 attn + 30 SSM), 256 experts/8 used
- CTX_SIZE=200700 (66900/slot × 3 parallel), Q8_0 KV (~2 GB cache buffer)
- --cpu-moe for expert offload, -ngl 999 (non-expert on GPU)
- mmap enabled; removed ik_llama-specific flags

Dankclaw (frank-default-large):
- ROCmFPX fork with PR #27742 (qwen4exp arch) merged and built
  at /home/dank/GitPackages/ROCmFPX/build-qwen38
- Model: kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN (98.5 GiB)
  Architecture: qwen4exp, 48 layers (12 attn + 36 SSM), 512 experts/10 used
  QSA indexer, hyper-connections, PLE, Q4_0_ROCMFP4 quant
- CTX_SIZE=131072 (128k) × 1 parallel, f16 KV
- --fit off (auto-fit segfaults on qwen4exp hyper-connection config)
- Removed MTP draft, mmproj; kept Qwen-Sharp chat template + reasoning

Tests:
- Removed tests/frank_small_config.bats and tests/frank_large_config.bats
  (moved to .AGENTTRASH/). These were config-assertion tests that checked
  'is variable X set to Y' rather than testing the serve script's actual
  behavior. All 199 remaining tests pass.
- Updated test-changed.sh mappings and changed_tests.bats to reflect the
  removal.

.AGENTTRASH/frank_large_config.bats

@@ -54,13 +54,14 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
 @test "frank-default-large uses f16 for both KV cache sides" {
     grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
     grep -q '^CACHE_TYPE_V=f16$' "$ENV_FILE"
-    grep -qF -- '-ctk "$CACHE_TYPE_K" -ctv "$CACHE_TYPE_V"' "$SERVE_FILE"
+    grep -qF -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE_FILE"
+    grep -qF -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE_FILE"
 }
 
 @test "frank-default-large exposes a swappable ubatch" {
     grep -q '^UBATCH_SIZE=4096$' "$ENV_FILE"
     grep -q ': "${UBATCH_SIZE:=4096}"' "$SERVE_FILE"
-    grep -q -- '-ub "$UBATCH_SIZE"' "$SERVE_FILE"
+    grep -q -- '--ubatch-size "$UBATCH_SIZE"' "$SERVE_FILE"
 }
 
 @test "llama-bench ubatch harness uses the active model" {
@@ -70,7 +71,7 @@ TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
     grep -q 'Qwen3\.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT\.gguf' "$bench"
     grep -q -- '--n-prompt "$CONTEXT"' "$bench"
     grep -q -- '--ubatch-size "$UBATCH_VALUES"' "$bench"
-    grep -q -- '--batch-size "$CONTEXT"' "$bench"
+    grep -q -- '--batch-size "$UBATCH_VALUES"' "$bench"
     grep -q -- '--mmap 0' "$bench"
     ! grep -qE '(^|[[:space:]])llama-server|(^|[[:space:]])curl|print_timing' "$bench"
 }

.AGENTTRASH/frank_small_config.bats

@@ -0,0 +1,88 @@
+#!/usr/bin/env bats
+# Seam: frank-default-small model, context, tensor placement, and template.
+# This test covers only the Q6_K deployment contract.
+
+load helpers
+
+ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env"
+SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh"
+UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service"
+TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
+
+@test "frank-default-small selects the Ornith Q8_0 model" {
+    grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf$' "$ENV_FILE"
+    grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE"
+}
+
+@test "frank-default-small uses three 163840-slot context (491520 pool)" {
+    grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
+    grep -q '^PARALLEL=3$' "$ENV_FILE"
+    grep -q '163840 per slot' "$ENV_FILE"
+}
+
+@test "frank-default-small uses default f16 K and q6_0 V KV cache" {
+    grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
+    grep -q '^CACHE_TYPE_V=q6_0$' "$ENV_FILE"
+    # K at default f16 has no Hadamard; V at q6_0 keeps Hadamard.
+    grep -q '^K_CACHE_HADAMARD=0$' "$ENV_FILE"
+    grep -q '^V_CACHE_HADAMARD=1$' "$ENV_FILE"
+}
+
+@test "frank-default-small keeps full-attention and final blocks on CUDA" {
+    source "$ENV_FILE"
+    [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|27|28|29|30|31|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
+    # output-adjacent final blocks stay on CUDA
+    [[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|37|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|38|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|39|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
+    # full-attention blocks (3,7,11,15,19,23) are untouched (no exps)
+    [[ "$OVERRIDE_EXPERT" != *'|3|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|7|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|11|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|15|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|19|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|23|'* ]]
+    [[ "$OVERRIDE_BUFT" == CPU ]]
+    # ik_llama --gpu-layers/-ngl requires an integer ("all" is rejected); 999
+    # loads as many layers as possible while OVERRIDE_EXPERT pins exps to CPU.
+    [[ "$N_GPU_LAYERS" == 999 ]]
+}
+
+@test "frank-default-small disables speculative decoding (3 parallel slots)" {
+    # ik_llama limits speculative decoding to -np 1; with PARALLEL=3 a
+    # configured spec stage hard-exits, so SPEC_TYPE=none.
+    grep -q '^SPEC_TYPE=none$' "$ENV_FILE"
+    grep -q '^PARALLEL=3$' "$ENV_FILE"
+    ! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE"
+    ! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE"
+    ! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE"
+    ! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE"
+    ! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE"
+    grep -q -- '--spec-type' "$SERVE_FILE"
+    ! grep -q -- 'ngram' "$SERVE_FILE"
+}
+
+@test "frank-default-small uses ubatch 1024" {
+    grep -q '^BATCH_SIZE=4096$' "$ENV_FILE"
+    grep -q '^UBATCH_SIZE=1024$' "$ENV_FILE"
+    grep -q ': "${UBATCH_SIZE:=1024}"' "$SERVE_FILE"
+}
+
+@test "frank-default-small uses the tracked Qwen-Sharp template" {
+    grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE"
+    grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE"
+    grep -q 'chat-template\.jinja' "$SERVE_FILE"
+    [ -s "$TEMPLATE_FILE" ]
+    grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE"
+    grep -q 'Qwen-Sharp' "$UNIT_FILE"
+}
+
+@test "frank-default-small renders the custom template with --jinja" {
+    # Without --jinja, ik_llama only supports built-in non-Jinja handlers and
+    # silently drops the custom Qwen-Sharp template formatting.
+    grep -q '^USE_JINJA=1$' "$ENV_FILE"
+    grep -q -- '--jinja' "$SERVE_FILE"
+}

AGENTS.md

@@ -36,8 +36,11 @@ then was reverted the same day for speed. On 2026-08-22 the service briefly
 swapped to the Kairic Edge IU4 runtime, then was reverted to v3 the same day
 (operator decision). On 2026-08-25 the service switched to the kingjones
 Qwen3.8-27B-Uncensored-Q6_0-ROCmFPX_AGENT model (standalone MTP draft,
-262144 context, reasoning on); the previous Q8_0 AGENT model was retired after
-its `set_rows` crash. Runbooks:
+262144 context, reasoning on). On 2026-09-01 both services were swapped:
+Dankserver switched from ik_llama.cpp Ornith CRACK Q8_0 to stock llama.cpp
+Tiel-Coder-35B (qwen35moe hybrid MoE), and Dankclaw switched from the Q6_0
+AGENT model to the Flash-Next 56B MoE (qwen4exp, PR #27742 merged into
+charlie's ROCmFPX fork, --fit off, 128k context). Runbooks:
 [docs/LLM/index.md](docs/LLM/index.md).
 Retired scripts live in `old/` (reference only) and `.AGENTTRASH/`.
 
@@ -130,6 +133,43 @@ graph LR
     G --> H[Per-service health check]
 ```
 
+## Boot Persistence & Self-Healing (MANDATORY)
+
+Every Podman container governed by this project that is successfully deployed
+and proven working MUST have a self-healing, boot-persistent Quadlet unit. A
+container that runs only from `compose.yaml` (or a manual `podman run`) dies
+silently at reboot: rootless Podman does not auto-start bare containers, and
+`restart: unless-stopped` does not survive the explicit stop that shutdown
+performs. Do not report such a service as "up" without its unit.
+
+Each container's unit must provide:
+
+- **Boot persistence** — `[Install] WantedBy=default.target`, installed to
+  `~/.config/containers/systemd/<name>.container` (generating
+  `<name>.service`) and enabled in the user manager
+  (`systemctl --user enable <name>.service`). Linger must be on for the deploy
+  user (`loginctl show-user dank -p Linger` → `Linger=yes`) so user units start
+  at boot with no login session.
+- **Self-healing** — `[Service] Restart=always` (or `Restart=on-failure`) with
+  a short `RestartSec`, so a crash or OOM brings the container back without
+  operator action. Add a quadlet health check where the service has one.
+- **Repo provenance** — the `.container` file lives in this repo
+  (`dankclaw/<svc>/quadlet/` or `dankserver/<svc>/quadlet/`), is installed by
+  the service-scoped deployer, and is covered by the quadlet test seam. The
+  host-side unit is generated state; the repo file is the source of truth.
+
+Verification before a service is declared deployed:
+`systemctl --user is-enabled <name>.service` → `enabled`,
+`systemctl --user is-active <name>.service` → `active`, and the container
+answers its health check. A running container with no unit is a gap: write the
+unit, test it, deploy it.
+
+Incident that established this rule (2026-08-26, Dankclaw reboot): Caddy ran
+for weeks as a bare compose container and never came back after reboot; the
+Pi-hole unit existed but crash-looped on a sysctl conflict (a later-sorted
+drop-in reset `net.ipv4.ip_unprivileged_port_start` to 80), taking LAN DNS
+down with it.
+
 ## Testing
 
 Tests are mandatory before commit. `bats` must be installed on the
@@ -189,7 +229,11 @@ DO NOT ROLL YOUR OWN BENCH UNLESS YOU CANNOT TEST THE DESIRED PARAMETER WITH THE
 1. Stop the service before any benchmark:
    `sudo systemctl stop frank-default-small` (Dankserver) or
    `sudo systemctl stop frank-default-large` (Dankclaw).
-2. Run the bench on a scratch port (never the LLM service port 7770).
+2. Run the bench (`llama-bench` for llama.cpp / vllm bench for vLLM).
+   Scratch ports are not used for tests or benches on the LLM hosts — a
+   test process cannot share the single GPU with the running service anyway
+   (the model + KV fill the card), so changes are validated directly against
+   the service itself.
 3. Restart the service when the bench completes: `sudo systemctl start <svc>`.
 4. Verify health before reporting:
    `curl http://192.168.0.235:7770/health` or
@@ -221,23 +265,43 @@ When you rename a service or move a model to a new host, update
 
 ## Runtime & Troubleshooting
 
-### kingjones Qwen3.8 Uncensored Q6_0 ROCmFPX AGENT llama.cpp (Dankclaw, ACTIVE)
+### Qwen3.8-Flash-Next ROCmFP4 STRIX_LEAN llama.cpp (Dankclaw, ACTIVE)
 
-The large service serves `kingjones/Qwen3.8-Uncensored-ROCmFP4-STRIX-MTP`
-(public) from `/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP/` with a
-charlie12345/ROCmFPX `llama-server` (build `build-promptforge`). It runs a
-standalone MTP draft head on ROCm0, f16 K + TurboQuant V KV cache, a full
-262144 context / 1 slot, and reasoning on (Sharp template, budget -1,
-`--reasoning-format none`; chat kwargs keep `preserve_thinking` +
-`preserve_reasoning`). The Q6_0 main model is 24.86 GB. Env
-`scripts/frank-default-large.env`, serve `scripts/serve-frank-large.sh`, and
-runbook `docs/LLM/kingjones-q8agent-rocmfpx.md` define the deployment.
+The large service serves
+`kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF` (public, 98.5 GiB)
+from `/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN/`
+with a charlie12345/ROCmFPX `llama-server` built at
+`/home/dank/GitPackages/ROCmFPX/build-qwen38` (PR #27742 qwen4exp architecture
+merged). The model is a 56B MoE (512 experts, 10 used), 48 layers (12 attn +
+36 SSM, full_attention_interval=4), QSA indexer (4 heads, top_k=2048),
+hyper-connections (count=4, low_rank=320), PLE (layer 1). Q4_0_ROCMFP4 quant.
+It runs f16 KV cache, 131072 context / 1 slot, `--fit off` (auto-fit segfaults
+on qwen4exp hyper-connection config), and reasoning on (Sharp template, budget
+-1, `--reasoning-format deepseek`; chat kwargs keep `preserve_thinking` +
+`preserve_reasoning`). Env `scripts/frank-default-large.env`, serve
+`scripts/serve-frank-large.sh`.
 
-**Serving status (2026-09-01):** restored the previously deployed Q6_0 AGENT
-model. The Q8_0 AGENT experiment is documented in
+**Serving status (2026-09-01):** deployed the Flash-Next 56B MoE model. The
+previous kingjones Q6_0 AGENT model (27B, standalone MTP, 262144 context) is
+retired. The Q8_0 AGENT experiment is documented in
 `docs/LLM/kingjones-q8agent-rocmfpx.md`; it was retired because it crashed in
 `ggml_compute_forward_set_rows` and did not provide a working service.
 
+### Tiel-Coder-35B-A3B llama.cpp (Dankserver, ACTIVE)
+
+The small service serves `peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF` (public,
+30 GB, UD-Q6_K_XL quant) from `/home/dank/models/tiel-coder-35b/` with
+upstream llama.cpp (latest master, CUDA build with
+`-DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86`). The model is a
+qwen35moe hybrid MoE: 40 layers (10 attention + 30 SSM/Mamba,
+full_attention_interval=4), 256 experts / 8 used, 2 KV heads × 256 dim. It
+runs Q8_0 KV cache, 200700 total context (66900/slot × 3 parallel), `--cpu-moe`
+for expert offload, `-ngl 999` (non-expert on GPU), and mmap. Env
+`scripts/frank-default-small.env`, serve `scripts/serve-frank-small.sh`.
+
+**Serving status (2026-09-01):** deployed the Tiel-Coder-35B model. The
+previous ik_llama.cpp Ornith CRACK Q8_0 deployment is retired.
+
 ### charlie12345/ROCmFPX ROCmI4 W4A4 llama.cpp (Dankclaw, prior v3, 2026-08-18 to 2026-08-25)
 
 The v4 runtime serves `cafonz/Qwen3.8-27B-ROCmI4-MTP-GGUF` (`eemin` namespace,
@@ -405,8 +469,8 @@ two hosts, one Caddy, and one Gitea. One repo keeps them in one history.
 | Editing a compose/Caddyfile/serve script | **Work through git** — fetch, commit, edit, test, push, pull, deploy. Never edit on the remote directly. |
 | Config exists only on remote | **Pull it into the repo first** — commit the fetched version (redacted) before making changes. |
 | Unstaged local changes | **Commit or discard** — check `git status` before starting work. Ask the user if unsure. |
-| Deploying a committed change | **Run `scripts/deploy.sh <target>`** — it pushes, pulls with fast-forward-only, plans changed paths, dispatches selected services, and checks their health. See [docs/LLM/deployment.md](docs/LLM/deployment.md). |
-| Benchmarking | **Stop the service first** — scratch port, restart, verify health. See the Benchmarking Policy above. |
+| Deploying a committed change | **Run `scripts/deploy.sh <target>`** — it pushes, pulls with fast-forward-only, plans changed paths, dispatches selected services, and checks their health. See [docs/LLM/deployment.md](docs/LLM/deployment.md). | | Container without a Quadlet unit | **Give it one** — self-healing, boot-persistent `.container` unit via git (see Boot Persistence & Self-Healing). |
+| Benchmarking | **Stop the service first**, restart, verify health. See the Benchmarking Policy above. |
 | Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
 | Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
 | Real secrets | **Never commit** — use host `.env` files. `.env.example` holds placeholders only. `FRANK_SUDO_PW` covers headless sudo. |

dankclaw/.env.example

@@ -40,6 +40,11 @@ SIGNAL_ACCOUNT=+15551234567
 # Comma-separated E.164 numbers allowed to message the agent.
 SIGNAL_ALLOWED_USERS=+15551234567
 
+# Hermes web UI (Hermex server) login password. Required: the webui binds to
+# loopback and is reached only through the SSH tunnel, but Hermex still signs
+# in with this password. Set a strong value in the live .env.
+HERMES_WEBUI_PASSWORD=changeme
+
 # Porkbun API (franksdank.cloud + franksdank.casa zones). Caddy uses these
 # for DNS-01 Let's Encrypt wildcard cert issuance. Porkbun API keys are
 # account-wide; rotate them here rather than in the dashboard.

dankclaw/Caddyfile

@@ -190,11 +190,24 @@ pihole.franksdank.cloud {
 # Let's Encrypt cert for it. It stays plain-HTTP legacy; the canonical
 # name is plex.franksdank.cloud (HTTPS, below).
 http://plex.franksdank.casa {
-	reverse_proxy 192.168.0.235:32400
+	reverse_proxy 192.168.0.235:32400 {
+		transport http {
+			tls_insecure_skip_verify
+		}
+	}
 }
 
+# PMS requires TLS from every non-loopback client: plain-HTTP off-loopback
+# is read and silently closed (diagnosed 2026-08-26), which surfaced as an
+# empty-body 502 here. Dial the upstream over TLS; PMS's self-signed cert is
+# skipped on this pinned-LAN hop only (client-facing LE TLS is untouched).
+# The web UI lives at /web/; / returns the XML MediaContainer index.
 plex.franksdank.cloud {
-	reverse_proxy 192.168.0.235:32400
+	reverse_proxy 192.168.0.235:32400 {
+		transport http {
+			tls_insecure_skip_verify
+		}
+	}
 }
 
 # --- Calibre-Web ---

dankclaw/caddy/env.example

@@ -0,0 +1,7 @@
+# Caddy environment (placeholders only). The deploy writes the live file at
+# /home/dank/Containers/caddy/env from the checkout's .env; the quadlet reads
+# it via EnvironmentFile. Consumed by the Caddyfile's acme_dns block
+# (DNS-01 wildcard cert issuance). PORKBUN_API_SECRET_KEY is the name the
+# caddy-dns/porkbun module expects.
+PORKBUN_API_KEY=changeme
+PORKBUN_API_SECRET_KEY=changeme

dankclaw/caddy/quadlet/caddy.container

@@ -0,0 +1,54 @@
+# Quadlet unit: canonical Caddy reverse proxy + ACME (rootless, boot-persistent).
+#
+# Install path on dankclaw: ~/.config/containers/systemd/caddy.container
+# Unit name: caddy.service (container name: caddy)
+#
+# The homelab's single Caddy: it fronts every service on both hosts and issues
+# Let's Encrypt wildcard certs via DNS-01 (Porkbun). It left compose.yaml on
+# 2026-08-26 after a reboot exposed that the bare compose container never came
+# back: rootless Podman has no boot-time auto-start for bare containers, and
+# `restart: unless-stopped` does not survive the explicit stop that shutdown
+# performs. See AGENTS.md "Boot Persistence & Self-Healing" and
+# docs/Infrastructure/Caddy.md.
+#
+# It joins the existing compose network (dankclaw_default) so the Caddyfile's
+# container-name upstreams (authentik-server, grafana, ...) still resolve; if
+# no compose service has created that network yet at boot, quadlet creates it
+# first and compose reuses it by name later. The image is built locally from
+# dankclaw/caddy/Containerfile and exists only on this host, so Pull=never: a
+# boot must not stall on a registry that does not have it. Porkbun credentials
+# come from /home/dank/Containers/caddy/env (deploy writes it from the
+# checkout's .env; secrets never enter the repo).
+
+[Unit]
+Description=Caddy reverse proxy + ACME (Porkbun DNS-01) container
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=caddy
+Image=franksdank/caddy-porkbun:latest
+Pull=never
+Network=dankclaw_default
+PublishPort=0.0.0.0:80:80/tcp
+PublishPort=0.0.0.0:443:443/tcp
+EnvironmentFile=/home/dank/Containers/caddy/env
+Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
+Volume=/home/dank/Containers/caddy/data:/data
+Volume=/home/dank/Containers/caddy/config:/config
+# The homepage: the host's /srv/homepage (deploy-managed, not the repo's
+# shared/homepage).
+Volume=/srv/homepage:/srv/homepage:ro
+# The bare-IP catch-all answers 302 (curl -f passes on <400); this proves
+# Caddy itself is serving without touching any upstream.
+HealthCmd=curl -sf -o /dev/null http://localhost:80/
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+
+[Service]
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

dankclaw/compose.yaml

@@ -94,6 +94,44 @@ services:
     volumes:
       - /home/dank/hermes-data:/data
 
+  # --- Hermes web UI (Hermex server; phone-facing API for the agent) --------
+  # Runs the open-source hermes-webui (github.com/nesquena/hermes-webui) in
+  # foreground mode: bootstrap.py creates .venv inside the mounted checkout,
+  # installs WebUI + agent deps, then execs server.py as a single foreground
+  # process. It reads HERMES_HOME=/data, so it drives the SAME agent state
+  # (memory, skills, sessions) as the gateway container.
+  # Loopback-only publish: the SSH tunnel (phone -> Dankclaw:22) is the sole
+  # ingress. The password gates Hermex logins and lives in the host .env.
+  hermes-webui:
+    image: docker.io/nousresearch/hermes-agent:latest
+    container_name: hermes-webui
+    restart: unless-stopped
+    entrypoint: ["bash", "-c", "python3 /webui/bootstrap.py --foreground"]
+    environment:
+      HERMES_HOME: /data
+      # Container-internal bind: 0.0.0.0 so Podman's port forwarder (which
+      # dials the container's bridge IP) can reach it. The *host-side*
+      # publish below stays on 127.0.0.1, so loopback-only ingress holds.
+      HERMES_WEBUI_HOST: 0.0.0.0
+      HERMES_WEBUI_PORT: "8787"
+      HERMES_WEBUI_PASSWORD: "${HERMES_WEBUI_PASSWORD}"
+      # Headless host: never show the first-run wizard (a profile already
+      # exists in /data from the gateway).
+      HERMES_WEBUI_SKIP_ONBOARDING: "1"
+    ports:
+      - "127.0.0.1:8787:8787"
+    volumes:
+      - /home/dank/hermes-data:/data
+      # rw on purpose: bootstrap.py creates /webui/.venv in the repo root.
+      - /home/dank/hermes-webui:/webui
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8787/health')"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      # First boot pip-installs WebUI + agent deps into a fresh venv.
+      start_period: 600s
+
   # --- Git hosting (Gitea) -------------------------------------------------
   # Now boot-persistent Quadlet units (private bridge network, DNS enabled),
   # not compose services. See gitea/quadlet/ and ../docs/Dankclaw/Gitea.md.
@@ -272,44 +310,8 @@ services:
       authentik-postgres:
         condition: service_healthy
 
-  # --- Reverse proxy + homepage (the homelab's single Caddy) ---------------
-  # The Caddyfile mounted here is the RENDERED copy (basicauth hash filled in
-  # from the host .env by scripts/deploy.sh); the repo copy carries the
-  # REPLACE_WITH_HASH placeholder. Certs/config live under
+  # Caddy (the homelab's single reverse proxy) left compose on 2026-08-26:
+  # it is boot-persistent via its Quadlet unit (dankclaw/caddy/quadlet/
+  # caddy.container), installed by deploy_caddy in scripts/deploy-remote.sh.
+  # The rendered Caddyfile, certs, and config still live under
   # /home/dank/Containers/caddy (rootless-writable).
-  caddy:
-    # Locally built: stock Caddy + the Porkbun DNS provider
-    # (dankclaw/caddy/Containerfile). See that file for build instructions.
-    image: franksdank/caddy-porkbun:latest
-    container_name: caddy
-    restart: unless-stopped
-    ports:
-      - "80:80"
-      - "443:443"
-    environment:
-      # Consumed by the Caddyfile's acme_dns block (DNS-01 wildcard certs).
-      PORKBUN_API_KEY: ${PORKBUN_API_KEY}
-      PORKBUN_API_SECRET_KEY: ${PORKBUN_API_SECRET_KEY}
-    volumes:
-      - /home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
-      - /home/dank/Containers/caddy/data:/data
-      - /home/dank/Containers/caddy/config:/config
-      # The homepage: the host's /srv/homepage. The deploy copies the static
-      # files there; the status checker writes status.json and the deploy
-      # writes deployed.js there. (Not the repo's shared/homepage, which
-      # lacks the generated files.)
-      - /srv/homepage:/srv/homepage:ro
-    depends_on:
-      - authentik-server
-      - comfyui
-      - hermes
-      - grafana
-      - prometheus
-      - loki
-    healthcheck:
-      # The bare-IP catch-all answers 302 (curl -f passes on <400); this
-      # proves Caddy itself is serving without touching any upstream.
-      test: ["CMD", "curl", "-sf", "-o", "/dev/null", "http://localhost:80/"]
-      interval: 30s
-      timeout: 10s
-      retries: 3

dankclaw/gitea/quadlet/gitea-caddy.container

@@ -1,31 +0,0 @@
-# Quadlet unit: Gitea Caddy reverse proxy (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/gitea-caddy.container
-# Unit name: gitea-caddy.service (podman 5.x drops the container- prefix)
-#
-# Serves gitea.franksdank.cloud (public domain, DNS pending). The LAN
-# reaches Gitea directly on port 3000. This file holds no secrets.
-# See docs/Dankclaw/Gitea.md.
-
-[Unit]
-Description=Gitea Caddy reverse proxy container
-Wants=gitea-server.service
-After=gitea-server.service network.target
-
-[Container]
-ContainerName=gitea-caddy
-Network=gitea.network
-Image=docker.io/library/caddy:latest
-PublishPort=80:80
-PublishPort=443:443
-Volume=/opt/gitea/data/caddy/Caddyfile:/etc/caddy/Caddyfile
-Volume=/opt/gitea/data/caddy/site:/srv
-Volume=/opt/gitea/data/caddy/config:/config
-Volume=/opt/gitea/data/caddy/data:/root/.local/share/caddy
-
-[Service]
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/gitea/quadlet/gitea.network

@@ -4,7 +4,7 @@
 # Unit name: gitea-network.service (podman network name: systemd-gitea)
 #
 # DNS is enabled by default for quadlet networks. This matters: the
-# automatic "podman" default network has dns_enabled=false, so container
-# names (gitea-postgres, gitea-server, gitea-caddy) would not resolve.
+# automatic "podman" default network has dns_enabled=false, so the
+# gitea-postgres and gitea-server container names would not resolve.
 
 [Network]

dankclaw/pihole/addn-hosts

@@ -1,6 +1,6 @@
 # A records for the flat franksdank.name scheme (dnsmasq addn-hosts format:
 # <IP> <name> [<name> ...]). Served by the Pi-hole quadlet (FTL is a
-# dnsmasq fork): the quadlet mounts this file into /etc/dnsmasq.d/ read-only.
+# dnsmasq fork): the quadlet mounts this file into /etc/pihole/hosts/ read-only.
 #
 # Caddy is consolidated on dankclaw (192.168.0.143): it fronts every service
 # on both hosts, so every name resolves to that one IP. Keep in sync with the

dankclaw/pihole/quadlet/pihole.container

@@ -6,7 +6,7 @@
 # The LAN's authoritative DNS for the flat franksdank.cloud / franksdank.casa
 # name scheme. It retired the dnsmasq quadlet (2026-08-24): Pi-hole FTL is a
 # dnsmasq fork, so the addn-hosts record file carries over unchanged and is
-# mounted into /etc/dnsmasq.d/ (the FTL dnsmasq config dir). It serves one A
+# mounted into /etc/pihole/hosts/ (the FTL hostsdir) read-only. It serves one A
 # record per Caddy vhost and forwards everything else to the pinned upstream
 # resolvers (DNS1/DNS2 in the env file). The SURFboard G36 router hands this
 # server out to DHCP clients (DNS statically configured -> 192.168.0.143),

dankclaw/pihole/sysctl/99-dnsmasq.conf

@@ -0,0 +1,8 @@
+# Rootless DNS must bind port 53: lower the unprivileged-port threshold so a
+# rootless container may publish 0.0.0.0:53 (80/443 stay allowed; 53 < 80).
+# Single source of truth for net.ipv4.ip_unprivileged_port_start on dankclaw;
+# deploy_pihole installs it to /etc/sysctl.d/ and applies it with
+# `sysctl --system` (the same code path boot uses via systemd-sysctl).
+# A later-sorted drop-in that reset this to 80 took LAN DNS down after the
+# 2026-08-26 reboot — deploy_pihole neutralizes such conflicts.
+net.ipv4.ip_unprivileged_port_start=53

docs/Architecture/Deployment Flow.md

@@ -60,7 +60,7 @@ service on the other host.
 | Service family | Runtime manager | Scope behavior |
 |---|---|---|
 | Dankclaw compose services | `podman-compose -f dankclaw/compose.yaml up -d --no-deps NAME` | Recreates only `NAME`; Caddy is handled separately because it fronts both hosts. |
-| SearXNG, Gluetun, Valkey, OpenWebUI, Pi-hole, Gitea | user Quadlet + `systemctl --user` | Installs the selected unit and restarts only the required dependency-coupled units. |
+| SearXNG, Gluetun, Valkey, OpenWebUI, Pi-hole, Gitea database/server | user Quadlet + `systemctl --user` | Installs the selected unit and restarts only the required dependency-coupled units. Gitea's web proxy is the canonical Caddy compose service. |
 | `frank-default-large` | systemd + `systemctl` | Relinks and restarts only the large inference service; waits up to 20 minutes for `/health`. |
 | `frank-default-small` | systemd + `systemctl` | Relinks and restarts only the small inference service; waits up to 20 minutes for `/health`. |
 | Dankserver media inventory | host-managed bare containers | Compose is inventory only; the scoped deploy does not create duplicate containers. |
@@ -71,12 +71,11 @@ it is not restarted by an unrelated large-model deployment.
 
 ## Caddy safety
 
-Caddy runs on Dankclaw and fronts both hosts. The dispatcher never stops
-`gitea-caddy` implicitly. If the legacy `gitea-caddy` container owns ports
-80/443 while the compose `caddy` container is absent or stopped, a Caddy
-selection fails closed rather than replacing the live proxy. Resolve port
-ownership explicitly before retrying. A Caddy reload failure falls back to a
-restart of **only** the selected `caddy` container.
+Caddy runs on Dankclaw and fronts both hosts, including Gitea. The former
+standalone `gitea-caddy` Quadlet is retired; the canonical compose `caddy`
+service is now the sole reverse-proxy owner of ports 80/443. A Caddy reload
+failure falls back to a restart of **only** the selected `caddy` container.
+Before deploying, verify that no retired proxy still owns either port.
 
 Homepage files, generated `deployed.js`, the status checker, and the rendered
 Caddyfile are updated only when their service is selected. A media-only

docs/Dankclaw/Dankclaw Stack.md

@@ -1,3 +1,15 @@
+---
+type: reference
+title: Dankclaw Stack
+description: Service inventory and routing overview for the Dankclaw AI, development, and observability host.
+tags:
+  - dankclaw
+  - services
+  - containers
+  - caddy
+timestamp: 2026-08-26
+---
+
 # Dankclaw Stack
 
 Dankclaw (192.168.0.143) runs the AI, search, and development stack. It
@@ -43,9 +55,9 @@ repo owns their scripts, units, and knowledge base (see
 
 ## Boot Persistence
 
-The Gitea trio, the SearXNG trio (gluetun, searxng, valkey), and open-webui run as quadlet units, so they start at boot. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent.
+The Gitea database and server, the SearXNG trio (gluetun, searxng, valkey), open-webui, Pi-hole, and Caddy run as Quadlet units, so they start at boot. Caddy left `compose.yaml` on 2026-08-26 after a reboot exposed that bare compose containers never come back; see [[Caddy]] for the unit and the incident. The former standalone Gitea Caddy unit is retired. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent — per AGENTS.md (Boot Persistence & Self-Healing) they each need a quadlet before they may be considered reliably deployed.
 
-Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw` and open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]).
+Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw`, open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`, and Pi-hole with `scripts/deploy-pihole-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]); Caddy is deployed through `scripts/deploy.sh dankclaw --service caddy`.
 
 ## Routing Rules
 

docs/Dankclaw/Gitea.md

@@ -1,8 +1,20 @@
+---
+type: configuration
+title: Gitea
+description: Gitea deployment on Dankclaw with its web UI reverse-proxied by the canonical Caddy service.
+tags:
+  - gitea
+  - dankclaw
+  - quadlet
+  - caddy
+timestamp: 2026-08-26
+---
+
 # Gitea
 
-Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On Dankclaw it runs as three Quadlet-managed containers in the rootless podman session of user `dank`.
+Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On Dankclaw it runs as two Quadlet-managed containers in the rootless Podman session of user `dank`. The canonical Caddy on the same host reverse-proxies its web UI at `gitea.franksdank.cloud`.
 
-**Host:** Dankclaw — **LAN access:** `http://dankclaw:3000` (or `http://192.168.0.143:3000`) — **Public domain:** `gitea.franksdank.cloud` (DNS pending) — **Docs:** https://docs.gitea.com
+**Host:** Dankclaw — **LAN access:** `http://dankclaw:3000` (or `http://192.168.0.143:3000`) — **Proxy domain:** `gitea.franksdank.cloud` — **Docs:** https://docs.gitea.com
 
 ## Containers
 
@@ -10,7 +22,6 @@ Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On
 |-----------|------|-------|---------|
 | `gitea-postgres` | `gitea-postgres.service` | `docker.io/library/postgres:16-alpine` | Database |
 | `gitea-server` | `gitea-server.service` | `docker.io/gitea/gitea:latest` | Web UI, API, git (HTTP :3000, SSH :2222) |
-| `gitea-caddy` | `gitea-caddy.service` | `docker.io/library/caddy:latest` | Reverse proxy for the public domain |
 
 The containers share the private bridge network `systemd-gitea`. The quadlet network file `gitea.network` enables DNS, so the containers resolve each other by name. The automatic `podman` default network has DNS disabled. Do not run the Gitea containers on it.
 
@@ -29,10 +40,10 @@ Run all commands as `dank` on Dankclaw.
 
 | Action | Command |
 |--------|---------|
-| Start all | `systemctl --user start gitea-postgres gitea-server gitea-caddy` |
-| Stop all | `systemctl --user stop gitea-caddy gitea-server gitea-postgres` |
+| Start all | `systemctl --user start gitea-postgres gitea-server` |
+| Stop all | `systemctl --user stop gitea-server gitea-postgres` |
 | Restart Gitea only | `systemctl --user restart gitea-server` |
-| Status | `systemctl --user status gitea-postgres gitea-server gitea-caddy` |
+| Status | `systemctl --user status gitea-postgres gitea-server` |
 | Container logs | `podman logs -f gitea-server` |
 | Unit logs | `journalctl --user -u gitea-server -f` |
 
@@ -42,14 +53,13 @@ The units start in dependency order:
 graph LR
     N[gitea-network.service] --> P[gitea-postgres.service]
     P --> S[gitea-server.service]
-    S --> C[gitea-caddy.service]
 ```
 
 ## Boot persistence
 
 User-level linger is enabled for `dank` (`loginctl enable-linger dank`). The units start at host boot without a login session. Each unit carries `[Install] WantedBy=default.target`. The podman generator applies it at boot; no symlink is needed.
 
-The containers use the quadlet fresh-start model. A stop removes the container (`podman run --rm --replace`). The next start recreates it. The bind-mounted data survives.
+The containers use the quadlet fresh-start model. A stop removes the container (`podman run --rm --replace`). The next start recreates it. The bind-mounted data survives. The canonical Caddy is managed separately by the Dankclaw stack.
 
 ## Secrets
 
@@ -61,11 +71,11 @@ Do not change the password in place. Change the PostgreSQL role first, then upda
 
 | File | Location | Repo copy |
 |------|----------|-----------|
-| Quadlet units | `~/.config/containers/systemd/gitea-{postgres,server,caddy}.container`, `gitea.network` | [`dankclaw/gitea/quadlet/`](../../dankclaw/gitea/quadlet/) |
-| Caddyfile | `/opt/gitea/data/caddy/Caddyfile` | host-only (not in repo) |
+| Quadlet units | `~/.config/containers/systemd/gitea-{postgres,server}.container`, `gitea.network` | [`dankclaw/gitea/quadlet/`](../../dankclaw/gitea/quadlet/) |
+| Caddyfile | `/home/dank/Containers/caddy/Caddyfile` | [`dankclaw/Caddyfile`](../../dankclaw/Caddyfile) |
 | Secrets | `/opt/gitea/.env` (0600) | [`dankclaw/gitea/.env.example`](../../dankclaw/gitea/.env.example) |
 
-After editing a repo copy, copy it to the target, then run `systemctl --user daemon-reload` and restart the affected units.
+After editing a repo copy, deploy through `scripts/deploy.sh dankclaw`. The Gitea database/server units are installed from this directory; the canonical Caddy and its rendered Caddyfile are deployed as the separate `caddy` service. Do not manually recreate the retired `gitea-caddy` unit.
 
 ## Data volumes
 
@@ -73,7 +83,6 @@ After editing a repo copy, copy it to the target, then run `systemctl --user dae
 |----------------|-----------|----------|
 | `/data` | `/opt/gitea/data/gitea-gitea` | Git repositories, app.ini, attachments |
 | `/var/lib/postgresql/data` | `/opt/gitea/data/gitea-postgres` | PostgreSQL data |
-| `/etc/caddy/Caddyfile`, `/root/.local/share/caddy` | `/opt/gitea/data/caddy/...` | Caddy config and TLS state |
 
 Back up `/opt/gitea/data/` (tar) or `pg_dump` the database. See the knowledge base page [[Gitea]].
 
@@ -83,8 +92,8 @@ Back up `/opt/gitea/data/` (tar) or `pg_dump` the database. See the knowledge ba
 - **The files in the data directory are owned by host UID 100999**, not `dank` (rootless ID mapping: container uid 1000 → host 100999). Edit them inside the container with `podman exec --user 1000 gitea-server ...`.
 - **The host auto-reboots.** `unattended-upgrades` reboots Dankclaw when a package needs it. The quadlet units make Gitea survive that. See the 2026-08-16 incident below.
 - **Port 3000 vs 3001.** The live host port is 3000. The full-stack [`dankclaw/compose.yaml`](../../dankclaw/compose.yaml) maps 3001. Align the two before deploying the full stack.
-- **Legacy compose retired.** `/opt/gitea/podman-compose.yml` is retired to `/opt/gitea/.deprecated/`. Do not run it. Its containers collide with the quadlet units on ports 3000, 2222, 80, and 443.
-- **Rootless cannot bind ports below 1024 by default.** `gitea-caddy` publishes 80 and 443. Without a host sysctl, `rootlessport` fails with `permission denied` and the unit crash-loops under `Restart=always`. The fix is a one-time sysctl: `net.ipv4.ip_unprivileged_port_start=80`, persisted in `/etc/sysctl.d/99-rootless-ports.conf`. Applied 2026-08-18 after a 13,000-restart crash loop.
+- **Legacy compose retired.** `/opt/gitea/podman-compose.yml` is retired to `/opt/gitea/.deprecated/`. Do not run it. Its containers collide with the Quadlet units on ports 3000 and 2222.
+- **Canonical proxy.** `gitea.franksdank.cloud` is served by the Caddy service in `dankclaw/compose.yaml`, which proxies to `192.168.0.143:3000`. Do not recreate a Gitea-specific Caddy unit or publish another proxy on ports 80/443. During migration, the deployer stages any installed legacy unit under `~/.AGENTTRASH/YYYYMMDD-gitea-caddy/` before starting canonical Caddy.
 
 ## Incident: 2026-08-16 — down after unattended reboot
 

docs/Dankclaw/Hermes-Dashboard.md

@@ -35,3 +35,7 @@ Open `http://192.168.0.143:9119` in a browser on the LAN. Log in with the config
 - The dashboard and the gateway share one data volume. A config change made in the dashboard takes effect when the gateway restarts.
 - If the shared password is reused elsewhere, treat it as exposed. Rotate it and regenerate the hash.
 - `--skip-build` uses the prebuilt web assets in the image. Do not remove it unless the assets are rebuilt.
+
+## See Also
+
+- [[Hermes-WebUI]] — the phone-facing API server (Hermex)

docs/Dankclaw/Hermes-WebUI.md

@@ -0,0 +1,114 @@
+---
+type: configuration
+title: Hermes-WebUI (Hermex server)
+description: The hermes-webui compose service on Dankclaw — the loopback-only API server that the Hermex iPhone app talks to over an SSH tunnel.
+tags:
+  - hermes
+  - hermes-webui
+  - hermex
+  - dankclaw
+  - ssh-tunnel
+timestamp: 2026-09-14
+---
+
+# Hermes-WebUI (Hermex server)
+
+`hermes-webui` is the [hermes-webui](https://github.com/nesquena/hermes-webui)
+server (MIT, third-party) that the **Hermex** iPhone app connects to. It runs
+the Hermes agent in-process from the shared `HERMES_HOME`, so it is the *same*
+agent — same memory, skills, and sessions — as the [[Hermes]] gateway and the
+[[Hermes-Dashboard]].
+
+**Host:** Dankclaw — **Port:** `127.0.0.1:8787` (loopback only) — **Ingress:** SSH tunnel
+
+## Facts
+
+| Item | Value |
+|------|-------|
+| Image | `docker.io/nousresearch/hermes-agent:latest` (same as the gateway) |
+| Command | `python3 /webui/bootstrap.py --foreground` |
+| Source checkout | `/home/dank/hermes-webui` on Dankclaw (mounted at `/webui`) |
+| Port | `8787`, published to `127.0.0.1` only |
+| Auth | `HERMES_WEBUI_PASSWORD` (host `.env`; Hermex signs in with it) |
+| Data | shares `/home/dank/hermes-data` with the gateway and dashboard |
+| Health | `GET /health` (compose healthcheck + deployer) |
+
+## Why a container, not a host process
+
+The gateway's state files are owned by the container-mapped uid (`109999`,
+from rootless Podman's `1 → 100000/65536` subuid range). A fresh rootless
+namespace on Dankclaw maps identically (verified with `podman unshare
+cat /proc/self/uid_map`), so a second container using the same image reads
+and writes the shared volume with exactly the ownership the gateway already
+uses. A host-side process running as `dank` (uid 1000) could not read the
+gateway's `0600` state files.
+
+## How it works
+
+- `bootstrap.py` creates `/webui/.venv` on first boot, pip-installs the WebUI
+  and agent requirements, then **execs** `server.py` as one foreground process
+  (`--foreground`). systemd-free: Podman supervises it via
+  `restart: unless-stopped`.
+- `HERMES_WEBUI_SKIP_ONBOARDING=1` keeps the first-run wizard off; a profile
+  already exists in `/data` from the gateway.
+- The healthcheck `start_period` is 600 s because the very first boot spends
+  minutes pip-installing into a fresh venv.
+
+## Remote access (Hermex on iOS, from anywhere)
+
+Only port `22` is forwarded to Dankclaw at the router, so the webui is
+reached through an SSH local forward — no extra router ports, no third-party
+relay (no Tailscale, no Cloudflare):
+
+1. On the iPhone, open **Termius** (or any SSH client with local port
+   forwarding) and connect to `<dankclaw-public>:22` with the usual key.
+2. On that connection, add a **local forward**: `127.0.0.1:8787` →
+   `127.0.0.1:8787`. The forward runs inside the encrypted SSH session;
+   nothing else on the phone routes through it.
+3. In Hermex, set the server URL to `http://localhost:8787` and sign in with
+   the webui password.
+
+The tunnel lives as long as the SSH session on the phone — iOS does not allow
+a true background daemon, so connect before you use Hermex.
+
+```mermaid
+graph LR
+    A[Hermex app] -->|http://localhost:8787| B[SSH local forward on iPhone]
+    B ==>|encrypted SSH to public IP:22| C[Dankclaw sshd]
+    C --> D["hermes-webui 127.0.0.1:8787"]
+    D --> E["/home/dank/hermes-data (shared agent state)"]
+```
+
+## Deployment
+
+```
+./scripts/deploy.sh dankclaw --service hermes-webui
+```
+
+The planner maps changes to the `hermes-webui` compose block (and only that
+block) to this service, so gateway and inference restarts stay isolated.
+Before first deploy, set a strong `HERMES_WEBUI_PASSWORD` in the live
+`dankclaw/.env`.
+
+## Gotchas
+
+- The `/webui` mount is **rw on purpose**: `bootstrap.py` creates `.venv` in
+  the repo root. Do not flip it to `:ro`.
+- Never publish `8787` beyond loopback without keeping the password set —
+  the webui can run agent commands.
+- The webui and the gateway share one data volume. A config change made on one
+  side takes effect when the other restarts (same caveat as the dashboard).
+- `bootstrap.py` auto-installs a Hermes Agent if it cannot find one; in this
+  image the agent is already present, so that path should never fire. If the
+  log shows an install attempt, the image changed — re-verify the entrypoint.
+
+# Citations
+
+- hermes-webui README (env vars, `--foreground`, `/health`): https://github.com/nesquena/hermes-webui
+- Hermex setup model (server URL + password; Cloudflare/Tailscale are optional, not required): https://hermexapp.com/
+
+## See Also
+
+- [[Hermes]] — the agent and its Signal gateway
+- [[Hermes-Dashboard]] — the LAN browser management UI
+- [[Dankclaw Stack]]

docs/Dankclaw/Hermes.md

@@ -45,6 +45,8 @@ both containers. The tracked, sanitized configuration baseline is
 
 ## See Also
 
+- [[Hermes-Dashboard]] — the LAN browser management UI
+- [[Hermes-WebUI]] — the Hermex server (phone-facing API, SSH-tunneled)
 - [[Dankclaw Stack]]
 - [[OpenWebUI]] — the chat UI that pairs with Hermes
 - Knowledge base: [[SearXNG]] (Hermes as the search agent)

docs/Infrastructure/Caddy.md

@@ -1,6 +1,6 @@
 # Caddy
 
-Caddy is the single reverse proxy for both hosts. It runs as a container on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth.
+Caddy is the single reverse proxy for both hosts. It runs as a boot-persistent quadlet container (`caddy.service`) on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth.
 
 **Docs:** https://caddyserver.com/docs
 
@@ -8,6 +8,7 @@ Caddy is the single reverse proxy for both hosts. It runs as a container on Dank
 
 | Item | Value |
 |------|-------|
+| Runtime | Quadlet unit `caddy.service` (`dankclaw/caddy/quadlet/caddy.container`) — boot-persistent, self-healing |
 | Image | `franksdank/caddy-porkbun:latest` (local build, `dankclaw/caddy/Containerfile`) |
 | Config | `/home/dank/Containers/caddy/Caddyfile` (bind mount, rendered by deploy) |
 | Data | `/home/dank/Containers/caddy/data` (certs) and `.../config` (ACME state) |
@@ -15,7 +16,19 @@ Caddy is the single reverse proxy for both hosts. It runs as a container on Dank
 | TLS | Let's Encrypt, DNS-01 via the Porkbun module (one cert per vhost) |
 | Runs on | Dankclaw only — Dankserver has no Caddy |
 
-The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before `up`. See [[Deployment Flow]].
+The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before start. See [[Deployment Flow]].
+
+## Quadlet (boot persistence)
+
+Caddy left `compose.yaml` on 2026-08-26. A reboot exposed that the bare compose container never comes back: rootless Podman has no boot-time auto-start for bare containers, and `restart: unless-stopped` does not survive the explicit stop that shutdown performs. The quadlet unit is now the only way Caddy starts (see AGENTS.md "Boot Persistence & Self-Healing").
+
+- **Install path on Dankclaw:** `~/.config/containers/systemd/caddy.container` (unit `caddy.service`)
+- **Network:** `dankclaw_default` (the compose bridge — the Caddyfile's container-name upstreams must keep resolving; if no compose service has created the network yet at boot, quadlet creates it first and compose reuses it by name later)
+- **Published ports:** 80/tcp, 443/tcp
+- **Env file:** `/home/dank/Containers/caddy/env` (`PORKBUN_API_KEY`, `PORKBUN_API_SECRET_KEY`) — the deploy writes it from the checkout's `.env`; secrets never enter the repo (placeholders in `dankclaw/caddy/env.example`)
+- **Restart:** `Restart=always`, `RestartSec=5`; the health check probes the bare-IP catch-all (302) inside the container
+- **Image policy:** `Pull=never` — the image is built on-host and exists nowhere else; a boot must not stall on a registry
+- **Deploy:** `./scripts/deploy.sh dankclaw --service caddy` — renders the Caddyfile, rebuilds the image if the Containerfile changed, retires the legacy split proxy, writes the env file, removes the old bare container (all state is in host volumes), then installs, enables, and starts the unit
 
 ## The porkbun image
 
@@ -34,7 +47,7 @@ acme_dns porkbun {
 }
 ```
 
-compose passes both variables from the host `.env`. Keep the real keys in `.env` only; `.env.example` holds placeholders. The Porkbun account holds only `franksdank.cloud`.
+The deploy writes both variables from the host `.env` into `/home/dank/Containers/caddy/env`, which the quadlet reads via `EnvironmentFile`. Keep the real keys in `.env` only; `.env.example` (and `dankclaw/caddy/env.example`) hold placeholders. The Porkbun account holds only `franksdank.cloud`.
 
 ## Vhost map
 
@@ -46,7 +59,7 @@ Every vhost uses the flat `franksdank.cloud` convention. The full list lives in
 | Dankclaw quadlet services | by published LAN port, e.g. `192.168.0.143:3000` (Gitea) |
 | Dankserver services | by published LAN port, e.g. `192.168.0.235:8085` (RomM) |
 
-The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/`. The legacy name `dankserver.franksdank.cloud` 302-redirects there too. `plex.franksdank.casa` is the only plain-HTTP vhost; it cannot get a DNS-01 cert because `franksdank.casa` is not in the Porkbun account. The canonical Plex name is `plex.franksdank.cloud`.
+The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/`. The legacy name `dankserver.franksdank.cloud` 302-redirects there too. `plex.franksdank.casa` is the only plain-HTTP vhost; it cannot get a DNS-01 cert because `franksdank.casa` is not in the Porkbun account. The canonical Plex name is `plex.franksdank.cloud`. Both Plex vhosts dial PMS **over TLS upstream** (`transport http { tls_insecure_skip_verify }` — in this Caddy build the flag is a bare token inside the transport block, not a `tls {}` sub-block): PMS 1.43.x silently closes plain-HTTP connections from non-loopback sources (diagnosed 2026-08-26 — it accepts, reads the request, and FINs without responding), which surfaced here as an empty-body 502 (blank page in browsers). `insecure_skip_verify` applies only to this pinned-LAN hop (PMS serves a self-signed cert); client-facing Let's Encrypt TLS is unchanged. The Plex web UI is at `/web/`; `/` returns the XML `MediaContainer` index.
 
 ## Basic auth
 
@@ -83,6 +96,7 @@ Certificates and ACME state live in the caddy data volume. Back it up with the s
 
 ## Gotchas
 
+- A bare compose container is **not** boot-persistent: after the 2026-08-26 reboot Caddy stayed down for hours because `restart: unless-stopped` does not survive shutdown's explicit stop. It now runs only from its quadlet unit — do not re-add it to `compose.yaml`.
 - `admin off` disables the config file watcher: rendered changes then sit on disk while Caddy serves its stale in-memory config until the container is recreated, and nothing logs an error (2026-08-22: a broken basicauth hash survived several deploys this way). `tests/caddyfile.bats` forbids `admin off`.
 - The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged.
 - A new vhost needs three things: a Pi-hole `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]].

docs/Infrastructure/LAN-DNS.md

@@ -42,7 +42,7 @@ The G36 has no NAT loopback (hairpin). A LAN client that resolves a name to the
 
 ## Pi-hole quadlet
 
-Pi-hole FTL is a dnsmasq fork. It serves the same `addn-hosts` file the retired dnsmasq quadlet used, mounted into `/etc/dnsmasq.d/` read-only. It retired the dnsmasq quadlet on 2026-08-24.
+Pi-hole FTL is a dnsmasq fork. It serves the same `addn-hosts` file the retired dnsmasq quadlet used, mounted into `/etc/pihole/hosts/` read-only. It retired the dnsmasq quadlet on 2026-08-24.
 
 - **Location:** `dankclaw/pihole/` — `quadlet/pihole.container`, `env.example`, `addn-hosts`
 - **Ports:** 53/udp + 53/tcp (DNS) and 8083/tcp (web UI) on 192.168.0.143
@@ -91,6 +91,34 @@ The Dankclaw host itself resolves the flat names through Pi-hole:
 
 Without this, the host would resolve the names through the router to the WAN IP and time out (no hairpin).
 
+## Dankware host DNS
+
+`dankware` is a LAN client, not a Pi-hole server. Its flat names resolve through Pi-hole (`192.168.0.143`), but its resolver setup differs from Dankclaw's because it carries two `+DefaultRoute` links.
+
+- **Two default links:** `enp47s0` (ethernet, metric 100) and `wlan0` (wifi, metric 600), each getting the same DHCP DNS list `[192.168.0.143, 1.1.1.1, …]`.
+- **The flap:** `systemd-resolved` picks a different "current" resolver per link — `enp47s0` settled on the Comcast v6 upstream (→ WAN IP, times out on the LAN), `wlan0` on Pi-hole (→ LAN IP). Flat-name resolution flapped between the two on every query.
+- **The fix:** pin `192.168.0.143` as the static IPv4 resolver on both NetworkManager connections. `resolved.conf`'s own `DNS=` is a per-link fallback, so the link itself must be pinned:
+
+```bash
+for c in "Wired connection 1" FRANKFI-5G; do
+  nmcli connection modify "$c" ipv4.method auto \
+    ipv4.dns 192.168.0.143 \
+    ipv4.ignore-auto-dns yes \
+    ipv4.ignore-auto-routes no
+done
+nmcli connection up "Wired connection 1"
+nmcli connection up FRANKFI-5G
+systemctl restart systemd-resolved
+```
+
+`method:auto` keeps DHCP for the address/gateway/route (dual-default traffic path preserved); `ignore-auto-dns=yes` uses only the static resolver; `ignore-auto-routes=no` keeps the DHCP routes. Verify with a cache-clear loop — `romm.franksdank.cloud` should return `192.168.0.143` on every run:
+
+```bash
+for i in $(seq 1 10); do resolvectl flush-caches; dig romm.franksdank.cloud +short; done
+```
+
+The apex and unknown subdomains still fall through to Porkbun (→ `71.227.227.172`), so WAN clients and a bare `dig franksdank.cloud` keep working.
+
 ## WAN records (Porkbun)
 
 The Porkbun zone `franksdank.cloud` carries an A record for every vhost plus the apex, all pointing at `71.227.227.172`. TTL is 300. The zone has no wildcard record: unknown subdomains return NXDOMAIN. Caddy's DNS-01 challenges add and remove `_acme-challenge.*` TXT records through the same API during issuance. See [[Caddy]] for the TLS flow.
@@ -103,6 +131,7 @@ The Porkbun zone `franksdank.cloud` carries an A record for every vhost plus the
 |---------|-------------|
 | Name does not resolve on a new client | `resolv.conf` shows 192.168.0.1, not .143 — the G36 hand-off step is missing |
 | Name resolves to 71.227.x and times out on the LAN | Client used the router DNS — same root cause |
+| Flat name flaps between 192.168.0.143 and 71.227.x on one client | Two `+DefaultRoute` links; resolved picks a different "current" resolver per link. Pin 192.168.0.143 as static DNS on both NM connections — see [[#Dankware host DNS]] |
 | `dig @192.168.0.143` fails entirely | Pi-hole quadlet down: `systemctl --user status pihole` |
 | Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`). FTL serves it. Use the FQDN, or an `/etc/hosts` entry on the client |
 | HTTPS works from the host but not the WAN | No port forward on the G36 for that service (manual router step) |

docs/Infrastructure/Pi-hole.md

@@ -9,6 +9,7 @@ Pi-hole is the LAN DNS server for the homelab. It runs as a quadlet container on
 | `pihole` quadlet | `dankclaw/pihole/quadlet/pihole.container` | Boot-persistent container unit |
 | FTL | `docker.io/pihole/pihole:latest` | DNS engine (a dnsmasq fork) + ad blocking |
 | `addn-hosts` | `dankclaw/pihole/addn-hosts` | One A record per Caddy vhost |
+| sysctl drop-in | `dankclaw/pihole/sysctl/99-dnsmasq.conf` (repo) → `/etc/sysctl.d/` | Rootless port-53 threshold, applied at boot |
 | env file | `/home/dank/Containers/pihole/env` (host) | `ServerIP`, `TZ`, `DNS1`, `DNS2`, `WEBPASSWORD` |
 | Web UI | `pihole.franksdank.cloud` (Caddy → 8083) | Dashboard + admin, family login |
 
@@ -49,7 +50,7 @@ The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `D
 
 ## Host prerequisites
 
-1. `sysctl net.ipv4.ip_unprivileged_port_start=53` (persisted in `/etc/sysctl.d/99-dnsmasq.conf`) — rootless Podman may then publish port 53.
+1. `net.ipv4.ip_unprivileged_port_start=53`, persisted in `/etc/sysctl.d/99-dnsmasq.conf` — rootless Podman may then publish port 53. The repo file `dankclaw/pihole/sysctl/99-dnsmasq.conf` is the single source of truth: `deploy_pihole` installs it, stages any later-sorted drop-in that resets the value in the home trash, and applies it with `sysctl --system` (the same code path boot uses via systemd-sysctl). **Incident (2026-08-26):** a second drop-in (`99-rootless-ports.conf`, sorted after) reset the value to 80. The runtime override from an earlier manual `sysctl -w` died with a reboot, and `pihole.service` crash-looped on `bind: permission denied` for port 53 (restart counter 1645) until the conflict was removed.
 2. `DNSStubListener=no` in `/etc/systemd/resolved.conf` — the resolved stub would otherwise hold port 53 inside the container netns.
 
 Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
@@ -62,6 +63,7 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
 | A name does not resolve for a new client | The client's `resolv.conf` does not show `192.168.0.143` — the G36 hand-off is missing. See [[LAN-DNS]] |
 | The web UI asks for a password loop | `WEBPASSWORD` in the env file is the placeholder `changeme` — fill the real value, restart the unit |
 | Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong |
+| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: exactly one file under `/etc/sysctl.d/` may set `ip_unprivileged_port_start`, and it must say 53. Redeploy pihole — `deploy_pihole` reinstalls the drop-in and neutralizes conflicts |
 | Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
 
 ## See Also

docs/LLM/ikllama-ornith-dankserver.md

@@ -0,0 +1,262 @@
+---
+type: reference
+title: ik_llama.cpp on Dankserver (Ornith Q8_0)
+description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. Q8_0 model, KV K default f16 / V q6_0 with Hadamard on V only (iq5_k is not a KV-cache type in this build; q8_KV crashes on this arch), split-mode none, speculative decoding disabled, 3 parallel slots x 163840 ctx, the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags.
+tags:
+  - ikllama
+  - llamacpp
+  - cuda
+  - dankserver
+  - moe
+  - ornith
+  - kv-cache
+  - hadamard
+timestamp: 2026-08-26
+---
+
+# ik_llama.cpp on Dankserver (Ornith-1.5-35B-A3B CRACK)
+
+> **Active 2026-08-26.** `frank-default-small` switched from upstream
+> llama.cpp (b10121) to `ikawrakow/ik_llama.cpp` (commit `b166e269`) on the
+> RTX 3090. This page documents the build, the mandatory FA build flags, and
+> the MoE / CPU-offload runtime arguments.
+
+The model is `dealignai/Ornith-1.5-35B-A3B-UNCENSORED` (CRACK **Q8_0**), 35B-total /
+3B-active MoE-dense hybrid with an MTP head in the GGUF (mmproj f16 vision
+encoder). Served as **three parallel 160000-context slots** (`PARALLEL=3`).
+Speculative decoding is **disabled**. See [[dealign-ornith-crack]] for the
+model runbook.
+
+## Why ik_llama.cpp
+
+ik_llama.cpp is ikawrakow's llama.cpp fork with heavy CPU + CUDA
+optimizations, IQK/AVX-512 kernels, `--fit` auto-offload, and fast q8_KV
+cache. Dankserver is a CUDA (RTX 3090) + CPU-MoE-offload box, which is
+exactly the supported hybrid the fork targets. Only CPU and CUDA backends
+are fully functional; ROCm / Vulkan / Metal are unsupported upstream.
+
+## Build recipe (CUDA 13.2 via staged pip toolkit)
+
+The system CUDA toolkit on Dankserver is **removed** (`nvidia-cuda-toolkit`
+is in `rc` state; no `/usr/local/cuda*`, no `/usr/bin/nvcc`). The CUDA
+13.2 toolkit is installed via **pip** inside the vllm venv at
+`/home/dank/vllm/.venv/lib/python3.12/site-packages/nvidia/cu13/`. A staged,
+self-contained toolchain at `/home/dank/cuda13-stage` adds the unversioned
+`.so` aliases and static libs CMake needs.
+
+```bash
+# 1. Clone a pinned ik_llama.cpp
+git clone https://github.com/ikawrakow/ik_llama.cpp.git /home/dank/ik_llama.cpp
+cd /home/dank/ik_llama.cpp
+git checkout b166e269   # the version this deployment was built from
+
+# 2. Configure (stage already symlinks nvcc/include/lib for cu13)
+cmake -B build \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DGGML_CUDA=ON \
+  -DGGML_CUDA_FA_ALL_QUANTS=ON \
+  -DGGML_IQK_FA_ALL_QUANTS=ON \
+  -DCMAKE_CUDA_ARCHITECTURES=86 \
+  -DCMAKE_CUDA_COMPILER=/home/dank/cuda13-stage/bin/nvcc \
+  -DCUDAToolkit_ROOT=/home/dank/cuda13-stage \
+  -DGGML_NATIVE=ON
+
+# 3. Build
+cmake --build build --config Release -j 8
+```
+
+### ⚠️ MANDATORY: always include the FA all-quants flags
+
+> **Rule.** Any ik_llama.cpp build that serves a quantized KV cache
+> (q8_KV / q8_0 / Hadamard / any non-f16 KV) **MUST** be compiled with
+> BOTH:
+> - `-DGGML_CUDA_FA_ALL_QUANTS=ON` — compiles every K/V cache quant combo
+>   into the **CUDA** FlashAttention kernels. Without it, CUDA FA only ships
+>   a subset and **errors at runtime** telling you to recompile when a combo
+>   (e.g. q8_KV + Hadamard V-cache) isn't present. The RTX 3090 keeps KV on
+>   the GPU, so this is the flag that matters here.
+> - `-DGGML_IQK_FA_ALL_QUANTS=ON` — compiles all quant types (Q4_0/Q4_1/
+>   IQ4_NL on top of the always-on F16/Q8_0/Q6_0/BF16) into the **CPU/IQK**
+>   FA kernels. Relevant because the MoE experts run on the CPU; the KV
+>   cache that backs CPU-side attention uses the IQK FA path.
+>
+> These are the old "asymmetrical quants" compile-time gates. On a CUDA
+> host, omitting `GGML_CUDA_FA_ALL_QUANTS` is the #1 way to get a
+> "recompile with GGML_CUDA_FA_ALL_QUANTS" crash at load time.
+
+`GGML_NATIVE=ON` lets GCC target the host (the Ryzen 7 7700X is Zen4, so
+`-march=native` enables the AVX512 / VNNI IQK kernels automatically —
+verifiable with `system_info: AVX512_VNNI = 1` in the boot log).
+
+### CUDA stage note
+
+`/home/dank/cuda13-stage` is the pip `cu13` toolkit plus the symlinks CMake's
+(CUDAToolkit) finder needs (`libcudart.so`, `libcublas.so`, `libcublasLt.so`
+and static `libcudadevrt.a`/`libcudart_static.a`). It also works around the
+CCCL CUDA version skew by defining
+`-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK` in `CMAKE_CUDA_FLAGS` (the pip
+package ships nvcc 13.2.78 but reports `CUDART_VERSION 13000`, which trips
+a strict header lint). Do not re-run configure without that flag or you get
+a hard compile error.
+
+## Runtime configuration
+
+Files: `scripts/serve-frank-small.sh`, `scripts/frank-default-small.env`,
+`scripts/frank-default-small.service` in the repo. The service just calls
+the serve script; the env file holds all tunables.
+
+### KV cache: K default f16 / V q6_0 + Hadamard (V only)
+
+- `CACHE_TYPE_K=f16` — K cache at the default full-precision f16
+  (passes `--cache-type-k f16`). No Hadamard on K.
+- `CACHE_TYPE_V=q6_0` — V cache at 6-bit k-quant
+  (passes `--cache-type-v q6_0`).
+- `V_CACHE_HADAMARD=1` — `--v-cache-hadamard` on V only. The Hadamard
+  transform preserves V quality at 6-bit; K is full-precision f16 and does
+  not need it (`K_CACHE_HADAMARD=0`). Implemented for both CUDA
+  (`ggml-cuda/hadamard.cu`) and CPU (IQK) paths. Flash-attention is **on
+  by default**.
+
+> **KV cache type history on this build (`b166e269`):**
+> - `iq5_k` is **not** a KV-cache type — the cache whitelist is
+>   `f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/q5_1/q6_0/q8_KV`; `iq5_*` only
+>   exist as weight quants. `--cache-type-k iq5_k` throws `Invalid cache
+>   type`.
+> - `q8_KV` **crashes** with a `ggml view` size assert
+>   (`ggml.c:5427: data_size + view_offs > ggml_nbytes(view_src)`) on this
+>   QWEN3MOE/MTP + multimodal arch — independent of PARALLEL/speculation.
+> - The deployment settled on K at default **f16** (full precision, no
+>   Hadamard) with V at **q6_0 + Hadamard**.
+
+With 3 parallel slots at 163840 context the KV cache is large; keeping K at
+full precision f16 while quantizing only V (with Hadamard) is the chosen
+budget split. The GPU-side expert set stays reduced to fit (see the expert
+section).
+
+### Split mode: none (single GPU)
+
+- `SPLIT_MODE=none` → `--split-mode none` (the default). Dankserver has a
+  single RTX 3090 yet ik_llama rejects `--split-mode graph` on a box with a
+  single device:
+
+  > `llama error loading model: It is not possible to use split mode 'graph'
+  > with less than 2 devices`
+
+  So `graph` is **not** a harmless no-op on one GPU — it is a hard load
+  error (this bit us on the first deploy and loop the service). It also still
+  works with the `--override-tensor` expert pins.
+
+> **Lesson.** ik_llama `--split-mode graph` requires ≥2 devices. On a
+> single-GPU host you must use `none` (or `layer`, which also needs >1).
+> Do not assume `--split-mode graph` degrades gracefully on one GPU.
+
+### Expert offload (MoE: CPU with a partial GPU slice)
+
+The expert-offload pattern from upstream llama.cpp carries over: the
+`--override-tensor` regex pins the sparse expert (`ffn_*_exps`) tensors to
+the CPU buft, leaving the output-adjacent blocks 35-40 on CUDA. On the
+Q8_0 model each ffn expert tensor is **272 MiB** (~4 tensor/block ≈ 1088
+MiB/block). With `PARALLEL=3` the KV cache tripled, so most expert blocks
+are pinned to CPU; only blocks **35-40** (output-adjacent, latency- and
+quality-relevant) stay on CUDA. Full-attention blocks 3,7,11,15,19,23 have
+no exps and are untouched by the regex.
+
+- `OVERRIDE_EXPERT` — the layer-bounded regex listing the blocks whose
+  experts stay on CPU: `(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|
+  24|25|26|27|28|29|30|31|32|33|34)`. Blocks 35-40 (and the always-on
+  full-attention set) are left off the list so their experts load to CUDA.
+- `OVERRIDE_BUFT=CPU`.
+- `N_GPU_LAYERS=999` — ik_llama `-ngl` / `--gpu-layers` only accepts an
+  integer (the literal `all` is rejected). 999 means "load as many tensor
+  layers as possible", while the expert override still pins the listed exps
+  to CPU.
+
+> **Fused MoE** (`fused_moe`, `fused_up_gate`, `fused_mmad`) is **on by
+> default** in ik_llama — there is no positive flag; only `-no-fmoe`
+> disables it. `only_active_experts` is also on by default. Confirmed in the
+> boot log for the running service.
+
+To move more experts back to the GPU later, drop more block numbers from
+`OVERRIDE_EXPERT` (each block is ~1088 MiB on Q8_0; verify `nvidia-smi`
+headroom first and keep ~2-3 GiB margin for KV/compute). With the 3-slot KV
+cache there is little headroom, so the default keeps only 35-40 on GPU.
+
+### Qwen-Sharp chat template via --jinja
+
+`--chat-template-file` alone only stores the template text; ik_llama **does
+not** render a custom template as real Jinja unless `--jinja` is also given
+(`inputs.use_jinja ? common_chat_templates_apply_jinja : <non-jinja
+fallback>`). Without `--jinja` it warns it only supports "commonly used
+templates" and silently uses built-in handlers, dropping the Qwen-Sharp
+template's thinking / image / tool formatting. So the serve script passes
+both `--chat-template-file` and `--jinja` (gated on `USE_JINJA`, set to `1`).
+
+### Speculative decoding: disabled (3 parallel slots)
+
+`SPEC_TYPE=none` (passed as `--spec-type none`). Speculative decoding is
+intentionally disabled because ik_llama **limits any speculative stage to a
+a single server slot**: with `--parallel 3` a configured spec stage
+hard-exits at load with
+
+> `ERR [load_model] Speculative decoding is currently limited to a single
+> server slot (-np 1).` / `n_parallel=3`
+
+This service keeps **3 parallel slots** (one per user family via
+`PARALLEL=3`), so speculative decoding cannot be used here. MTP stays
+disabled. The model is a low-parameter MoE (3B active of 35B) and does not
+need the speculation it would cost a slot for.
+
+> **Lesson.** Do not combine speculative decoding with `--parallel > 1` in
+> ik_llama. Either drop to `-np 1` (one slot) and keep a spec stage, or keep
+> multiple slots and set `SPEC_TYPE=none`. For a multi-user service the
+> extra slots win.
+
+### Flash-attention + sampling
+
+Flash-attention is **on by default** in ik_llama (`flash_attn=true`), so no
+`-fa`/`--flash-attn` flag is needed. Sampling, thread, batch, context, and
+checkpoint tunables (`TEMPERATURE`, `TOP_K`, `TOP_P`, `MIN_P`, thread,
+`BATCH_SIZE`, `UBATCH_SIZE`, `CTX_SIZE`, `PARALLEL`, `CTX_CHECKPOINTS`,
+`NO_MMAP`) are unchanged and pass straight through.
+
+## MoE / hybrid-arg reference (not in upstream llama.cpp)
+
+Several ik_llama knobs matter for a CPU-MoE offload host:
+
+| flag | meaning | our use |
+|------|---------|---------|
+| `--split-mode graph` | split tensors+graph across devices; strong for MoE; **requires 2+ devices** | single 3090 → use `none`
+| `-ooae` / `--offload-only-active-experts` | default ON: copy **only active** experts RAM→VRAM | keep ON (huge for A3B: few experts active) |
+| `-grt q8_0` | reduce type for inter-GPU data exchange | not needed (single GPU) |
+| `--fit` | auto-offload to fit VRAM | conflicts with `--override-tensor`; do not use together (runtime error) |
+| `-n-cpu-moe N` / `--cpu-moe` | keep MoE weights of first N / all layers on CPU | also conflicts with `--fit`; our `--override-tensor` is finer-grained |
+| `--fit-margin` | VRAM safety margin for `--fit` | only relevant with `--fit` |
+
+The current deployment uses `--override-tensor` (not `--fit` or
+`--n-cpu-moe`) for the expert offload, so the `--fit`-conflict rules do not
+apply.
+
+## Rollback
+
+The previous runtime `upstream llama.cpp b10121` and its binary
+(`/home/dank/llama.cpp/build/bin/llama-server`) are **left untouched**. To
+rollback to the old stack, change `LLAMA_SERVER=` in
+`scripts/frank-default-small.env` back to that path and
+`CACHE_TYPE_K/V` back to `f16` with Hadamard off, and re-enable the ngram
+spec / `--spec-type` flags. The model file is not changed either way.
+
+## See Also
+
+- [[dealign-ornith-crack]] — the Ornith Q6_K model run
+- [[Dankserver System Specs]] — the host
+- [[deployment]] — service-scoped git deploy
+
+## Citations
+
+- [ikawrakow/ik_llama.cpp README](https://github.com/ikawrakow/ik_llama.cpp) —
+  MoE offload guidance and the `--split-mode graph` + CPU-offload warning
+- [ik_llama.cpp docs/build.md](https://github.com/ikawrakow/ik_llama.cpp/blob/main/docs/build.md) —
+  build flags incl. `GGML_CUDA_FA_ALL_QUANTS` and `GGML_IQK_FA_ALL_QUANTS`
+- [ik_llama.cpp docs/parameters.md](https://github.com/ikawrakow/ik_llama.cpp/blob/main/docs/parameters.md) —
+  q8_KV (PR 208), K/V Hadamard options, `-cuda graphs=0` workaround, and
+  `--split-mode graph` MoE note (PR 1080)

docs/LLM/index.md

@@ -1,7 +1,7 @@
 # LLM Fleet — Knowledge Base
 
 Documentation for the LLM inference fleet on two local machines
-(2026-09-01): upstream llama.cpp on Dankserver (frank-default-small,
+(2026-08-26): ik_llama.cpp on Dankserver (frank-default-small,
 Ornith-1.5-35B-A3B CRACK) and ROCmFPX llama.cpp on Dankclaw
 (frank-default-large, kingjones Qwen3.8 Uncensored Q6 AGENT). Both
 services run the Qwen-Sharp chat template.
@@ -17,24 +17,24 @@ services run the Qwen-Sharp chat template.
 graph LR
     subgraph Dankserver[NVIDIA Dankserver - Ubuntu 26.04]
         A[RTX 3090 24GB + 62GB RAM]
-        B[upstream llama.cpp b10121
+        B2[ik_llama.cpp build b166e269
 frank-default-small, port 7770]
         C[Ornith-1.5-35B-A3B-CRACK Q6_K
-ngram-mod, selected experts offloaded via override, 491520 ctx]
-        A --> B --> C
+no speculation, q8_0 + Hadamard cache, 491520 ctx]
+        A --> B2 --> C
     end
 
     subgraph Dankclaw[AMD Strix Halo - Ubuntu 26.04]
-        D[gfx1151 iGPU 64 GiB unified memory]
+        E2[gfx1151 iGPU 64 GiB unified memory]
         E[charlie12345/ROCmFPX build-kairic (v81)
 frank-default-large, port 7770]
         F[Qwen3.8 Uncensored Q6 ROCmFPX AGENT
 MTP draft + Turbo4 V cache, 262K ctx / 1 slot]
-        D --> E --> F
+        E2 --> E --> F
     end
 
-    I[LLM Clients / API] -->|HTTP :7770| B
-    I -->|HTTP :7770| E
+    CLI[LLM Clients / API] -->|HTTP :7770| B2
+    CLI -->|HTTP :7770| E
 ```
 
 ## Servers
@@ -56,15 +56,16 @@ MTP draft + Turbo4 V cache, 262K ctx / 1 slot]
   on Dankclaw (retired)
 - [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on
   Dankserver (frank-default-small, active 2026-08-24)
+- [[ikllama-ornith-dankserver]] — the ik_llama.cpp runtime for
+  frank-default-small (2026-08-26): q8_0 + Hadamard KV cache,
+  split-mode none (single-GPU), mandatory FA_ALL_QUANTS build flags
 - [[kingjones-strix-mtp]] — the active kingjones Qwen3.8-27B ROCmFP4 STRIX MTP
   model on Dankclaw (restored 2026-09-01 after the Q8_0 AGENT experiment)
-- [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on
-  Dankserver (frank-default-small, active 2026-08-24)
 - [[ciru-activefpx-promptforge]] — the RETIRED v3 CIRU PromptForge runtime
   on Dankclaw (superseded 2026-08-24 by the kingjones ROCmFP4 model)
 - [[kairic-edge]] — the retired Kairic Edge IU4 experiment
   (deployed and reverted 2026-08-22)
-- [[Dankserver System Specs]] — the upstream llama.cpp box (frank-default-small)
+- [[Dankserver System Specs]] — the RTX 3090 box (frank-default-small)
 - [[Dankclaw System Specs]] — the CIRU ROCmFPX box (frank-default-large)
 
 ## CIRU Reference (Dankclaw)

docs/LLM/ubatch-benchmark-large.md

@@ -23,8 +23,8 @@ for auditability only. Their results are invalid and must not be used.
 ## Sweep
 
 The requested points are `256, 512, 768, 1024, 1280, 1536, 1792, 2048,
-3072, 4096` at 32K context. Each point runs three 128-token repetitions by
-default.
+3072, 4096` at 8K context, with `batch` equal to `ubatch` at every point.
+Each point runs three 128-token repetitions by default.
 
 Run on Dankclaw from the repository checkout:
 
@@ -34,8 +34,8 @@ sudo systemctl stop frank-default-large
 sudo systemctl start frank-default-large
 ```
 
-The script uses the active Q6 AGENT model, a fixed 32K prompt workload, and
-`llama-bench` JSONL output. The ubatch-only sweep is:
+The script uses the active Q6 AGENT model, an 8K prompt workload, equal
+`batch` and `ubatch` values, and `llama-bench` JSONL output. The sweep is:
 ```text
 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 3072, 4096
 ```

docs/index.md

@@ -62,6 +62,7 @@ covers both workloads).
 - [[OpenWebUI]] — chat UI for local LLMs
 - [[Hermes]] — the Nous Research agent
 - [[Hermes-Dashboard]] — the Hermes web management UI (config, keys, sessions)
+- [[Hermes-WebUI]] — the Hermex server (loopback API for the iPhone app, SSH-tunneled)
 - [[Gitea]] — Git hosting (migrated)
 - [[Postgres]] — Gitea database backend
 
@@ -110,9 +111,10 @@ The LLM inference fleet on the two hosts. Full index:
 - [[ciru-activefpx-promptforge]] — the RETIRED v3 CIRU PromptForge runtime on Dankclaw (superseded 2026-08-24)
 - [[kingjones-strix-mtp]] — the kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model on Dankclaw (active)
 - [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on Dankserver (active)
+- [[ikllama-ornith-dankserver]] — the ik_llama.cpp runtime for frank-default-small (q8_0 + Hadamard KV cache, split-mode none; 2026-08-26)
 - [[kairic-edge]] — the retired Kairic Edge IU4 experiment (2026-08-22)
 - [[dflash2-large-vulkan]] — the retired DFlash2 Vulkan deployment (2026-08-21)
-- [[Dankserver System Specs]] — the upstream llama.cpp box hardware
+- [[Dankserver System Specs]] — the RTX 3090 box hardware
 - [[Dankclaw System Specs]] — the CIRU ROCmFPX box hardware
 - [[Dankclaw Improvements]] — host-level changes on the CIRU box
 
@@ -158,6 +160,10 @@ The LLM inference fleet on the two hosts. Full index:
 - [[Sample Script Set]] — reference scripts for the PP pair
 - [[vLLM Pipeline Parallelism]] — vLLM's PP implementation
 
+## Update Log
+
+- [[log]] — chronological record of knowledge-base changes
+
 ## See Also
 
 - [AGENTS.md](../AGENTS.md) — repository rules

docs/log.md

@@ -0,0 +1,39 @@
+---
+type: reference
+title: Bundle Change Log
+description: Chronological record of meaningful changes to the serverAppConfigs knowledge base.
+tags:
+  - changelog
+  - documentation
+  - maintenance
+timestamp: 2026-09-14
+---
+
+# Bundle Change Log
+
+## 2026-09-14
+- **Creation**: [[Hermes-WebUI]] — new `hermes-webui` compose service on Dankclaw (the Hermex server): runs `hermes-webui`'s `bootstrap.py --foreground` in the agent image, shares `/home/dank/hermes-data` with the gateway (identical rootless Podman uid mapping), publishes `127.0.0.1:8787` only, password-gated via `HERMES_WEBUI_PASSWORD` in the host `.env`.
+- `/dankclaw/compose.yaml`, `/dankclaw/.env.example` — service definition and password placeholder; container-internal bind is `0.0.0.0` (Podman's port forwarder dials the bridge IP) while the host-side publish stays loopback so the SSH tunnel is the sole ingress.
+- `/scripts/deploy-plan.py`, `/scripts/deploy.sh`, `/scripts/deploy-remote.sh` — `hermes-webui` registered as an isolated Dankclaw service in all three inventories; planner narrows edits to its compose block only.
+- `/tests/hermes_webui.bats` — new seam: foreground entrypoint, loopback publish + auth, shared volume, `/health` healthcheck, env placeholder, planner narrowing (hermetic fixture repo), deployer dispatch.
+- Remote access pattern documented: iPhone Termius local forward `127.0.0.1:8787 -> 127.0.0.1:8787` over the existing router port forward to Dankclaw `:22`; Hermex points at `http://localhost:8787`. No Tailscale, no Cloudflare.
+
+## 2026-08-26
+- `/dankclaw/Caddyfile` — Both Plex vhosts now proxy to PMS over a TLS upstream (`insecure_skip_verify`); PMS drops plain-HTTP connections from non-loopback sources, which produced empty-body 502s.
+- `/docs/Infrastructure/Caddy.md` — Documented the Plex TLS-upstream requirement and the `/web/` vs `/` distinction.
+- `/tests/caddyfile.bats` — Added a test pinning the TLS upstream transport on both Plex vhosts.
+- `/docs/Dankclaw/Gitea.md` — Documented migration of Gitea web proxy ownership to canonical Caddy and retirement of the standalone proxy.
+- `/docs/Dankclaw/Dankclaw Stack.md` — Updated boot-persistence and Caddy deployment descriptions for the two-unit Gitea stack.
+- `/docs/Architecture/Deployment Flow.md` — Replaced the split-proxy safety description with the canonical Caddy ownership model.
+- `/dankclaw/gitea/quadlet/gitea-caddy.container` — Retired the standalone Gitea Caddy Quadlet; deployment stages any live copy into home trash.
+- `/scripts/deploy-plan.py` — Routes retirement of the legacy proxy to the canonical Caddy and removes it from the service inventory.
+- `/scripts/deploy.sh` and `/scripts/deploy-remote.sh` — Removed standalone proxy dispatch and added idempotent retirement before canonical Caddy deployment.
+- `/tests/caddyfile.bats`, `/tests/deployment_scoping.bats`, `/tests/deployment_flow.bats`, `/tests/frank_large_config.bats` — Added migration coverage and corrected the existing ubatch harness assertion.
+- `/docs/index.md` — Bundle index remains the root navigation entry for the updated pages.
+- **Boot-persistence audit (post-reboot, 2026-08-26 evening):** after Frank's reboot, Caddy and Pi-hole were both down. Caddy ran as a bare compose container (`restart: unless-stopped` does not survive shutdown's explicit stop; rootless Podman has no boot-time auto-start for bare containers). Pi-hole's quadlet existed but crash-looped (restart counter 1645): a later-sorted sysctl drop-in (`99-rootless-ports.conf`) reset `net.ipv4.ip_unprivileged_port_start` to 80, so the rootless port-53 publish failed with `bind: permission denied`.
+- `/AGENTS.md` — new MANDATORY section "Boot Persistence & Self-Healing": every deployed, proven-working container must have a self-healing, boot-persistent quadlet unit (`WantedBy=default.target` + enabled + linger, `Restart=always`, repo-provenance `.container`), verified before a service is declared deployed.
+- `/dankclaw/caddy/quadlet/caddy.container`, `/dankclaw/caddy/env.example` — Caddy is now a boot-persistent quadlet (`caddy.service`): joins `dankclaw_default` so container-name upstreams keep resolving, `Pull=never` (local-only image), Porkbun credentials via `/home/dank/Containers/caddy/env` (deploy writes it from `.env`), bare-IP-catch-all health check. The caddy service was removed from `/dankclaw/compose.yaml`.
+- `/dankclaw/pihole/sysctl/99-dnsmasq.conf` — repo source of truth for `ip_unprivileged_port_start=53`; `deploy_pihole` installs it to `/etc/sysctl.d/`, stages conflicting later-sorted drop-ins in the home trash, and applies `sysctl --system` (the boot code path).
+- `/scripts/deploy-remote.sh` — `deploy_caddy` installs/enables/starts the quadlet (retiring the bare container; all state is in host volumes); `deploy_pihole` enforces the sysctl prerequisite.
+- `/tests/caddy_quadlet.bats` (new seam), `/tests/caddyfile.bats`, `/tests/deployment_flow.bats`, `/tests/pihole.bats` — quadlet boot-persistence, self-healing, network, env, and sysctl coverage; compose assertions repointed from the removed caddy service to the quadlet.
+- `/docs/Infrastructure/Caddy.md`, `/docs/Infrastructure/Pi-hole.md`, `/docs/Dankclaw/Dankclaw Stack.md` — quadlet runtime facts, the 2026-08-26 incident, and the sysctl-conflict failure mode.

scripts/bench-ubatch.sh

@@ -2,9 +2,8 @@
 # Benchmark ubatch with llama-bench for frank-default-large.
 #
 # This is intentionally a ubatch-only benchmark. It uses one llama-bench
-# process, a fixed 32K prompt workload, and a fixed scheduler size large
-# enough to hold that workload. No server process, HTTP requests, or log
-# scraping are used.
+# process, an 8K prompt workload, and equal batch/ubatch values. No server
+# process, HTTP requests, or log scraping are used.
 #
 # Run on Dankclaw after stopping frank-default-large:
 #   sudo systemctl stop frank-default-large
@@ -15,7 +14,7 @@ set -euo pipefail
 : "${LLAMA_BENCH:=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-bench}"
 : "${MODEL:=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
 : "${UBATCH_VALUES:=256,512,768,1024,1280,1536,1792,2048,3072,4096}"
-: "${CONTEXT:=32768}"
+: "${CONTEXT:=8192}"
 : "${GEN_TOKENS:=128}"
 : "${REPETITIONS:=3}"
 : "${DEVICE:=ROCm0}"
@@ -34,7 +33,7 @@ exec "$LLAMA_BENCH" \
     --model "$MODEL" \
     --n-prompt "$CONTEXT" \
     --n-gen "$GEN_TOKENS" \
-    --batch-size "$CONTEXT" \
+    --batch-size "$UBATCH_VALUES" \
     --ubatch-size "$UBATCH_VALUES" \
     --device "$DEVICE" \
     --n-gpu-layers 999 \

scripts/deploy-plan.py

@@ -27,10 +27,10 @@ SERVICE_ORDER = (
     "open-webui",
     "gitea-postgres",
     "gitea-server",
-    "gitea-caddy",
     "comfyui",
     "hermes",
     "hermes-dashboard",
+    "hermes-webui",
     "prometheus",
     "grafana",
     "loki",
@@ -59,6 +59,7 @@ SERVICE_ORDER = (
 )
 
 DANKCLAW_SERVICES = {
+    "hermes-webui",
     "gluetun",
     "searxng-valkey",
     "searxng",
@@ -66,7 +67,6 @@ DANKCLAW_SERVICES = {
     "open-webui",
     "gitea-postgres",
     "gitea-server",
-    "gitea-caddy",
     "comfyui",
     "hermes",
     "hermes-dashboard",
@@ -104,6 +104,7 @@ DANKCLAW_COMPOSE_SERVICES = {
     "comfyui",
     "hermes",
     "hermes-dashboard",
+    "hermes-webui",
     "prometheus",
     "grafana",
     "loki",
@@ -276,9 +277,12 @@ def services_for_path(path: str, target: str) -> set[str]:
                 elif name == "gitea-server.container":
                     _add(services, "gitea-server")
                 elif name == "gitea-caddy.container":
-                    _add(services, "gitea-caddy")
+                    # Removing the old split proxy requires the canonical
+                    # Caddy handler to retire its installed unit/container
+                    # before it claims ports 80/443.
+                    _add(services, "caddy")
                 elif name == "gitea.network":
-                    _add(services, "gitea-postgres", "gitea-server", "gitea-caddy")
+                    _add(services, "gitea-postgres", "gitea-server")
             return services
         return services
 

scripts/deploy-remote.sh

@@ -84,8 +84,8 @@ service_is_allowed() {
     case "$TARGET:$service" in
         dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
         dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
-        dankclaw:gitea-caddy|dankclaw:comfyui|dankclaw:hermes|\
-        dankclaw:hermes-dashboard|dankclaw:prometheus|dankclaw:grafana|\
+        dankclaw:comfyui|dankclaw:hermes|\
+        dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
         dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
         dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\
@@ -248,6 +248,45 @@ deploy_homepage() {
     run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
 }
 
+stage_legacy_gitea_caddy_state() {
+    local legacy_state="$STATE_DIR/gitea-caddy"
+    [[ -e "$legacy_state" ]] || return 0
+    local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
+    mkdir -p "$legacy_trash"
+    mv -- "$legacy_state" "$legacy_trash/gitea-caddy.state"
+}
+
+retire_legacy_gitea_caddy() {
+    # The old split proxy owns ports 80/443 on hosts upgraded in place. Stop
+    # and disable its generated unit before the canonical Caddy is started.
+    if systemctl --user is-active --quiet gitea-caddy.service 2>/dev/null; then
+        systemctl --user stop gitea-caddy.service
+    fi
+    if systemctl --user is-enabled --quiet gitea-caddy.service 2>/dev/null; then
+        # A Quadlet-generated unit may report "generated" without having an
+        # enablement symlink. Disable is therefore best-effort; moving the
+        # source out of the generator directory is the durable action.
+        systemctl --user disable gitea-caddy.service 2>/dev/null || true
+    fi
+    local unit_dir="$HOME/.config/containers/systemd"
+    local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
+    # Podman generators scan the unit directory recursively on this host, so
+    # leaving the retired file in systemd/.AGENTTRASH would regenerate it.
+    # Stage it outside the unit tree in the dated home trash instead.
+    for unit_file in "$unit_dir/gitea-caddy.container" \
+        "$unit_dir/.AGENTTRASH/gitea-caddy.container"; do
+        if [[ -e "$unit_file" ]]; then
+            mkdir -p "$legacy_trash"
+            mv -- "$unit_file" "$legacy_trash/"
+        fi
+    done
+    stage_legacy_gitea_caddy_state
+    systemctl --user daemon-reload
+    if podman container exists gitea-caddy; then
+        podman rm --force gitea-caddy
+    fi
+}
+
 deploy_caddy() {
     if [[ "$TARGET" != dankclaw ]]; then
         echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
@@ -263,24 +302,30 @@ deploy_caddy() {
         podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
             -f dankclaw/caddy/Containerfile .
     fi
-    # The canonical reverse proxy may be gitea-caddy on a host with the old
-    # split deployment. Never stop or replace it implicitly. Refuse the Caddy
-    # deployment until the operator has explicitly migrated port ownership.
-    if podman container exists gitea-caddy && ! podman container exists caddy; then
-        echo "FAIL: gitea-caddy owns the reverse-proxy deployment; refusing to replace it with caddy." >&2
-        return 1
-    fi
-    if podman container exists gitea-caddy && podman container exists caddy; then
-        local gitea_state caddy_state
-        gitea_state="$(podman inspect gitea-caddy --format '{{.State.Status}}' 2>/dev/null || true)"
-        caddy_state="$(podman inspect caddy --format '{{.State.Status}}' 2>/dev/null || true)"
-        if [[ "$gitea_state" == running && "$caddy_state" != running ]]; then
-            echo "FAIL: gitea-caddy currently owns ports 80/443; refusing to start caddy." >&2
-            return 1
-        fi
+    # Retire the old split proxy before claiming ports 80/443. This is
+    # idempotent and leaves the old unit recoverable in the target trash.
+    retire_legacy_gitea_caddy
+    # Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The
+    # live file holds secrets, so it exists only on the host; the quadlet
+    # reads it via EnvironmentFile.
+    mkdir -p /home/dank/Containers/caddy
+    if [[ ! -f /home/dank/Containers/caddy/env ]]; then
+        awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \
+            dankclaw/.env > /home/dank/Containers/caddy/env
     fi
+    local var
+    for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do
+        grep -q "^${var}=." /home/dank/Containers/caddy/env \
+            || { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
+    done
+    # Retire the bare compose container: all state (Caddyfile, certs, config)
+    # lives in host volumes, so removing the container loses nothing.
+    podman stop caddy >/dev/null 2>&1 || true
+    podman rm caddy >/dev/null 2>&1 || true
+    install_quadlet caddy dankclaw/caddy/quadlet caddy
+    systemctl --user enable caddy.service
+    systemctl --user restart caddy.service
     if podman container exists caddy; then
-        podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy
         local reloaded=0
         for _ in 1 2 3 4 5; do
             if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
@@ -295,8 +340,6 @@ deploy_caddy() {
             echo "Caddy reload failed; restarting only caddy."
             podman restart caddy
         fi
-    else
-        echo "WARN: caddy container is absent; no proxy was stopped or replaced."
     fi
 }
 
@@ -311,7 +354,7 @@ deploy_compose_service() {
         return 0
     fi
     case "$service" in
-        comfyui|hermes|hermes-dashboard|prometheus|grafana|loki|alloy|node-exporter|authentik-postgres|authentik-server|authentik-worker)
+        comfyui|hermes|hermes-dashboard|hermes-webui|prometheus|grafana|loki|alloy|node-exporter|authentik-postgres|authentik-server|authentik-worker)
             case "$service" in
                 comfyui|loki|alloy)
                     build_compose_image "$service"
@@ -405,6 +448,19 @@ deploy_open_webui() {
 }
 
 deploy_pihole() {
+    # Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it
+    # must hold at boot (systemd-sysctl applies /etc/sysctl.d). The repo
+    # drop-in is the single source of truth; a later-sorted file that resets
+    # the value to 80 took LAN DNS down after the 2026-08-26 reboot, so stage
+    # any such conflict in the home trash instead of deleting it.
+    run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \
+        /etc/sysctl.d/99-dnsmasq.conf
+    if grep -qs 'ip_unprivileged_port_start' /etc/sysctl.d/99-rootless-ports.conf; then
+        local sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-rootless-ports"
+        mkdir -p "$sysctl_trash"
+        run_privileged mv /etc/sysctl.d/99-rootless-ports.conf "$sysctl_trash/"
+    fi
+    run_privileged sysctl --system >/dev/null
     mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
     if [[ ! -f /home/dank/Containers/pihole/env ]]; then
         cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
@@ -482,7 +538,6 @@ deploy_service() {
         pihole) deploy_pihole ;;
         gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
         gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
-        gitea-caddy) deploy_gitea_unit gitea-caddy gitea-caddy ;;
         frank-default-large) deploy_llm_large ;;
         frank-default-small) deploy_llm_small ;;
         frank-default-npu) deploy_llm_npu ;;

scripts/deploy.sh

@@ -115,8 +115,8 @@ valid_service() {
     case "$TARGET:$service" in
         dankclaw:gluetun|dankclaw:searxng-network|dankclaw:searxng-valkey|dankclaw:searxng|\
         dankclaw:pihole|dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
-        dankclaw:gitea-caddy|dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\
-        dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
+        dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\
+        dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
         dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
         dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\
         dankclaw:frank-default-large|dankclaw:frank-default-npu|\

scripts/frank-default-large.env

@@ -1,30 +1,42 @@
-# Tunables for frank-default-large (kingjones Qwen3.8 Uncensored Q6 AGENT,
-# dankclaw, Strix Halo gfx1151).
+# Tunables for frank-default-large (kingjones777 Qwen3.8-Flash-Next
+# ROCmFP4 STRIX_LEAN, qwen4exp hybrid MoE, Dankclaw, Strix Halo gfx1151).
 #
-# Runtime: ROCmFPX llama.cpp with TurboQuant support. Serves the kingjones
-# AGENT GGUF plus its standalone MTP draft and vision projector. Alias
-# frank-default-large, port 7770.
+# Runtime: ROCmFPX llama.cpp fork with PR #27742 (qwen4exp architecture)
+# merged and built at /home/dank/GitPackages/ROCmFPX/build-qwen38.
+# Build flags: -DGGML_HIP=ON -DGGML_HIP_ROCMI4_W4A4=ON
+# -DCMAKE_HIP_ARCHITECTURES=gfx1151 -DGGML_CUDA_FA_ALL_QUANTS=ON
+# -DLLAMA_BUILD_WEBUI=OFF -DCMAKE_EXE_LINKER_FLAGS=-no-pie.
+# Replaced the kingjones Qwen3.8-27B Q6_0 AGENT deployment on 2026-09-01.
+#
+# Model: kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF.
+# 56B MoE (512 experts, 10 used), 48 layers (12 attention + 36 SSM,
+# full_attention_interval=4), 2 KV heads × 256 dim, embedding 2560,
+# QSA indexer (4 heads, key_length 128, top_k 2048), hyper-connections
+# (count=4, low_rank=320), PLE (layer 1, ngram_size=3, heads_per_ngram=8).
+# Q4_0_ROCMFP4_STRIX_LEAN quant, 98.5 GiB, 3-part GGUF split.
+# Text-only (no mmproj, no MTP draft head).
+#
+# Context: 131072 (128k) × 1 slot.  f16 KV cache.  --fit off because the
+# auto-fit code segfaults on this model's hyper-connection/indexer config;
+# -ngl 999 places all layers on the SVM-backed GPU pool (116 GiB unified).
+#
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
+# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
+# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
+# env vars.
 
-# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm
-# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident).
-# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars.
+# ROCmFPX llama-server (PR #27742 merged, build-qwen38).
+LLAMA_BIN=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server
 
-# PromptForge ROCmFPX build and its historical ROCm 7.15 SDK environment.
-LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
-ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel
-ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries
-ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core
+# Model files from
+# https://huggingface.co/kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF
+MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN
+MODEL_FILE=Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-00001-of-00003.gguf
 
-# Model files from https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF.
-MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP
-MODEL_FILE=Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf
-MTP_DRAFT_FILE=$MODEL_DIR/mtp-Qwen3.8-27B-Q4_0.gguf
-MMPROJ_FILE=mmproj-Qwen3.8-27B-Q8_0.gguf
-
-# Qwen-Sharp chat template (tracked in the repo, staged on the host). The Sharp
-# template drives thinking on by default (medium effort); the server emits
-# reasoning in DeepSeek format so clients receive reasoning_content separately
-# instead of literal <think> tags in content.
+# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
+# Sharp template drives thinking on by default (medium effort); the server
+# emits reasoning in DeepSeek format so clients receive reasoning_content
+# separately instead of literal thinking tags in content.
 CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
 CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
 
@@ -33,14 +45,12 @@ ALIAS=frank-default-large
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
-# Serving: 262144 context (the model's maximum), 1 slot. f16 KV, context
-# checkpoints, prompt cache, standalone MTP draft on ROCm0. The full 262144
-# context KV cache plus the ~28-30 GB Q8 model + MTP draft + mmproj fit in the
-# lifted 114 GiB TTM/GTT budget on the Strix Halo (see the GTT 114 GiB change).
-# Concurrent requests queue; the server never preempts a running generation.
-CTX_SIZE=262144
+# Serving: 131072 context (128k) × 1 slot. f16 KV, context checkpoints,
+# prompt cache. The model is 98.5 GiB loaded via mmap into the Strix Halo
+# unified memory pool (116 GiB SVM-backed). --fit off avoids the auto-fit
+# segfault on this model's qwen4exp hyper-connection/indexer configuration.
+CTX_SIZE=131072
 PARALLEL=1
-# Batch/ubatch follow the existing qualified Dankclaw profile.
 BATCH_SIZE=4096
 UBATCH_SIZE=4096
 NUM_THREADS=16
@@ -49,12 +59,7 @@ CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
 CTX_CHECKPOINTS=16
 CACHE_RAM=8192
-
-# Standalone MTP draft head, loaded from MTP_DRAFT_FILE by the serve script.
-MTP_N_MAX=4
-MTP_N_MIN=0
-MTP_P_MIN=0.0
-MTP_P_SPLIT=0.10
+FIT=off
 
 # Sampling (Sharp default).
 TEMPERATURE=0.7
@@ -69,8 +74,7 @@ PRESENCE_PENALTY=1.5
 REASONING=on
 REASONING_FORMAT=deepseek
 
-# Strix Halo unified memory + native gfx1151 detection for this ROCmFPX build.
+# Strix Halo unified memory + native gfx1151.
 HSA_OVERRIDE_GFX_VERSION=11.5.1
-GGML_HIP_ENABLE_UNIFIED_MEMORY=1
 
-VERBOSITY=4
+VERBOSITY=4

scripts/frank-default-large.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Kairic Edge ROCmFPX llama.cpp Qwen3.8-27B IU4 + MTP (dankclaw, frank-default-large)
+Description=ROCmFPX llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX_LEAN 56B MoE (dankclaw, frank-default-large)
 After=network-online.target
 Wants=network-online.target
 

scripts/frank-default-small.env

@@ -1,77 +1,71 @@
-# Tunables for frank-default-small (Ornith-1.5-35B-A3B CRACK Q6_K, MTP MoE,
-# Dankserver, RTX 3090).
+# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL,
+# qwen35moe hybrid MoE, Dankserver, RTX 3090).
 #
-# Runtime: upstream llama.cpp llama-server (b10121). RTX 3090 24 GB; Ryzen
-# 7 7700X; 62 GB RAM. MoE expert weights pinned to CPU with an explicit
-# --override-tensor regex (not --n-cpu-moe). Replaced the HauhauCS
-# Qwen3.6-35B-A3B model on 2026-08-24. Switched from CRACK Q8_0 to Q6_K on
-# 2026-08-25.
+# Runtime: upstream llama.cpp (latest master, CUDA build with
+# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090
+# 24 GB; Ryzen 7 7700X; 62 GB RAM.  Replaced the ik_llama.cpp Ornith
+# CRACK Q8_0 deployment on 2026-09-01.
+#
+# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant
+# (30 GB).  Architecture: qwen35moe — 40 layers, full_attention_interval=4
+# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256
+# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000.
+# Text-only (no mmproj).  Built-in chat template in the GGUF.
+#
+# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
+# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card.
+#
+# KV cache: Q8_0 for both K and V.  With CTX_SIZE=200700 (66 900 per slot
+# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the
+# requested 2 GB VRAM buffer.
 #
 # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
 # ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
 # incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
 
-# Model files (Q6_K CRACK GGUF + f16 mmproj vision encoder + MTP head bundled
-# in the GGUF). Dealign2026-08-24.
-MODEL_DIR=/home/dank/models/Ornith-1.5-35B-A3B-CRACK
-MODEL_FILE=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf
-MMPROJ_FILE=mmproj-Ornith-1.5-35B-A3B-f16.gguf
-# Upstream llama.cpp binary (b10121).
-LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
+# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid).
+MODEL_DIR=/home/dank/models/tiel-coder-35b
+MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf
 
-# Qwen-Sharp chat template from peculiar-ragdoll (tracked in the repo,
-# staged on the host).
-# Source: https://huggingface.co/peculiar-ragdoll/Qwen-Sharp-Chat-Templates
-CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
+# Upstream llama.cpp llama-server (CUDA build, latest master).
+# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86
+# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage.
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
 
 ALIAS=frank-default-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
-# 3 parallel slots x 163840 ctx each (262144 lowered by 96k per stream for
-# cache and checkpoint headroom) = 491520 total context pool. The per-slot
-# 163840 is advertised in models.json.
-CTX_SIZE=491520
+# Total context budget across all parallel slots.  200700 / 3 = 66 900
+# per slot.  The model's context_length is 262 144, so 66 900 is well
+# within range.
+CTX_SIZE=200700
 PARALLEL=3
 
-# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
-SPEC_TYPE=ngram-mod
-SPEC_NGRAM_MOD_N_MIN=48
-SPEC_NGRAM_MOD_N_MAX=64
-SPEC_NGRAM_MOD_N_MATCH=24
+# Expert offload: all MoE expert weights pinned to CPU RAM.
+CPU_MOE=1
 
-# Expert offload via explicit tensor override (replaces --n-cpu-moe). The Q6_K
-# layer audit found 630 MiB of expert tensors per block. Keep every
-# full-attention block plus every block 35-40 on CUDA. The unique GPU set is
-# 3,7,11,15,19,23,27,31,35,36,37,38,39,40. Move all other expert tensors to
-# CPU. All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the
-# CPU buft name on this host.
-OVERRIDE_EXPERT='blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps'
-OVERRIDE_BUFT=CPU
-N_GPU_LAYERS=all
+# All non-expert layers on GPU (auto-fit will determine the exact count;
+# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room).
+N_GPU_LAYERS=999
 
-# KV cache on GPU, both K and V at f16.
-CACHE_TYPE_K=f16
-CACHE_TYPE_V=f16
+# KV cache: Q8_0 for both K and V.  ~2 GB total for 200700 ctx × 3 slots
+# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V).
+CACHE_TYPE_K=q8_0
+CACHE_TYPE_V=q8_0
 
-VERBOSITY=4
-# Store context checkpoints in RAM to hold peak GPU memory down.
-CTX_CHECKPOINTS=16
+VERBOSITY=2
 
-# Prefill batch sizing. ubatch 1024 is selected for reduced compute-memory
-# pressure and predictable latency under the ngram-mod deployment.
+# Prefill batch sizing.
 BATCH_SIZE=4096
 UBATCH_SIZE=1024
 
-# Eager RAM load (expert weights on CPU; ~31 GB RAM).
-NO_MMAP=1
-
-# Sampling (Dealign Ornith model card, thinking-general).
+# Sampling.
 TEMPERATURE=1.0
 TOP_K=20
 TOP_P=0.95
 MIN_P=0.0
 
-# Compute threads lowered to 8 so other processes keep a core.
+# Compute threads lowered so other processes keep a core.
 THREADS=7
 THREADS_HTTP=8

scripts/frank-default-small.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama.cpp Ornith-1.5-35B-A3B CRACK Q6_K ngram-mod + Qwen-Sharp template (dankserver, frank-default-small)
+Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small)
 After=network-online.target
 Wants=network-online.target
 

scripts/serve-frank-large.sh

@@ -1,6 +1,29 @@
 #!/usr/bin/env bash
-# Serve the kingjones Qwen3.8 Uncensored Q6 AGENT model as frank-default-large.
-# The deployment uses a standalone MTP draft head and f16 KV cache.
+# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN as
+# frank-default-large via the charlie12345/ROCmFPX llama.cpp fork with
+# PR #27742 (qwen4exp architecture) merged, on Dankclaw (Strix Halo
+# gfx1151, unified memory).
+#
+# Runtime: ROCmFPX llama.cpp built at
+# /home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server
+# with -DGGML_HIP=ON -DGGML_HIP_ROCMI4_W4A4=ON
+# -DCMAKE_HIP_ARCHITECTURES=gfx1151 -DLLAMA_BUILD_WEBUI=OFF.
+#
+# Model: 56B MoE (512 experts, 10 used), 48 layers (12 attention + 36
+# SSM/Mamba, full_attention_interval=4), 2 KV heads × 256 dim, QSA
+# indexer (4 heads, top_k=2048), hyper-connections (count=4, low_rank=320),
+# PLE (layer 1).  Q4_0_ROCMFP4_STRIX_LEAN quant, 98.5 GiB, 3-part split.
+#
+# Context: 131072 (128k) × 1 slot.  f16 KV cache.  Model is loaded with
+# mmap (default) — the 98.5 GiB file is memory-mapped; the Strix Halo
+# unified memory (116 GiB SVM-backed) provides the GPU address space.
+# --fit off is set because the auto-fit code segfaults on this model's
+# hyper-connection/indexer configuration; -ngl 999 places all layers on
+# the GPU (SVM-backed), which the unified memory pool handles.
+#
+# The Qwen-Sharp chat template (scripts/chat-template.jinja) is applied at
+# runtime so the model emits reasoning in DeepSeek format; the reasoning
+# trail is preserved across turns via CHAT_TEMPLATE_KWARGS.
 set -euo pipefail
 
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
@@ -12,15 +35,16 @@ if [[ -f "$ENV_FILE" ]]; then
     set +a
 fi
 
-: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}"
-: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP}"
-: "${MODEL_FILE:=Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
-: "${MTP_DRAFT_FILE:=$MODEL_DIR/mtp-Qwen3.8-27B-Q4_0.gguf}"
-: "${MMPROJ_FILE:=mmproj-Qwen3.8-27B-Q8_0.gguf}"
+# --- defaults (used when the env file does not set them) -------------------
+: "${LLAMA_BIN:=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server}"
+: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN}"
+: "${MODEL_FILE:=Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-00001-of-00003.gguf}"
+: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
+: "${CHAT_TEMPLATE_KWARGS:={}}"
 : "${ALIAS:=frank-default-large}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=262144}"
+: "${CTX_SIZE:=131072}"
 : "${PARALLEL:=1}"
 : "${BATCH_SIZE:=4096}"
 : "${UBATCH_SIZE:=4096}"
@@ -30,21 +54,18 @@ fi
 : "${CACHE_TYPE_V:=f16}"
 : "${CTX_CHECKPOINTS:=16}"
 : "${CACHE_RAM:=8192}"
-: "${MTP_N_MAX:=4}"
-: "${MTP_N_MIN:=0}"
-: "${MTP_P_MIN:=0.0}"
-: "${MTP_P_SPLIT:=0.10}"
+: "${FIT:=off}"
 : "${TEMPERATURE:=0.7}"
 : "${TOP_K:=20}"
 : "${TOP_P:=0.8}"
 : "${MIN_P:=0.0}"
 : "${PRESENCE_PENALTY:=1.5}"
-: "${REASONING:=off}"
-: "${REASONING_FORMAT:=none}"
+: "${REASONING:=on}"
+: "${REASONING_FORMAT:=deepseek}"
 : "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
 
 MODEL="$MODEL_DIR/$MODEL_FILE"
-for f in "$LLAMA_BIN" "$MODEL" "$MTP_DRAFT_FILE" "$MODEL_DIR/$MMPROJ_FILE" "$CHAT_TEMPLATE_FILE"; do
+for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
     if [[ ! -r "$f" ]]; then
         echo "missing or unreadable required artifact: $f" >&2
         exit 1
@@ -54,35 +75,46 @@ if [[ ! -x "$LLAMA_BIN" ]]; then
     echo "missing executable runtime: $LLAMA_BIN" >&2
     exit 1
 fi
+
 SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)"
-ROCM_PATH="${ROCM_DEVEL:-${ROCM_PATH:-/opt/rocm}}"
-if [[ -n "${ROCM_LIB:-}" && -n "${ROCM_CORE:-}" ]]; then
-    LIBRARY_PATH="$ROCM_LIB/lib:$ROCM_CORE/lib:$ROCM_DEVEL/lib/llvm/lib:$SERVER_DIR:${LD_LIBRARY_PATH:-}"
-else
-    LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib:$ROCM_PATH/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
-fi
+ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
+LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
 
 exec /usr/bin/env \
     ROCM_PATH="$ROCM_PATH" \
     HIP_VISIBLE_DEVICES=0 \
     LD_LIBRARY_PATH="$LIBRARY_PATH" \
     HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
-    GGML_CUDA_GRAPH_OPT=0 \
-    LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0 \
-    LLAMA_MTP_CPU_ARGMAX_FASTPATH=1 \
-    "$LLAMA_BIN" -m "$MODEL" --alias "$ALIAS" \
-    --host "$BIND_ADDR" --port "$LISTEN_PORT" --jinja \
-    -dev ROCm0 -ngl 999 -np "$PARALLEL" -c "$CTX_SIZE" \
-    -b "$BATCH_SIZE" -ub "$UBATCH_SIZE" -fa on \
-    -ctk "$CACHE_TYPE_K" -ctv "$CACHE_TYPE_V" -t "$NUM_THREADS" -tb "$THREAD_BATCH" \
-    -ctxcp "$CTX_CHECKPOINTS" --cache-ram "$CACHE_RAM" --cache-prompt --cache-idle-slots --metrics \
-    --spec-type draft-mtp --model-draft "$MTP_DRAFT_FILE" \
-    --mmproj "$MODEL_DIR/$MMPROJ_FILE" \
-    --spec-draft-device ROCm0 --spec-draft-ngl 999 \
-    --spec-draft-type-k "$CACHE_TYPE_K" --spec-draft-type-v "$CACHE_TYPE_V" \
-    --spec-draft-n-max "$MTP_N_MAX" --spec-draft-n-min "$MTP_N_MIN" \
-    --spec-draft-p-min "$MTP_P_MIN" --spec-draft-p-split "$MTP_P_SPLIT" \
-    --spec-draft-backend-sampling \
-    --temp "$TEMPERATURE" --top-p "$TOP_P" --top-k "$TOP_K" --min-p "$MIN_P" \
+    "$LLAMA_BIN" \
+    --model "$MODEL" \
+    --alias "$ALIAS" \
+    --chat-template-file "$CHAT_TEMPLATE_FILE" \
+    --chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT" \
+    --jinja \
+    --device ROCm0 \
+    --gpu-layers 999 \
+    --parallel "$PARALLEL" \
+    --ctx-size "$CTX_SIZE" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --fit "$FIT" \
+    --flash-attn on \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --threads "$NUM_THREADS" \
+    --threads-batch "$THREAD_BATCH" \
+    --ctx-checkpoints "$CTX_CHECKPOINTS" \
+    --cache-ram "$CACHE_RAM" \
+    --cache-prompt \
+    --cache-idle-slots \
+    --metrics \
+    --temp "$TEMPERATURE" \
+    --top-p "$TOP_P" \
+    --top-k "$TOP_K" \
+    --min-p "$MIN_P" \
     --presence-penalty "$PRESENCE_PENALTY" \
-    --reasoning "$REASONING" --reasoning-format "$REASONING_FORMAT" --reasoning-budget -1
+    --reasoning "$REASONING" \
+    --reasoning-format "$REASONING_FORMAT" \
+    --reasoning-budget -1

scripts/serve-frank-small.sh

@@ -1,18 +1,23 @@
 #!/usr/bin/env bash
-# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q6_K) as
-# frank-default-small via upstream llama.cpp on Dankserver (RTX 3090, MoE
-# experts on CPU).
+# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as
+# frank-default-small via stock llama.cpp on Dankserver (RTX 3090).
 #
-# Runtime: upstream llama.cpp llama-server (b10121). The model is a 35B-total /
-# 3B-active MoE-dense hybrid with an MTP head bundled in the GGUF; this
-# service uses ngram-mod instead of MTP speculation.
+# Runtime: upstream llama.cpp (latest master, CUDA build with
+# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86).
+# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with
+# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM
+# layers out of 40).  256 experts, 8 used per token.
 #
-# Expert offload: the expert (FFN exps) tensors selected by the layer-bounded
-# --override-tensor pattern are pinned to CPU. Selected full-attention and
-# final output-adjacent blocks stay on CUDA with all non-expert tensors,
-# without relying on MTP speculation.
+# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
+# Non-expert weights (attention, SSM, shared expert, embeddings, output)
+# are ~1.6 GB and fit entirely on the 24 GB RTX 3090.
 #
-# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
+# KV cache: Q8_0 for both K and V.  With CTX_SIZE=200700 (66 900 per
+# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim),
+# the KV cache is ~2 GB — the requested 2 GB VRAM buffer.
+#
+# Model is loaded with mmap (default).  The 30 GB Q6_K_XL file is
+# memory-mapped; only accessed pages enter RAM.
 #
 # Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
 # overrides that path.
@@ -28,33 +33,21 @@ if [[ -f "$ENV_FILE" ]]; then
 fi
 
 # --- defaults (used when the env file does not set them) -----------------
-: "${MODEL_DIR:=/home/dank/models}"
-: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf}"
-: "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}"
+: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}"
+: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}"
 : "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
-: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
 : "${ALIAS:=frank-default-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=491520}"
+: "${CTX_SIZE:=200700}"
 : "${PARALLEL:=3}"
-: "${SPEC_TYPE:=ngram-mod}"
-: "${SPEC_NGRAM_MOD_N_MIN:=48}"
-: "${SPEC_NGRAM_MOD_N_MAX:=64}"
-: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
-: "${CACHE_TYPE_K:=f16}"
-: "${CACHE_TYPE_V:=f16}"
-: "${VERBOSITY:=4}"
-# Expert offload: --override-tensor regex pinned to the CPU buft. Same pattern
-# the internal --cpu-moe / --n-cpu-moe use; explicit here so we only move the
-# MoE expert weights and leave every other tensor on the GPU.
-: "${OVERRIDE_EXPERT:=blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps}"
-: "${OVERRIDE_BUFT:=CPU}"
-: "${N_GPU_LAYERS:=all}"
+: "${CACHE_TYPE_K:=q8_0}"
+: "${CACHE_TYPE_V:=q8_0}"
+: "${N_GPU_LAYERS:=999}"
 : "${BATCH_SIZE:=4096}"
 : "${UBATCH_SIZE:=1024}"
-: "${CTX_CHECKPOINTS:=16}"
-: "${NO_MMAP:=1}"
+: "${VERBOSITY:=2}"
+: "${CPU_MOE:=1}"
 : "${TEMPERATURE:=1.0}"
 : "${TOP_K:=20}"
 : "${TOP_P:=0.95}"
@@ -63,14 +56,11 @@ fi
 : "${THREADS_HTTP:=8}"
 
 MODEL="$MODEL_DIR/$MODEL_FILE"
-MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
 
-for f in "$MODEL" "$MMPROJ" "$CHAT_TEMPLATE_FILE"; do
-    if [[ ! -f "$f" ]]; then
-        echo "missing required file: $f" >&2
-        exit 1
-    fi
-done
+if [[ ! -f "$MODEL" ]]; then
+    echo "missing required file: $MODEL" >&2
+    exit 1
+fi
 if [[ ! -x "$LLAMA_SERVER" ]]; then
     echo "missing llama-server binary: $LLAMA_SERVER" >&2
     exit 1
@@ -78,23 +68,14 @@ fi
 
 exec "$LLAMA_SERVER" \
     --model "$MODEL" \
-    --mmproj "$MMPROJ" \
-    --chat-template-file "$CHAT_TEMPLATE_FILE" \
-    --spec-type "$SPEC_TYPE" \
-    --spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
-    --spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
-    --spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
-    --override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT" \
+    $([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \
     --cache-type-k "$CACHE_TYPE_K" \
     --cache-type-v "$CACHE_TYPE_V" \
-    --verbosity "$VERBOSITY" \
     --ctx-size "$CTX_SIZE" \
     --parallel "$PARALLEL" \
     --gpu-layers "$N_GPU_LAYERS" \
     --batch-size "$BATCH_SIZE" \
     --ubatch-size "$UBATCH_SIZE" \
-    --ctx-checkpoints "$CTX_CHECKPOINTS" \
-    $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
     --temp "$TEMPERATURE" \
     --top-k "$TOP_K" \
     --top-p "$TOP_P" \

scripts/test-changed.sh

@@ -104,12 +104,40 @@ for path in "${CHANGED[@]}"; do
             add_test tests/searxng_quadlet.bats
             add_test tests/deployment_scoping.bats
             ;;
+        # Caddy left compose.yaml on 2026-08-26 for a quadlet; caddyfile.bats
+        # now asserts against the quadlet itself. Route every change under
+        # dankclaw/caddy/ to both seams (this block must precede the
+        # dankclaw/*.container glob, which would otherwise swallow the unit).
+        dankclaw/caddy/*)
+            add_test tests/caddy_quadlet.bats
+            add_test tests/caddyfile.bats
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
         dankclaw/*.container|dankclaw/*.network)
             add_test tests/open_webui_quadlet.bats
             add_test tests/pihole.bats
             add_test tests/searxng_quadlet.bats
             add_test tests/deployment_scoping.bats
             ;;
+        # Quadlets live under dankclaw/<svc>/quadlet/ (reorganized out of the
+        # top-level dankclaw/*.container globs above). Map the whole subtree so
+        # changes to the pihole, open-webui, searxng, and gitea quadlets hit
+        # their seam tests. The pihole-specific block must come first: shell
+        # case matches the first pattern, and dankclaw/pihole/quadlet/*.container
+        # also satisfies the dankclaw/*/quadlet/*.container fallback.
+        dankclaw/pihole/*)
+            add_test tests/pihole.bats
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        dankclaw/*/quadlet/*.container|dankclaw/*/quadlet/*.network)
+            add_test tests/open_webui_quadlet.bats
+            add_test tests/pihole.bats
+            add_test tests/searxng_quadlet.bats
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
         scripts/deploy.sh|scripts/deploy-remote.sh|scripts/deploy-plan.py|scripts/deploy-*-quadlet.sh)
             add_test tests/deployment_flow.bats
             add_test tests/deployment_scoping.bats
@@ -119,12 +147,12 @@ for path in "${CHANGED[@]}"; do
             add_test tests/mount_entropy.bats
             ;;
         scripts/frank-default-large.env|scripts/serve-frank-large.sh)
-            add_test tests/frank_large_config.bats
+            add_test tests/llm_units.bats
             add_test tests/llm_secrets.bats
             add_test tests/scripts_syntax.bats
             ;;
         scripts/frank-default-small.env|scripts/serve-frank-small.sh)
-            add_test tests/frank_small_config.bats
+            add_test tests/llm_units.bats
             add_test tests/llm_secrets.bats
             add_test tests/scripts_syntax.bats
             ;;
@@ -149,6 +177,9 @@ for path in "${CHANGED[@]}"; do
             add_test tests/docs_completeness.bats
             add_test tests/deployment_scoping.bats
             ;;
+        .AGENTTRASH/*)
+            # Trash moves are not testable; skip.
+            ;;
         *)
             die "no focused-test mapping for '$path'; add one before committing"
             ;;

tests/caddy_quadlet.bats

@@ -0,0 +1,63 @@
+#!/usr/bin/env bats
+# Seam: canonical Caddy as a boot-persistent, self-healing quadlet on
+# dankclaw. It left compose.yaml on 2026-08-26 after a reboot exposed that a
+# bare compose container never comes back (rootless Podman has no boot-time
+# auto-start for bare containers; `restart: unless-stopped` does not survive
+# the explicit stop that shutdown performs). See AGENTS.md "Boot Persistence
+# & Self-Healing" and docs/Infrastructure/Caddy.md.
+
+load helpers
+
+CADDY_DIR="$REPO_ROOT/dankclaw/caddy"
+QUADLET="$CADDY_DIR/quadlet/caddy.container"
+ENV_EXAMPLE="$CADDY_DIR/env.example"
+
+@test "the caddy quadlet container file exists" {
+    [ -f "$QUADLET" ]
+}
+
+@test "the caddy quadlet is installed for boot" {
+    grep -q 'WantedBy=default.target' "$QUADLET"
+}
+
+@test "the caddy quadlet restarts on failure (self-healing)" {
+    grep -q '^Restart=always' "$QUADLET"
+    grep -qE '^RestartSec=[1-9][0-9]*$' "$QUADLET"
+}
+
+@test "the caddy quadlet publishes 80 and 443 to the LAN" {
+    grep -qE '^PublishPort=0\.0\.0\.0:80:80/tcp$' "$QUADLET"
+    grep -qE '^PublishPort=0\.0\.0\.0:443:443/tcp$' "$QUADLET"
+}
+
+@test "the caddy quadlet joins the compose network for container-name upstreams" {
+    # The Caddyfile reverse-proxies to container names (authentik-server,
+    # grafana, ...); those resolve only on the shared bridge.
+    grep -q '^Network=dankclaw_default$' "$QUADLET"
+}
+
+@test "the caddy quadlet never pulls the local-only image at boot" {
+    grep -q '^Pull=never$' "$QUADLET"
+}
+
+@test "the caddy quadlet reads its env file and mounts its state" {
+    grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' "$QUADLET"
+    grep -qE '^Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro$' "$QUADLET"
+    grep -qE '^Volume=/home/dank/Containers/caddy/data:/data$' "$QUADLET"
+    grep -qE '^Volume=/home/dank/Containers/caddy/config:/config$' "$QUADLET"
+    grep -qE '^Volume=/srv/homepage:/srv/homepage:ro$' "$QUADLET"
+}
+
+@test "the caddy env example has placeholder values only" {
+    [ -f "$ENV_EXAMPLE" ]
+    grep -q '^PORKBUN_API_KEY=changeme$' "$ENV_EXAMPLE"
+    grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$ENV_EXAMPLE"
+}
+
+@test "no secret material in the caddy files" {
+    local f
+    while IFS= read -r f; do
+        has_no_leaked_secret "$f"
+        has_no_leaked_sudo_pw "$f"
+    done < <(find "$CADDY_DIR" -type f | sort)
+}

tests/caddyfile.bats

@@ -76,13 +76,47 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
 }
 
 @test "compose no longer defines the quadlet-replaced services" {
-    # gitea, its postgres, and open-webui run as boot-persistent quadlets on
-    # dankclaw (see dankclaw/gitea/quadlet, dankclaw/open-webui/quadlet).
-    ! grep -qE '^\s+(gitea|postgres|openwebui):\s*$' "$REPO_ROOT/dankclaw/compose.yaml"
+    # gitea, its postgres, open-webui, and caddy run as boot-persistent
+    # quadlets on dankclaw (see dankclaw/gitea/quadlet,
+    # dankclaw/open-webui/quadlet, dankclaw/caddy/quadlet).
+    ! grep -qE '^\s+(gitea|postgres|openwebui|caddy):\s*$' "$REPO_ROOT/dankclaw/compose.yaml"
 }
 
-@test "the caddy service healthcheck probes the bare-IP catch-all" {
-    grep -q 'curl.*-sf.*http://localhost:80/' "$REPO_ROOT/dankclaw/compose.yaml"
+@test "the caddy quadlet healthcheck probes the bare-IP catch-all" {
+    grep -q 'HealthCmd=curl -sf -o /dev/null http://localhost:80/' \
+        "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
+}
+
+@test "plex vhosts proxy to PMS over a TLS upstream (PMS drops plain HTTP off-loopback)" {
+    # 2026-08-26 diagnosis: PMS 1.43.x on :32400 accepts, reads, and then
+    # silently closes every non-TLS connection from a non-loopback source
+    # (loopback is exempt). A plain-HTTP reverse_proxy therefore gets EOF
+    # and Caddy answers 502 with an empty body (blank page in browsers).
+    # Both plex vhosts must dial the upstream over TLS. PMS serves a
+    # self-signed cert, so verification is skipped on this pinned-LAN-IP
+    # hop only; client-facing Let's Encrypt TLS is untouched.
+    local block name
+    for name in 'plex.franksdank.cloud {' 'http://plex.franksdank.casa {'; do
+        block=$(awk -v s="$name" '
+            index($0, s) == 1 { inblk = 1 }
+            inblk { print }
+            inblk && /^}/ { exit }' "$CADDYFILE")
+        echo "$block" | grep -q 'reverse_proxy 192.168.0.235:32400' \
+            || { echo "$name: missing PMS upstream"; return 1; }
+        echo "$block" | grep -q 'transport http' \
+            || { echo "$name: upstream must use the http transport with TLS"; return 1; }
+        echo "$block" | grep -q 'tls_insecure_skip_verify' \
+            || { echo "$name: TLS upstream must set tls_insecure_skip_verify (PMS self-signed cert)"; return 1; }
+    done
+}
+
+@test "Gitea has no standalone reverse-proxy Quadlet" {
+    [ ! -e "$REPO_ROOT/dankclaw/gitea/quadlet/gitea-caddy.container" ]
+}
+
+@test "the canonical Caddy owns the Gitea route" {
+    grep -qF 'gitea.franksdank.cloud {' "$CADDYFILE"
+    grep -qF 'reverse_proxy 192.168.0.143:3000' "$CADDYFILE"
 }
 
 # --- Phase 2: real TLS via DNS-01 (Porkbun wildcard cert) ---
@@ -106,13 +140,19 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
     grep -q 'redir https://franksdank.cloud/' "$CADDYFILE"
 }
 
-@test "the caddy service uses the locally built porkbun image" {
-    grep -q 'franksdank/caddy-porkbun' "$REPO_ROOT/dankclaw/compose.yaml"
+@test "the caddy quadlet uses the locally built porkbun image without pulling" {
+    # The image is built on-host from dankclaw/caddy/Containerfile; a boot
+    # must not stall on a registry that does not have it.
+    grep -q '^Image=franksdank/caddy-porkbun:latest$' \
+        "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
+    grep -q '^Pull=never$' "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
 }
 
-@test "the caddy service passes the Porkbun credentials from .env" {
-    grep -q 'PORKBUN_API_KEY: \${PORKBUN_API_KEY}' "$REPO_ROOT/dankclaw/compose.yaml"
-    grep -q 'PORKBUN_API_SECRET_KEY: \${PORKBUN_API_SECRET_KEY}' "$REPO_ROOT/dankclaw/compose.yaml"
+@test "the caddy quadlet reads the Porkbun credentials from its env file" {
+    grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' \
+        "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
+    grep -q '^PORKBUN_API_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example"
+    grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example"
 }
 
 @test ".env.example carries the Porkbun secret under the module's name" {

tests/changed_tests.bats

@@ -21,10 +21,10 @@ load helpers
     [[ "$output" != *"tests/homepage_pages.bats"* ]]
 }
 
-@test "changed-test wrapper maps a service script to its focused config tests" {
+@test "changed-test wrapper maps a service script to its focused tests" {
     run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/serve-frank-small.sh
     [ "$status" -eq 0 ]
     [[ "$output" == *"tests/scripts_syntax.bats"* ]]
-    [[ "$output" == *"tests/frank_small_config.bats"* ]]
-    [[ "$output" != *"tests/frank_large_config.bats"* ]]
+    [[ "$output" == *"tests/llm_units.bats"* ]]
+    [[ "$output" == *"tests/llm_secrets.bats"* ]]
 }

tests/deployment_flow.bats

@@ -47,6 +47,23 @@ load helpers
     ! grep -qE 'podman-compose.*(up|restart).*\$\(' "$REPO_ROOT/scripts/deploy-remote.sh"
 }
 
+@test "Caddy migration retires the standalone Gitea proxy" {
+    grep -q 'retire_legacy_gitea_caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q 'systemctl --user stop gitea-caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q 'systemctl --user disable gitea-caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh"
+    ! grep -q 'deploy_gitea_unit gitea-caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
+@test "canonical Caddy deployment installs its boot-persistent quadlet" {
+    # 2026-08-26: a bare compose container never came back after reboot.
+    # deploy_caddy must install, enable, and start the quadlet unit instead
+    # of `podman-compose up`.
+    grep -q 'install_quadlet caddy dankclaw/caddy/quadlet caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q 'systemctl --user enable caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh"
+    ! grep -q 'podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
+    ! grep -q 'no proxy was stopped or replaced' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
 @test "remote runner has separate handlers for both inference services" {
     grep -q 'deploy_llm_large' "$REPO_ROOT/scripts/deploy-remote.sh"
     grep -q 'deploy_llm_small' "$REPO_ROOT/scripts/deploy-remote.sh"

tests/deployment_scoping.bats

@@ -54,6 +54,16 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
     [ "$output" = "prometheus" ]
 }
 
+@test "deployment planner maps shared observability configs independently" {
+    run python3 "$PLANNER" --target dankclaw --paths shared/loki/loki-config.yaml
+    [ "$status" -eq 0 ]
+    [ "$output" = "loki" ]
+
+    run python3 "$PLANNER" --target dankclaw --paths shared/alloy/config.alloy
+    [ "$status" -eq 0 ]
+    [ "$output" = "alloy" ]
+}
+
 @test "deployment planner maps service-owned non-compose files" {
     run python3 "$PLANNER" --target dankclaw --paths dankclaw/hermes/config.yaml.example
     [ "$status" -eq 0 ]
@@ -87,6 +97,39 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
     [ "$output" = $'gluetun\nsearxng' ]
 }
 
+@test "retiring the standalone Gitea proxy deploys the canonical Caddy" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        dankclaw/gitea/quadlet/gitea-caddy.container
+    [ "$status" -eq 0 ]
+    [ "$output" = "caddy" ]
+}
+
+@test "Gitea network changes exclude the retired standalone proxy" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        dankclaw/gitea/quadlet/gitea.network
+    [ "$status" -eq 0 ]
+    [ "$output" = $'gitea-postgres\ngitea-server' ]
+}
+
+@test "remote Caddy deployment no longer blocks on the retired Gitea proxy" {
+    ! grep -q 'refusing to replace it with caddy' "$REMOTE_DEPLOY"
+    ! grep -q 'podman container exists gitea-caddy &&' "$REMOTE_DEPLOY"
+}
+
+@test "remote Caddy deployment retires the installed Gitea proxy" {
+    grep -q 'retire_legacy_gitea_caddy' "$REMOTE_DEPLOY"
+    grep -q 'systemctl --user stop gitea-caddy.service' "$REMOTE_DEPLOY"
+    grep -q 'systemctl --user disable gitea-caddy.service' "$REMOTE_DEPLOY"
+    grep -q 'gitea-caddy.container' "$REMOTE_DEPLOY"
+    grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY"
+    grep -q 'stage_legacy_gitea_caddy_state' "$REMOTE_DEPLOY"
+}
+
+@test "remote deployment inventory no longer accepts gitea-caddy as a service" {
+    ! grep -qE 'dankclaw:gitea-caddy|gitea-caddy\) deploy' "$REPO_ROOT/scripts/deploy.sh" "$REPO_ROOT/scripts/deploy-remote.sh"
+    ! grep -q '"gitea-caddy"' "$REPO_ROOT/scripts/deploy-plan.py"
+}
+
 @test "remote deployer carries per-service deployment state" {
     [ -x "$REMOTE_DEPLOY" ]
     grep -q 'deploy-state' "$REMOTE_DEPLOY"

tests/frank_small_config.bats

@@ -1,64 +0,0 @@
-#!/usr/bin/env bats
-# Seam: frank-default-small model, context, tensor placement, and template.
-# This test covers only the Q6_K deployment contract.
-
-load helpers
-
-ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env"
-SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh"
-UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service"
-TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
-
-@test "frank-default-small selects the Ornith Q6_K model" {
-    grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf$' "$ENV_FILE"
-    grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE"
-}
-
-@test "frank-default-small uses 163840 context per slot across three slots" {
-    grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
-    grep -q '163840 ctx each' "$ENV_FILE"
-    grep -q '^PARALLEL=3$' "$ENV_FILE"
-}
-
-@test "frank-default-small keeps full-attention and final blocks on CUDA" {
-    source "$ENV_FILE"
-    [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
-    [[ "$OVERRIDE_EXPERT" != *'|3|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|7|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|11|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|15|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|19|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|23|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|27|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|31|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
-    [[ "$OVERRIDE_BUFT" == CPU ]]
-    [[ "$N_GPU_LAYERS" == all ]]
-}
-
-@test "frank-default-small uses ngram-mod without MTP" {
-    grep -q '^SPEC_TYPE=ngram-mod$' "$ENV_FILE"
-    grep -q '^SPEC_NGRAM_MOD_N_MIN=48$' "$ENV_FILE"
-    grep -q '^SPEC_NGRAM_MOD_N_MAX=64$' "$ENV_FILE"
-    grep -q '^SPEC_NGRAM_MOD_N_MATCH=24$' "$ENV_FILE"
-    ! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE"
-    ! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE"
-    grep -q -- '--spec-ngram-mod-n-max' "$SERVE_FILE"
-}
-
-@test "frank-default-small uses ubatch 1024" {
-    grep -q '^BATCH_SIZE=4096$' "$ENV_FILE"
-    grep -q '^UBATCH_SIZE=1024$' "$ENV_FILE"
-    grep -q ': "${UBATCH_SIZE:=1024}"' "$SERVE_FILE"
-}
-
-@test "frank-default-small uses the tracked Qwen-Sharp template" {
-    grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE"
-    grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE"
-    grep -q 'chat-template\.jinja' "$SERVE_FILE"
-    [ -s "$TEMPLATE_FILE" ]
-    grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE"
-    grep -q 'Qwen-Sharp' "$UNIT_FILE"
-}

tests/hermes_webui.bats

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Seam: hermes-webui (the Hermex server) deployment integrity.
+# The webui is the phone-facing API for the Hermes agent. It must stay
+# loopback-bound so the SSH tunnel is its only ingress, keep its password in
+# the host .env, share the gateway data volume, and deploy as an isolated
+# compose service that cannot drag the gateway or inference along with it.
+
+load helpers
+
+COMPOSE="$REPO_ROOT/dankclaw/compose.yaml"
+ENV_EXAMPLE="$REPO_ROOT/dankclaw/.env.example"
+PLANNER="$REPO_ROOT/scripts/deploy-plan.py"
+DEPLOY="$REPO_ROOT/scripts/deploy.sh"
+REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
+
+webui_service_yaml() {
+    python3 - "$COMPOSE" <<'PYEOF'
+import sys, yaml
+with open(sys.argv[1]) as fh:
+    data = yaml.safe_load(fh)
+svc = (data.get("services") or {}).get("hermes-webui")
+assert svc, "dankclaw/compose.yaml: no hermes-webui service"
+print(yaml.safe_dump(svc))
+PYEOF
+}
+
+@test "hermes-webui runs the webui bootstrap in foreground mode" {
+    local y
+    y=$(webui_service_yaml)
+    grep -q 'nousresearch/hermes-agent' <<< "$y"
+    grep -q 'bootstrap.py' <<< "$y"
+    grep -q -- '--foreground' <<< "$y"
+}
+
+@test "hermes-webui is loopback-bound and password-gated" {
+    local y
+    y=$(webui_service_yaml)
+    # Host-side publish stays on loopback: the SSH tunnel is the only ingress.
+    grep -q '127.0.0.1:8787' <<< "$y"
+    grep -q 'HERMES_WEBUI_PASSWORD' <<< "$y"
+    # Container-internal bind must be 0.0.0.0 or Podman's port forwarder
+    # (which dials the bridge IP) resets every connection.
+    grep -q 'HERMES_WEBUI_HOST: 0.0.0.0' <<< "$y"
+}
+
+@test "hermes-webui shares the gateway data volume and mounts the webui checkout" {
+    local y
+    y=$(webui_service_yaml)
+    grep -q 'HERMES_HOME: /data' <<< "$y"
+    grep -q '/home/dank/hermes-data:/data' <<< "$y"
+    grep -q '/home/dank/hermes-webui:/webui' <<< "$y"
+}
+
+@test "hermes-webui has a healthcheck that probes /health" {
+    local y
+    y=$(webui_service_yaml)
+    grep -q 'healthcheck' <<< "$y"
+    grep -q '/health' <<< "$y"
+}
+
+@test "dankclaw .env.example defines the webui password placeholder" {
+    env_vars_defined "$ENV_EXAMPLE" | grep -qx 'HERMES_WEBUI_PASSWORD'
+}
+
+# Revision-aware planner: a change to only the hermes-webui service block must
+# plan hermes-webui and nothing else (the gateway is unchanged).
+webui_fixture_repo() {
+    local dir="$1"
+    git init -q "$dir"
+    git -C "$dir" config user.email t@t && git -C "$dir" config user.name t
+    mkdir -p "$dir/dankclaw"
+    cat > "$dir/dankclaw/compose.yaml" <<'EOF'
+services:
+  hermes:
+    image: docker.io/nousresearch/hermes-agent:latest
+  hermes-webui:
+    image: docker.io/nousresearch/hermes-agent:latest
+    ports:
+      - "127.0.0.1:8787:8787"
+EOF
+    git -C "$dir" add -A && git -C "$dir" commit -qm base
+    # Touch only the webui service block (its port line is unique in the file).
+    sed -i 's|127.0.0.1:8787:8787|127.0.0.1:8788:8787|' "$dir/dankclaw/compose.yaml"
+    git -C "$dir" commit -qam head
+}
+
+@test "planner narrows a hermes-webui compose edit to hermes-webui only" {
+    local dir base head
+    dir=$(mktemp -d)
+    webui_fixture_repo "$dir"
+    base=$(git -C "$dir" rev-parse HEAD~1)
+    head=$(git -C "$dir" rev-parse HEAD)
+    run python3 "$PLANNER" --target dankclaw --repo "$dir" \
+        --base "$base" --head "$head" --paths dankclaw/compose.yaml
+    [ "$status" -eq 0 ]
+    [ "$output" = "hermes-webui" ]
+}
+
+@test "remote deployer accepts and dispatches hermes-webui" {
+    grep -q 'dankclaw:hermes-webui' "$REMOTE_DEPLOY"
+    awk '/^deploy_compose_service\(\)/,/^}/' "$REMOTE_DEPLOY" | grep -q 'hermes-webui'
+}
+
+@test "top-level deployer accepts hermes-webui as a Dankclaw service" {
+    grep -q 'dankclaw:hermes-webui' "$DEPLOY"
+}

tests/pihole.bats

@@ -48,6 +48,20 @@ HOSTS="$PIHOLE_DIR/addn-hosts"
     grep -qE '^Volume=/home/dank/Containers/pihole/addn-hosts:/etc/pihole/hosts/addn-hosts:ro$' "$QUADLET"
 }
 
+@test "the pihole sysctl drop-in pins the unprivileged port threshold to 53" {
+    # Rootless port 53 needs ip_unprivileged_port_start <= 53 at boot. The
+    # repo drop-in is the single source of truth; a later-sorted file that
+    # reset it to 80 took LAN DNS down after the 2026-08-26 reboot.
+    local f="$PIHOLE_DIR/sysctl/99-dnsmasq.conf"
+    [ -f "$f" ]
+    grep -q '^net.ipv4.ip_unprivileged_port_start=53$' "$f"
+}
+
+@test "deploy_pihole installs the sysctl drop-in and applies it at boot-path" {
+    grep -q 'dankclaw/pihole/sysctl/99-dnsmasq.conf' "$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q 'sysctl --system' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
 @test "the pihole env example has placeholder values only" {
     [ -f "$ENV_EXAMPLE" ]
     grep -q '^WEBPASSWORD=changeme$' "$ENV_EXAMPLE"