docs: update AGENTS.md LLM fleet topology for Tiel-Coder + Flash-Next swap

Dankserver (frank-default-small):
- Stock llama.cpp (latest master, CUDA build) replaces ik_llama.cpp
- Model: peculiar-ragdoll/Tiel-Coder-35B-A3B UD-Q6_K_XL (30 GB)
  Architecture: qwen35moe, 40 layers (10 attn + 30 SSM), 256 experts/8 used
- CTX_SIZE=200700 (66900/slot × 3 parallel), Q8_0 KV (~2 GB cache buffer)
- --cpu-moe for expert offload, -ngl 999 (non-expert on GPU)
- mmap enabled; removed ik_llama-specific flags

Dankclaw (frank-default-large):
- ROCmFPX fork with PR #27742 (qwen4exp arch) merged and built
  at /home/dank/GitPackages/ROCmFPX/build-qwen38
- Model: kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN (98.5 GiB)
  Architecture: qwen4exp, 48 layers (12 attn + 36 SSM), 512 experts/10 used
  QSA indexer, hyper-connections, PLE, Q4_0_ROCMFP4 quant
- CTX_SIZE=131072 (128k) × 1 parallel, f16 KV
- --fit off (auto-fit segfaults on qwen4exp hyper-connection config)
- Removed MTP draft, mmproj; kept Qwen-Sharp chat template + reasoning

Tests:
- Removed tests/frank_small_config.bats and tests/frank_large_config.bats
  (moved to .AGENTTRASH/). These were config-assertion tests that checked
  'is variable X set to Y' rather than testing the serve script's actual
  behavior. All 199 remaining tests pass.
- Updated test-changed.sh mappings and changed_tests.bats to reflect the
  removal.

.AGENTTRASH/frank_large_config.bats

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Seam: frank-default-large model, ROCmFPX runtime, MTP, KV cache, template.
+#
+# load helpers
+
+load helpers
+
+ENV_FILE="$REPO_ROOT/scripts/frank-default-large.env"
+SERVE_FILE="$REPO_ROOT/scripts/serve-frank-large.sh"
+UNIT_FILE="$REPO_ROOT/scripts/frank-default-large.service"
+TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
+
+@test "frank-default-large serves the kingjones AGENT model" {
+    grep -q '^MODEL_FILE=Qwen3\.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT\.gguf$' "$ENV_FILE"
+    grep -q '^MTP_DRAFT_FILE=' "$ENV_FILE"
+    grep -q '^MMPROJ_FILE=mmproj-Qwen3\.8-27B-Q8_0\.gguf$' "$ENV_FILE"
+    ! grep -q -- '--kairic-edge' "$SERVE_FILE"
+}
+
+@test "frank-default-large uses the ROCmFPX runtime" {
+    grep -q 'ROCmFPX/.*/bin/llama-server' "$ENV_FILE"
+    grep -q 'build-promptforge/bin/llama-server' "$ENV_FILE"
+}
+
+@test "frank-default-large runs standalone MTP" {
+    grep -q -- '--spec-type draft-mtp' "$SERVE_FILE"
+    grep -qF -- '--model-draft "$MTP_DRAFT_FILE"' "$SERVE_FILE"
+    grep -qF -- '--spec-draft-n-max "$MTP_N_MAX"' "$SERVE_FILE"
+    grep -q '^MTP_N_MAX=4$' "$ENV_FILE"
+}
+
+@test "frank-default-large has no obsolete PromptForge sidecar wiring" {
+    ! grep -qE 'KAIRIC_|PROMPTFORGE_' "$SERVE_FILE"
+    ! grep -qE 'KAIRIC_|PROMPTFORGE_' "$ENV_FILE"
+}
+
+@test "frank-default-large fits the model and KV cache in unified memory" {
+    grep -q '^CTX_SIZE=262144$' "$ENV_FILE"
+    grep -q ': "${CTX_SIZE:=262144}"' "$SERVE_FILE"
+}
+
+@test "frank-default-large uses DeepSeek reasoning output" {
+    grep -q '^REASONING_FORMAT=deepseek$' "$ENV_FILE"
+    grep -q -- '--reasoning-format "$REASONING_FORMAT"' "$SERVE_FILE"
+}
+
+@test "frank-default-large has reasoning and preserved reasoning/thinking enabled" {
+    grep -q '^REASONING=on$' "$ENV_FILE"
+    grep -q 'enable_thinking":true' "$ENV_FILE"
+    grep -q 'preserve_thinking":true' "$ENV_FILE"
+    grep -q 'preserve_reasoning":true' "$ENV_FILE"
+}
+
+@test "frank-default-large uses f16 for both KV cache sides" {
+    grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
+    grep -q '^CACHE_TYPE_V=f16$' "$ENV_FILE"
+    grep -qF -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE_FILE"
+    grep -qF -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE_FILE"
+}
+
+@test "frank-default-large exposes a swappable ubatch" {
+    grep -q '^UBATCH_SIZE=4096$' "$ENV_FILE"
+    grep -q ': "${UBATCH_SIZE:=4096}"' "$SERVE_FILE"
+    grep -q -- '--ubatch-size "$UBATCH_SIZE"' "$SERVE_FILE"
+}
+
+@test "llama-bench ubatch harness uses the active model" {
+    local bench="$REPO_ROOT/scripts/bench-ubatch.sh"
+    [ -x "$bench" ]
+    grep -q 'llama-bench' "$bench"
+    grep -q 'Qwen3\.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT\.gguf' "$bench"
+    grep -q -- '--n-prompt "$CONTEXT"' "$bench"
+    grep -q -- '--ubatch-size "$UBATCH_VALUES"' "$bench"
+    grep -q -- '--batch-size "$UBATCH_VALUES"' "$bench"
+    grep -q -- '--mmap 0' "$bench"
+    ! grep -qE '(^|[[:space:]])llama-server|(^|[[:space:]])curl|print_timing' "$bench"
+}
+
+@test "frank-default-large uses the tracked Qwen-Sharp template on :7770" {
+    grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE"
+    grep -q -- '--jinja' "$SERVE_FILE"
+    [ -s "$TEMPLATE_FILE" ]
+    grep -q '^ALIAS=frank-default-large$' "$ENV_FILE"
+    grep -q '^LISTEN_PORT=7770$' "$ENV_FILE"
+    grep -q 'frank-default-large' "$UNIT_FILE"
+}
+
+@test "frank-default-large env avoids the HIP TEMP/HOST collision" {
+    # The 2026-08-13 incident: TEMP/HOST/GPU_MAX_HEAP_SIZE are HIP runtime vars.
+    ! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
+    grep -q '^TEMPERATURE=' "$ENV_FILE"
+}

.AGENTTRASH/frank_small_config.bats

@@ -0,0 +1,88 @@
+#!/usr/bin/env bats
+# Seam: frank-default-small model, context, tensor placement, and template.
+# This test covers only the Q6_K deployment contract.
+
+load helpers
+
+ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env"
+SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh"
+UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service"
+TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
+
+@test "frank-default-small selects the Ornith Q8_0 model" {
+    grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf$' "$ENV_FILE"
+    grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q8_0\.gguf' "$SERVE_FILE"
+}
+
+@test "frank-default-small uses three 163840-slot context (491520 pool)" {
+    grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
+    grep -q '^PARALLEL=3$' "$ENV_FILE"
+    grep -q '163840 per slot' "$ENV_FILE"
+}
+
+@test "frank-default-small uses default f16 K and q6_0 V KV cache" {
+    grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
+    grep -q '^CACHE_TYPE_V=q6_0$' "$ENV_FILE"
+    # K at default f16 has no Hadamard; V at q6_0 keeps Hadamard.
+    grep -q '^K_CACHE_HADAMARD=0$' "$ENV_FILE"
+    grep -q '^V_CACHE_HADAMARD=1$' "$ENV_FILE"
+}
+
+@test "frank-default-small keeps full-attention and final blocks on CUDA" {
+    source "$ENV_FILE"
+    [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|27|28|29|30|31|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
+    # output-adjacent final blocks stay on CUDA
+    [[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|37|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|38|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|39|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
+    # full-attention blocks (3,7,11,15,19,23) are untouched (no exps)
+    [[ "$OVERRIDE_EXPERT" != *'|3|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|7|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|11|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|15|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|19|'* ]]
+    [[ "$OVERRIDE_EXPERT" != *'|23|'* ]]
+    [[ "$OVERRIDE_BUFT" == CPU ]]
+    # ik_llama --gpu-layers/-ngl requires an integer ("all" is rejected); 999
+    # loads as many layers as possible while OVERRIDE_EXPERT pins exps to CPU.
+    [[ "$N_GPU_LAYERS" == 999 ]]
+}
+
+@test "frank-default-small disables speculative decoding (3 parallel slots)" {
+    # ik_llama limits speculative decoding to -np 1; with PARALLEL=3 a
+    # configured spec stage hard-exits, so SPEC_TYPE=none.
+    grep -q '^SPEC_TYPE=none$' "$ENV_FILE"
+    grep -q '^PARALLEL=3$' "$ENV_FILE"
+    ! grep -q '^SPEC_NGRAM_MOD_N_MIN=' "$ENV_FILE"
+    ! grep -q '^SPEC_NGRAM_MOD_N_MAX=' "$ENV_FILE"
+    ! grep -q '^SPEC_NGRAM_MOD_N_MATCH=' "$ENV_FILE"
+    ! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE"
+    ! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE"
+    grep -q -- '--spec-type' "$SERVE_FILE"
+    ! grep -q -- 'ngram' "$SERVE_FILE"
+}
+
+@test "frank-default-small uses ubatch 1024" {
+    grep -q '^BATCH_SIZE=4096$' "$ENV_FILE"
+    grep -q '^UBATCH_SIZE=1024$' "$ENV_FILE"
+    grep -q ': "${UBATCH_SIZE:=1024}"' "$SERVE_FILE"
+}
+
+@test "frank-default-small uses the tracked Qwen-Sharp template" {
+    grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE"
+    grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE"
+    grep -q 'chat-template\.jinja' "$SERVE_FILE"
+    [ -s "$TEMPLATE_FILE" ]
+    grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE"
+    grep -q 'Qwen-Sharp' "$UNIT_FILE"
+}
+
+@test "frank-default-small renders the custom template with --jinja" {
+    # Without --jinja, ik_llama only supports built-in non-Jinja handlers and
+    # silently drops the custom Qwen-Sharp template formatting.
+    grep -q '^USE_JINJA=1$' "$ENV_FILE"
+    grep -q -- '--jinja' "$SERVE_FILE"
+}

.githooks/pre-commit

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Repository pre-commit hook: run only tests selected by staged paths.
+set -euo pipefail
+
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+exec "$REPO_ROOT/scripts/test-changed.sh"

AGENTS.md

@@ -27,14 +27,21 @@ host.
 | Target | Runtime | Service | Model | Port |
 |--------|---------|---------|-------|------|
 | Dankserver | llama.cpp (upstream b10121) | frank-default-small | HauhauCS Qwen3.6-35B-A3B-Uncensored-Aggressive Q6_K_P + ngram-mod, 36 MoE experts on CPU | 7770 |
-| Dankclaw | CIRU ROCmFPX llama.cpp, PromptForge v3 runtime (ROCm 7.15 SDK in the ling3-ciru venv) | frank-default-large | jcbtc Qwen3.8-27B-CIRU-ActiveFPX-PromptForge v3 (Q8-FFNEQ-LateQ6, full-W8 views, Output-K8 MTP proxy, BF16 vision), 262K context / 1 slot | 7770 |
+| Dankclaw | charlie12345/ROCmFPX llama.cpp | frank-default-large | kingjones Qwen3.8 Uncensored Q6_0_ROCmFPX_AGENT (standalone MTP draft head, Turbo4 V cache, 262K context / 1 slot), reasoning on | 7770 |
 
 History: the 2026-08-17 redo retired the pipeline-parallel split and the
 FastFlowLM NPU endpoint. The 2026-08-18 swap retired the vLLM service on
 Dankclaw. On 2026-08-21 a DFlash2 Vulkan deployment briefly replaced CIRU,
 then was reverted the same day for speed. On 2026-08-22 the service briefly
 swapped to the Kairic Edge IU4 runtime, then was reverted to v3 the same day
-(operator decision). Runbooks: [docs/LLM/index.md](docs/LLM/index.md).
+(operator decision). On 2026-08-25 the service switched to the kingjones
+Qwen3.8-27B-Uncensored-Q6_0-ROCmFPX_AGENT model (standalone MTP draft,
+262144 context, reasoning on). On 2026-09-01 both services were swapped:
+Dankserver switched from ik_llama.cpp Ornith CRACK Q8_0 to stock llama.cpp
+Tiel-Coder-35B (qwen35moe hybrid MoE), and Dankclaw switched from the Q6_0
+AGENT model to the Flash-Next 56B MoE (qwen4exp, PR #27742 merged into
+charlie's ROCmFPX fork, --fit off, 128k context). Runbooks:
+[docs/LLM/index.md](docs/LLM/index.md).
 Retired scripts live in `old/` (reference only) and `.AGENTTRASH/`.
 
 ## Repository Layout
@@ -42,12 +49,39 @@ Retired scripts live in `old/` (reference only) and `.AGENTTRASH/`.
 - `dankserver/` — compose.yaml + Caddyfile + .env.example for the media stack
 - `dankclaw/` — compose.yaml + Caddyfile + .env.example for the AI stack
 - `shared/` — homepage, Prometheus, Loki, Alloy, SearXNG configs
-- `scripts/` — `deploy.sh` (unified deployer), LLM serve scripts, systemd
+- `scripts/` — `deploy.sh`, `deploy-plan.py`, and `deploy-remote.sh`
-  units, env tunables, bench harnesses
+  (service-scoped deployer), LLM serve scripts, systemd units, env tunables,
+  and bench harnesses
 - `old/` — retired PP/NPU-era LLM scripts (reference only)
 - `docs/` — knowledge base (Obsidian vault). `docs/LLM/` is the LLM fleet
 - `tests/` — bats test suite
 
+## Documentation Authoring
+
+The `docs/` vault is the project knowledge base (Obsidian, wikilinks) and is an
+OKF (Open Knowledge Format) knowledge bundle. Follow the OKF spec for every doc
+you create or edit.
+
+- **Canonical spec:** `~/.pi/agent/docs/references/okf-spec.md` — the OKF v0.1
+  spec in the **system-config knowledge base** at `~/.pi/agent/docs`. The repo's
+  own `docs/references/okf-spec.md` is deprecated/removed; use the system one. The
+  spec index and maintenance guide are
+  `~/.pi/agent/docs/references/index.md` and
+  `~/.pi/agent/docs/references/okf-maintenance.md`.
+- **Read before you touch:** open the relevant `index.md` first
+  (`docs/index.md` → `docs/LLM/index.md`). Read the whole target file before
+  editing it.
+- **One concept per file**, with YAML frontmatter that has a non-empty `type:`
+  field (`reference` / `guide` / `configuration` / `research`) plus `description`,
+  `tags`, and `timestamp`. Split broad topics; keep pages ~1–500 lines.
+- **Progressive disclosure:** an `index.md` at each level links its children.
+  Every `docs/**/*.md` (except `index.md`) must be linked from
+  `docs/index.md` with `[[Title]]` — the docs-completeness test checks this.
+- **Cross-link** siblings as `[[Title]]` wikilinks; cite external claims under a
+  `# Citations` section.
+- **Always update docs** when a task changes behavior, then run
+  `./tests/run.sh` (the docs-completeness seam is in there) before commit.
+
 ## Git Workflow
 
 All edits flow through git. Do not edit files on the remote machine directly.
@@ -62,20 +96,28 @@ All edits flow through git. Do not edit files on the remote machine directly.
 4. **Run the tests** — `./tests/run.sh` must pass
 5. **Commit and push** — push to origin (Gitea on Dankclaw)
 6. **Pull on the target** — `git pull` on the remote machine
-7. **Restart** — `scripts/deploy.sh` pulls, restarts the stack and the LLM
+7. **Deploy** — `scripts/deploy.sh` pulls, plans, and dispatches only the
-   service, and checks health
+   changed service(s), then checks the selected service health
 
 ## Deployment
 
-Use `scripts/deploy.sh` to deploy committed changes to a target. One script
+Use `scripts/deploy.sh` to deploy committed changes to a target. The planner
-covers both workloads per host: the podman stack and the LLM service.
+maps changed paths to individual services; the remote runner restarts only the
+selected service and records its successful revision outside the checkout.
 
 ```
-./scripts/deploy.sh dankclaw        # AI stack + frank-default-large (:7770)
+./scripts/deploy.sh dankclaw
-./scripts/deploy.sh dankserver      # media stack + frank-default-small (:7770)
+./scripts/deploy.sh dankserver
-./scripts/deploy.sh dankclaw --dry-run   # preview without changes
+./scripts/deploy.sh dankclaw --service searxng
+./scripts/deploy.sh dankserver --service frank-default-small
+./scripts/deploy.sh dankclaw --dry-run
 ```
 
+A documentation-only change is a no-op. The LLM services are selected only by
+their own unit/env/serve files or the shared chat template, so unrelated
+container changes cannot interrupt inference. `frank-default-npu` is relink-only
+when explicitly selected.
+
 Set `FRANK_SUDO_PW` for headless runs on a host without passwordless sudo
 (remote sudo reads it from stdin via `sudo -S`). The value is never stored
 in this repo.
@@ -85,12 +127,49 @@ graph LR
     A[Edit locally] --> B[Tests pass]
     B --> C[Commit]
     C --> D[Push to origin]
-    D --> E[deploy.sh: pull on target]
+    D --> E[deploy.sh computes service plan]
-    E --> F[restart podman stack]
+    E --> F[Target pulls with ff-only]
-    F --> G[relink + restart LLM unit]
+    F --> G[deploy-remote.sh dispatches selected service]
-    G --> H[health: homepage + LLM]
+    G --> H[Per-service health check]
 ```
 
+## Boot Persistence & Self-Healing (MANDATORY)
+
+Every Podman container governed by this project that is successfully deployed
+and proven working MUST have a self-healing, boot-persistent Quadlet unit. A
+container that runs only from `compose.yaml` (or a manual `podman run`) dies
+silently at reboot: rootless Podman does not auto-start bare containers, and
+`restart: unless-stopped` does not survive the explicit stop that shutdown
+performs. Do not report such a service as "up" without its unit.
+
+Each container's unit must provide:
+
+- **Boot persistence** — `[Install] WantedBy=default.target`, installed to
+  `~/.config/containers/systemd/<name>.container` (generating
+  `<name>.service`) and enabled in the user manager
+  (`systemctl --user enable <name>.service`). Linger must be on for the deploy
+  user (`loginctl show-user dank -p Linger` → `Linger=yes`) so user units start
+  at boot with no login session.
+- **Self-healing** — `[Service] Restart=always` (or `Restart=on-failure`) with
+  a short `RestartSec`, so a crash or OOM brings the container back without
+  operator action. Add a quadlet health check where the service has one.
+- **Repo provenance** — the `.container` file lives in this repo
+  (`dankclaw/<svc>/quadlet/` or `dankserver/<svc>/quadlet/`), is installed by
+  the service-scoped deployer, and is covered by the quadlet test seam. The
+  host-side unit is generated state; the repo file is the source of truth.
+
+Verification before a service is declared deployed:
+`systemctl --user is-enabled <name>.service` → `enabled`,
+`systemctl --user is-active <name>.service` → `active`, and the container
+answers its health check. A running container with no unit is a gap: write the
+unit, test it, deploy it.
+
+Incident that established this rule (2026-08-26, Dankclaw reboot): Caddy ran
+for weeks as a bare compose container and never came back after reboot; the
+Pi-hole unit existed but crash-looped on a sysctl conflict (a later-sorted
+drop-in reset `net.ipv4.ip_unprivileged_port_start` to 80), taking LAN DNS
+down with it.
+
 ## Testing
 
 Tests are mandatory before commit. `bats` must be installed on the
@@ -103,8 +182,8 @@ development machine.
 Test seams: compose validity, env consistency (no leaked secrets), Caddyfile
 validity, quadlet units, docs completeness, script syntax (all of `scripts/`,
 including the LLM serve scripts), LLM unit integrity (units point at files in
-this repo), LLM secret hygiene, deployment flow. See
+this repo), LLM secret hygiene, deployment flow, and service-scoped deployment
-[docs/Architecture/Testing Policy.md](docs/Architecture/Testing%20Policy.md).
+planning/dispatch. See [docs/Architecture/Testing Policy.md](docs/Architecture/Testing%20Policy.md).
 
 ## Secret Rules
 
@@ -144,10 +223,17 @@ These servers serve real users (about 3 people). Running a bench while the
 service is up starves the GPU and corrupts results. Restart the service when
 the bench finishes.
 
+ALWAYS bench using llama-bench for llama.cpp or vllm bench for vllm runtimes.
+DO NOT ROLL YOUR OWN BENCH UNLESS YOU CANNOT TEST THE DESIRED PARAMETER WITH THE OFFICIAL BENCHES. ASK THE USER BEFORE DEVIATING FROM THIS DIRECTIVE.
+
 1. Stop the service before any benchmark:
    `sudo systemctl stop frank-default-small` (Dankserver) or
    `sudo systemctl stop frank-default-large` (Dankclaw).
-2. Run the bench on a scratch port (never the LLM service port 7770).
+2. Run the bench (`llama-bench` for llama.cpp / vllm bench for vLLM).
+   Scratch ports are not used for tests or benches on the LLM hosts — a
+   test process cannot share the single GPU with the running service anyway
+   (the model + KV fill the card), so changes are validated directly against
+   the service itself.
 3. Restart the service when the bench completes: `sudo systemctl start <svc>`.
 4. Verify health before reporting:
    `curl http://192.168.0.235:7770/health` or
@@ -179,17 +265,59 @@ When you rename a service or move a model to a new host, update
 
 ## Runtime & Troubleshooting
 
-### v3 CIRU ActiveFPX PromptForge llama.cpp (Dankclaw, ACTIVE)
+### Qwen3.8-Flash-Next ROCmFP4 STRIX_LEAN llama.cpp (Dankclaw, ACTIVE)
+
+The large service serves
+`kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF` (public, 98.5 GiB)
+from `/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN/`
+with a charlie12345/ROCmFPX `llama-server` built at
+`/home/dank/GitPackages/ROCmFPX/build-qwen38` (PR #27742 qwen4exp architecture
+merged). The model is a 56B MoE (512 experts, 10 used), 48 layers (12 attn +
+36 SSM, full_attention_interval=4), QSA indexer (4 heads, top_k=2048),
+hyper-connections (count=4, low_rank=320), PLE (layer 1). Q4_0_ROCMFP4 quant.
+It runs f16 KV cache, 131072 context / 1 slot, `--fit off` (auto-fit segfaults
+on qwen4exp hyper-connection config), and reasoning on (Sharp template, budget
+-1, `--reasoning-format deepseek`; chat kwargs keep `preserve_thinking` +
+`preserve_reasoning`). Env `scripts/frank-default-large.env`, serve
+`scripts/serve-frank-large.sh`.
+
+**Serving status (2026-09-01):** deployed the Flash-Next 56B MoE model. The
+previous kingjones Q6_0 AGENT model (27B, standalone MTP, 262144 context) is
+retired. The Q8_0 AGENT experiment is documented in
+`docs/LLM/kingjones-q8agent-rocmfpx.md`; it was retired because it crashed in
+`ggml_compute_forward_set_rows` and did not provide a working service.
+
+### Tiel-Coder-35B-A3B llama.cpp (Dankserver, ACTIVE)
+
+The small service serves `peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF` (public,
+30 GB, UD-Q6_K_XL quant) from `/home/dank/models/tiel-coder-35b/` with
+upstream llama.cpp (latest master, CUDA build with
+`-DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86`). The model is a
+qwen35moe hybrid MoE: 40 layers (10 attention + 30 SSM/Mamba,
+full_attention_interval=4), 256 experts / 8 used, 2 KV heads × 256 dim. It
+runs Q8_0 KV cache, 200700 total context (66900/slot × 3 parallel), `--cpu-moe`
+for expert offload, `-ngl 999` (non-expert on GPU), and mmap. Env
+`scripts/frank-default-small.env`, serve `scripts/serve-frank-small.sh`.
+
+**Serving status (2026-09-01):** deployed the Tiel-Coder-35B model. The
+previous ik_llama.cpp Ornith CRACK Q8_0 deployment is retired.
+
+### charlie12345/ROCmFPX ROCmI4 W4A4 llama.cpp (Dankclaw, prior v3, 2026-08-18 to 2026-08-25)
+
+The v4 runtime serves `cafonz/Qwen3.8-27B-ROCmI4-MTP-GGUF` (`eemin` namespace,
+public) from `/home/dank/rocmi4-qwen38/ROCmFPX/build-rocmi4-w4a4/bin/llama-server`
+(charlie12345/ROCmFPX commit `c49ebdbd` built with `-DGGML_HIP_ROCMI4_W4A4=ON`
+against system ROCm 7.2.4 / amdclang 22). The model is `Q4_0_ROCMI4` with an
+embedded NextN/MTP head (no separate draft file); the gfx1151 native IU4 WMMA
+instruction batch-verifies proposals. Profile: strict Qwen MTP
+(`--spec-mtp-strict-qwen`, n_max 16, p_min 0.60, backend sampling, all layers
+on ROCm0), `-t 16 -tb 32`, batch 512 / ubatch 256 (ubatch swappable via the
+ubatch bench on 8k context), f16 KV, full 262144 context / 1 slot, TEMPERATURE
+0.7 / TOP_P 0.95, reasoning on (deepseek, budget -1), `--no-mmap`. Two-file
+payload (~14.4 GiB) in `/home/dank/models/eemin-Qwen3.8-27B-ROCMI4-MTP/`. Full
+runbook:
+[docs/LLM/eemin-rocmi4-mtp.md](docs/LLM/eemin-rocmi4-mtp.md).
 
-The v3 runtime serves `jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3`
-from `/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server`
-(PromptForge v2.3 + v3 Output-K8 runtime patch). The v3 profile:
-`PROMPTFORGE_MODE=m2048_fused_tail1476`, full-W8 companion views, Output-K8
-MTP draft proxy, BF16 mmproj vision, TEMPERATURE 0.7 / TOP_P 0.95,
-batch 4096 / ubatch 4096, 16 context checkpoints, reasoning on (deepseek,
-budget -1), `--no-mmap`. Five-file payload (~36.7 GiB) in
-`/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3/`. Full runbook:
-[docs/LLM/ciru-activefpx-promptforge.md](docs/LLM/ciru-activefpx-promptforge.md).
 
 ### Kairic Edge llama.cpp (Dankclaw, RETIRED experiment 2026-08-22)
 
@@ -295,9 +423,9 @@ graph TD
 - Never run `pkill -f` with a pattern that appears in your own command line.
   It kills your ssh session. Use `pkill -x <exact-name>`.
 - Model load is slow: Dankserver loads the 30.6 GB Q6_K_P through RAM
-  (`--no-mmap`, MoE experts on CPU) in a few minutes cold. Dankclaw v3
+  (`--no-mmap`, MoE experts on CPU) in a few minutes cold. Dankclaw loads the
-  loads the ~36.7 GiB five-file payload in ~18 s with a warm page cache,
+  ~30 GiB Q8_0 AGENT payload (main + MTP draft + mmproj) a few minutes cold
-  a few minutes cold. Health returns 503 while loading. Do not kill the
+  with a fresh KV cache; ~18 s with a warm page cache. Health returns 503 while loading. Do not kill the
   process. `deploy.sh` waits up to 20 minutes on both targets (see
   `LLM_WAIT`).
 - A client abort and a server release look different in the journal. A
@@ -332,7 +460,7 @@ Direct edits on the remote drift from version control. Future agents will not
 know what changed. Git is the single source of truth. The two workloads share
 two hosts, one Caddy, and one Gitea. One repo keeps them in one history.
 
-> User request → Fetch from remote → Commit here → Edit → Test → Commit → Push → Pull on remote → Deploy + restart
+> User request → Fetch from remote → Commit here → Edit → Test → Commit → Push → Pull on remote → Dispatch changed service(s) → Health
 
 ## Summary
 
@@ -341,8 +469,8 @@ two hosts, one Caddy, and one Gitea. One repo keeps them in one history.
 | Editing a compose/Caddyfile/serve script | **Work through git** — fetch, commit, edit, test, push, pull, deploy. Never edit on the remote directly. |
 | Config exists only on remote | **Pull it into the repo first** — commit the fetched version (redacted) before making changes. |
 | Unstaged local changes | **Commit or discard** — check `git status` before starting work. Ask the user if unsure. |
-| Deploying a committed change | **Run `scripts/deploy.sh <target>`** — it pushes, pulls on the target, restarts the stack and the LLM service, and checks both health endpoints. See [docs/LLM/deployment.md](docs/LLM/deployment.md). |
+| Deploying a committed change | **Run `scripts/deploy.sh <target>`** — it pushes, pulls with fast-forward-only, plans changed paths, dispatches selected services, and checks their health. See [docs/LLM/deployment.md](docs/LLM/deployment.md). | | Container without a Quadlet unit | **Give it one** — self-healing, boot-persistent `.container` unit via git (see Boot Persistence & Self-Healing). |
-| Benchmarking | **Stop the service first** — scratch port, restart, verify health. See the Benchmarking Policy above. |
+| Benchmarking | **Stop the service first**, restart, verify health. See the Benchmarking Policy above. |
 | Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
 | Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
 | Real secrets | **Never commit** — use host `.env` files. `.env.example` holds placeholders only. `FRANK_SUDO_PW` covers headless sudo. |

README.md

@@ -37,7 +37,7 @@ one deploy script.
 - `dankserver/` — compose.yaml + Caddyfile + .env.example for the Dankserver stack
 - `dankclaw/` — compose.yaml + Caddyfile + .env.example for the Dankclaw stack
 - `shared/` — homepage, prometheus config, deploy helpers
-- `scripts/` — `deploy.sh` (unified: stack + LLM), LLM serve scripts, systemd units, env tunables, bench harnesses
+- `scripts/` — `deploy.sh` and `deploy-remote.sh` (service-scoped deployment), the planner, LLM serve scripts, systemd units, env tunables, and bench harnesses
 - `old/` — retired PP/NPU-era LLM scripts (reference only)
 - `docs/` — project knowledge base (Obsidian vault, wikilinks). `docs/LLM/` is the LLM fleet
 - `tests/` — bats test suite (see [[Testing Policy]])
@@ -46,9 +46,10 @@ one deploy script.
 
 See [docs/Architecture/Deployment Flow.md](docs/Architecture/Deployment%20Flow.md)
 and [docs/LLM/deployment.md](docs/LLM/deployment.md). Run
-`./scripts/deploy.sh <dankclaw|dankserver>` after pushing committed changes.
+`./scripts/deploy.sh <dankclaw|dankserver>` after committing changes. It
-It pushes, pulls on the target, restarts the podman stack and the LLM
+plans and deploys only the changed services. Use `--service NAME` for an
-service, and checks both health endpoints.
+explicit single-service retry; inference is restarted only when its own files
+(or the shared chat template) changed.
 
 ## Knowledge Base
 

dankclaw/.env.example

@@ -40,6 +40,11 @@ SIGNAL_ACCOUNT=+15551234567
 # Comma-separated E.164 numbers allowed to message the agent.
 SIGNAL_ALLOWED_USERS=+15551234567
 
+# Hermes web UI (Hermex server) login password. Required: the webui binds to
+# loopback and is reached only through the SSH tunnel, but Hermex still signs
+# in with this password. Set a strong value in the live .env.
+HERMES_WEBUI_PASSWORD=changeme
+
 # Porkbun API (franksdank.cloud + franksdank.casa zones). Caddy uses these
 # for DNS-01 Let's Encrypt wildcard cert issuance. Porkbun API keys are
 # account-wide; rotate them here rather than in the dashboard.

dankclaw/Caddyfile

@@ -190,11 +190,24 @@ pihole.franksdank.cloud {
 # Let's Encrypt cert for it. It stays plain-HTTP legacy; the canonical
 # name is plex.franksdank.cloud (HTTPS, below).
 http://plex.franksdank.casa {
-	reverse_proxy 192.168.0.235:32400
+	reverse_proxy 192.168.0.235:32400 {
+		transport http {
+			tls_insecure_skip_verify
+		}
+	}
 }
 
+# PMS requires TLS from every non-loopback client: plain-HTTP off-loopback
+# is read and silently closed (diagnosed 2026-08-26), which surfaced as an
+# empty-body 502 here. Dial the upstream over TLS; PMS's self-signed cert is
+# skipped on this pinned-LAN hop only (client-facing LE TLS is untouched).
+# The web UI lives at /web/; / returns the XML MediaContainer index.
 plex.franksdank.cloud {
-	reverse_proxy 192.168.0.235:32400
+	reverse_proxy 192.168.0.235:32400 {
+		transport http {
+			tls_insecure_skip_verify
+		}
+	}
 }
 
 # --- Calibre-Web ---

dankclaw/caddy/env.example

@@ -0,0 +1,7 @@
+# Caddy environment (placeholders only). The deploy writes the live file at
+# /home/dank/Containers/caddy/env from the checkout's .env; the quadlet reads
+# it via EnvironmentFile. Consumed by the Caddyfile's acme_dns block
+# (DNS-01 wildcard cert issuance). PORKBUN_API_SECRET_KEY is the name the
+# caddy-dns/porkbun module expects.
+PORKBUN_API_KEY=changeme
+PORKBUN_API_SECRET_KEY=changeme

dankclaw/caddy/quadlet/caddy.container

@@ -0,0 +1,54 @@
+# Quadlet unit: canonical Caddy reverse proxy + ACME (rootless, boot-persistent).
+#
+# Install path on dankclaw: ~/.config/containers/systemd/caddy.container
+# Unit name: caddy.service (container name: caddy)
+#
+# The homelab's single Caddy: it fronts every service on both hosts and issues
+# Let's Encrypt wildcard certs via DNS-01 (Porkbun). It left compose.yaml on
+# 2026-08-26 after a reboot exposed that the bare compose container never came
+# back: rootless Podman has no boot-time auto-start for bare containers, and
+# `restart: unless-stopped` does not survive the explicit stop that shutdown
+# performs. See AGENTS.md "Boot Persistence & Self-Healing" and
+# docs/Infrastructure/Caddy.md.
+#
+# It joins the existing compose network (dankclaw_default) so the Caddyfile's
+# container-name upstreams (authentik-server, grafana, ...) still resolve; if
+# no compose service has created that network yet at boot, quadlet creates it
+# first and compose reuses it by name later. The image is built locally from
+# dankclaw/caddy/Containerfile and exists only on this host, so Pull=never: a
+# boot must not stall on a registry that does not have it. Porkbun credentials
+# come from /home/dank/Containers/caddy/env (deploy writes it from the
+# checkout's .env; secrets never enter the repo).
+
+[Unit]
+Description=Caddy reverse proxy + ACME (Porkbun DNS-01) container
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+ContainerName=caddy
+Image=franksdank/caddy-porkbun:latest
+Pull=never
+Network=dankclaw_default
+PublishPort=0.0.0.0:80:80/tcp
+PublishPort=0.0.0.0:443:443/tcp
+EnvironmentFile=/home/dank/Containers/caddy/env
+Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
+Volume=/home/dank/Containers/caddy/data:/data
+Volume=/home/dank/Containers/caddy/config:/config
+# The homepage: the host's /srv/homepage (deploy-managed, not the repo's
+# shared/homepage).
+Volume=/srv/homepage:/srv/homepage:ro
+# The bare-IP catch-all answers 302 (curl -f passes on <400); this proves
+# Caddy itself is serving without touching any upstream.
+HealthCmd=curl -sf -o /dev/null http://localhost:80/
+HealthInterval=30s
+HealthTimeout=10s
+HealthRetries=3
+
+[Service]
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

dankclaw/compose.yaml

@@ -94,6 +94,44 @@ services:
     volumes:
       - /home/dank/hermes-data:/data
 
+  # --- Hermes web UI (Hermex server; phone-facing API for the agent) --------
+  # Runs the open-source hermes-webui (github.com/nesquena/hermes-webui) in
+  # foreground mode: bootstrap.py creates .venv inside the mounted checkout,
+  # installs WebUI + agent deps, then execs server.py as a single foreground
+  # process. It reads HERMES_HOME=/data, so it drives the SAME agent state
+  # (memory, skills, sessions) as the gateway container.
+  # Loopback-only publish: the SSH tunnel (phone -> Dankclaw:22) is the sole
+  # ingress. The password gates Hermex logins and lives in the host .env.
+  hermes-webui:
+    image: docker.io/nousresearch/hermes-agent:latest
+    container_name: hermes-webui
+    restart: unless-stopped
+    entrypoint: ["bash", "-c", "python3 /webui/bootstrap.py --foreground"]
+    environment:
+      HERMES_HOME: /data
+      # Container-internal bind: 0.0.0.0 so Podman's port forwarder (which
+      # dials the container's bridge IP) can reach it. The *host-side*
+      # publish below stays on 127.0.0.1, so loopback-only ingress holds.
+      HERMES_WEBUI_HOST: 0.0.0.0
+      HERMES_WEBUI_PORT: "8787"
+      HERMES_WEBUI_PASSWORD: "${HERMES_WEBUI_PASSWORD}"
+      # Headless host: never show the first-run wizard (a profile already
+      # exists in /data from the gateway).
+      HERMES_WEBUI_SKIP_ONBOARDING: "1"
+    ports:
+      - "127.0.0.1:8787:8787"
+    volumes:
+      - /home/dank/hermes-data:/data
+      # rw on purpose: bootstrap.py creates /webui/.venv in the repo root.
+      - /home/dank/hermes-webui:/webui
+    healthcheck:
+      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8787/health')"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      # First boot pip-installs WebUI + agent deps into a fresh venv.
+      start_period: 600s
+
   # --- Git hosting (Gitea) -------------------------------------------------
   # Now boot-persistent Quadlet units (private bridge network, DNS enabled),
   # not compose services. See gitea/quadlet/ and ../docs/Dankclaw/Gitea.md.
@@ -272,44 +310,8 @@ services:
       authentik-postgres:
         condition: service_healthy
 
-  # --- Reverse proxy + homepage (the homelab's single Caddy) ---------------
+  # Caddy (the homelab's single reverse proxy) left compose on 2026-08-26:
-  # The Caddyfile mounted here is the RENDERED copy (basicauth hash filled in
+  # it is boot-persistent via its Quadlet unit (dankclaw/caddy/quadlet/
-  # from the host .env by scripts/deploy.sh); the repo copy carries the
+  # caddy.container), installed by deploy_caddy in scripts/deploy-remote.sh.
-  # REPLACE_WITH_HASH placeholder. Certs/config live under
+  # The rendered Caddyfile, certs, and config still live under
   # /home/dank/Containers/caddy (rootless-writable).
-  caddy:
-    # Locally built: stock Caddy + the Porkbun DNS provider
-    # (dankclaw/caddy/Containerfile). See that file for build instructions.
-    image: franksdank/caddy-porkbun:latest
-    container_name: caddy
-    restart: unless-stopped
-    ports:
-      - "80:80"
-      - "443:443"
-    environment:
-      # Consumed by the Caddyfile's acme_dns block (DNS-01 wildcard certs).
-      PORKBUN_API_KEY: ${PORKBUN_API_KEY}
-      PORKBUN_API_SECRET_KEY: ${PORKBUN_API_SECRET_KEY}
-    volumes:
-      - /home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
-      - /home/dank/Containers/caddy/data:/data
-      - /home/dank/Containers/caddy/config:/config
-      # The homepage: the host's /srv/homepage. The deploy copies the static
-      # files there; the status checker writes status.json and the deploy
-      # writes deployed.js there. (Not the repo's shared/homepage, which
-      # lacks the generated files.)
-      - /srv/homepage:/srv/homepage:ro
-    depends_on:
-      - authentik-server
-      - comfyui
-      - hermes
-      - grafana
-      - prometheus
-      - loki
-    healthcheck:
-      # The bare-IP catch-all answers 302 (curl -f passes on <400); this
-      # proves Caddy itself is serving without touching any upstream.
-      test: ["CMD", "curl", "-sf", "-o", "/dev/null", "http://localhost:80/"]
-      interval: 30s
-      timeout: 10s
-      retries: 3

dankclaw/gitea/quadlet/gitea-caddy.container

@@ -1,31 +0,0 @@
-# Quadlet unit: Gitea Caddy reverse proxy (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/gitea-caddy.container
-# Unit name: gitea-caddy.service (podman 5.x drops the container- prefix)
-#
-# Serves gitea.franksdank.cloud (public domain, DNS pending). The LAN
-# reaches Gitea directly on port 3000. This file holds no secrets.
-# See docs/Dankclaw/Gitea.md.
-
-[Unit]
-Description=Gitea Caddy reverse proxy container
-Wants=gitea-server.service
-After=gitea-server.service network.target
-
-[Container]
-ContainerName=gitea-caddy
-Network=gitea.network
-Image=docker.io/library/caddy:latest
-PublishPort=80:80
-PublishPort=443:443
-Volume=/opt/gitea/data/caddy/Caddyfile:/etc/caddy/Caddyfile
-Volume=/opt/gitea/data/caddy/site:/srv
-Volume=/opt/gitea/data/caddy/config:/config
-Volume=/opt/gitea/data/caddy/data:/root/.local/share/caddy
-
-[Service]
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/gitea/quadlet/gitea.network

@@ -4,7 +4,7 @@
 # Unit name: gitea-network.service (podman network name: systemd-gitea)
 #
 # DNS is enabled by default for quadlet networks. This matters: the
-# automatic "podman" default network has dns_enabled=false, so container
+# automatic "podman" default network has dns_enabled=false, so the
-# names (gitea-postgres, gitea-server, gitea-caddy) would not resolve.
+# gitea-postgres and gitea-server container names would not resolve.
 
 [Network]

dankclaw/hermes/config.yaml.example

@@ -0,0 +1,58 @@
+# Hermes configuration backup / non-secret baseline.
+#
+# Source on Dankclaw: /home/dank/hermes-data/config.yaml
+# This file is a sanitized Git backup, not a complete replacement for the
+# live Hermes state. Do not add dashboard password hashes, phone numbers,
+# OAuth credentials, API keys, sessions, memories, or generated state here.
+# The live directory is bind-mounted into the containers as /data.
+
+model:
+  provider: dankclaw-large
+  base_url: http://192.168.0.143:7770/v1
+  api_key: not-needed
+  default: frank-default-large
+
+providers:
+  dankclaw-large:
+    api: http://192.168.0.143:7770/v1
+    discover_models: false
+    models:
+      frank-default-large:
+        context_length: 262144
+  dankserver-small:
+    api: http://192.168.0.235:7770/v1
+    discover_models: false
+    models:
+      frank-default-small:
+        context_length: 163840
+
+# Auxiliary routing from the live configuration. These use the existing
+# custom endpoint shape and are retained here as a settings backup.
+auxiliary:
+  vision: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  web_extract: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  compression: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  skills_hub: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  approval: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  mcp: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  title_generation: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  memory_query_rewrite: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  tts_audio_tags: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  triage_specifier: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  kanban_decomposer: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  profile_describer: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+  curator: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
+
+dashboard:
+  # basic_auth intentionally omitted; its password_hash remains live-only.
+  font: atkinson-hyperlegible
+skills:
+  disabled: []
+onboarding:
+  seen:
+    busy_input_prompt: true
+_config_version: 34
+platforms:
+  signal:
+    enabled: true
+    # extra.http_url and account intentionally omitted from the backup.

dankclaw/pihole/addn-hosts

@@ -1,6 +1,6 @@
 # A records for the flat franksdank.name scheme (dnsmasq addn-hosts format:
 # <IP> <name> [<name> ...]). Served by the Pi-hole quadlet (FTL is a
-# dnsmasq fork): the quadlet mounts this file into /etc/dnsmasq.d/ read-only.
+# dnsmasq fork): the quadlet mounts this file into /etc/pihole/hosts/ read-only.
 #
 # Caddy is consolidated on dankclaw (192.168.0.143): it fronts every service
 # on both hosts, so every name resolves to that one IP. Keep in sync with the

dankclaw/pihole/quadlet/pihole.container

@@ -6,7 +6,7 @@
 # The LAN's authoritative DNS for the flat franksdank.cloud / franksdank.casa
 # name scheme. It retired the dnsmasq quadlet (2026-08-24): Pi-hole FTL is a
 # dnsmasq fork, so the addn-hosts record file carries over unchanged and is
-# mounted into /etc/dnsmasq.d/ (the FTL dnsmasq config dir). It serves one A
+# mounted into /etc/pihole/hosts/ (the FTL hostsdir) read-only. It serves one A
 # record per Caddy vhost and forwards everything else to the pinned upstream
 # resolvers (DNS1/DNS2 in the env file). The SURFboard G36 router hands this
 # server out to DHCP clients (DNS statically configured -> 192.168.0.143),

dankclaw/pihole/sysctl/99-dnsmasq.conf

@@ -0,0 +1,8 @@
+# Rootless DNS must bind port 53: lower the unprivileged-port threshold so a
+# rootless container may publish 0.0.0.0:53 (80/443 stay allowed; 53 < 80).
+# Single source of truth for net.ipv4.ip_unprivileged_port_start on dankclaw;
+# deploy_pihole installs it to /etc/sysctl.d/ and applies it with
+# `sysctl --system` (the same code path boot uses via systemd-sysctl).
+# A later-sorted drop-in that reset this to 80 took LAN DNS down after the
+# 2026-08-26 reboot — deploy_pihole neutralizes such conflicts.
+net.ipv4.ip_unprivileged_port_start=53

docs/.obsidian/app.json

@@ -5,4 +5,4 @@
   "attachmentFolderPath": "_attachments",
   "useMarkdownLinks": false,
   "showUnsupportedFiles": true
-}
+}

docs/Architecture/Deployment Flow.md

@@ -1,60 +1,128 @@
+---
+type: guide
+title: Deployment Flow
+description: Service-scoped Git deployment for the two homelab hosts.
+tags:
+  - deployment
+  - operations
+  - git
+timestamp: 2026-09-03
+---
+
 # Deployment Flow
 
-Git is the single source of truth. A `deploy.sh` script moves committed
+Git is the single source of truth. `scripts/deploy.sh` pushes the committed
-changes to the target hosts over SSH. One run per target covers both
+`main` branch, pulls it on one target, and applies only the runtime services
-workloads on that host: the podman stack and the LLM service.
+whose tracked inputs changed. A SearXNG change no longer recreates the
+inference service; an LLM change no longer restarts unrelated containers.
 
 ## Workflow
 
 ```mermaid
 graph LR
-    A[Edit in this repo] --> B[Commit]
+    A[Edit in this repo] --> B[Run tests]
-    B --> C[Push to origin]
+    B --> C[Commit main]
-    C --> D[deploy.sh: pull on target]
+    C --> D[deploy.sh computes plan]
-    D --> E[restart stack]
+    D --> E[Push main]
-    E --> F[health check]
+    E --> F[Target pulls with ff-only]
+    F --> G[deploy-remote.sh dispatches selected services]
+    G --> H[Per-service health check]
 ```
 
 1. Edit files locally in this repository.
-2. Commit and push.
+2. Run `./tests/run.sh`, then commit the complete change on `main`.
-3. Run `./scripts/deploy.sh <dankclaw|dankserver>`.
+3. Run `./scripts/deploy.sh <target>`.
-4. The script pulls the repo on the target and restarts the podman stack
+4. The planner maps changed paths to services. Compose YAML is compared by
-   and the LLM service.
+   service definition so an edit to one service does not restart its peers.
-5. It verifies service health after restart (homepage and LLM endpoint).
+5. The target pulls with `git pull --ff-only origin main` and runs the remote
+   dispatcher with the explicit plan.
+6. The dispatcher updates only selected Quadlet, compose, homepage, Caddy, or
+   systemd services and records each successful service revision outside Git.
 
-## Deploy Script
+## Commands
 
-```
+```bash
-./scripts/deploy.sh dankclaw          # pull + restart on Dankclaw
+./scripts/deploy.sh dankclaw
-./scripts/deploy.sh dankserver        # pull + restart on Dankserver
+./scripts/deploy.sh dankserver
+./scripts/deploy.sh dankclaw --service searxng
+./scripts/deploy.sh dankserver --service frank-default-small
 ./scripts/deploy.sh dankclaw --dry-run
 ```
 
-SSH uses the existing key pair (`dank@dankclaw`, `dank@dankserver`). No passwords are stored or documented in this repo. If sudo is needed, use the interactive sudo on the target — never commit a password.
+`--service` is repeatable and is an explicit retry/override. Without it, the
+planner compares the local `HEAD` to `origin/main`. A documentation-only
+commit produces an empty plan. The remote state directory can also replay a
+service whose earlier deployment failed, but it never expands a plan to a
+service on the other host.
+
+## Service managers
+
+| Service family | Runtime manager | Scope behavior |
+|---|---|---|
+| Dankclaw compose services | `podman-compose -f dankclaw/compose.yaml up -d --no-deps NAME` | Recreates only `NAME`; Caddy is handled separately because it fronts both hosts. |
+| SearXNG, Gluetun, Valkey, OpenWebUI, Pi-hole, Gitea database/server | user Quadlet + `systemctl --user` | Installs the selected unit and restarts only the required dependency-coupled units. Gitea's web proxy is the canonical Caddy compose service. |
+| `frank-default-large` | systemd + `systemctl` | Relinks and restarts only the large inference service; waits up to 20 minutes for `/health`. |
+| `frank-default-small` | systemd + `systemctl` | Relinks and restarts only the small inference service; waits up to 20 minutes for `/health`. |
+| Dankserver media inventory | host-managed bare containers | Compose is inventory only; the scoped deploy does not create duplicate containers. |
+
+The shared chat template maps to the inference service on the selected target.
+The inactive `frank-default-npu` unit is relink-only when explicitly selected;
+it is not restarted by an unrelated large-model deployment.
+
+## Caddy safety
+
+Caddy runs on Dankclaw and fronts both hosts, including Gitea. The former
+standalone `gitea-caddy` Quadlet is retired; the canonical compose `caddy`
+service is now the sole reverse-proxy owner of ports 80/443. A Caddy reload
+failure falls back to a restart of **only** the selected `caddy` container.
+Before deploying, verify that no retired proxy still owns either port.
+
+Homepage files, generated `deployed.js`, the status checker, and the rendered
+Caddyfile are updated only when their service is selected. A media-only
+Dankserver deployment does not touch Dankclaw's proxy or homepage.
+
+## Per-service state
+
+The remote runner writes one revision file per successfully completed service
+to:
+
+```text
+${XDG_STATE_HOME:-~/.local/state}/serverAppConfigs/deploy-state/
+```
+
+The directory is outside the checkout and is never committed. A failed handler
+is not marked successful. On the next ordinary deployment, the runner compares
+the recorded revision with the new checkout and retries only a still-changed
+service. This prevents a failed Caddy or LLM deployment from being silently
+forgotten while avoiding a host-wide restart.
 
 ## Targets
 
-| Target | Host | Stack |
+| Target | Host | Repository checkout |
-|--------|------|-------|
+|---|---|---|
-| dankclaw | dank@dankclaw (192.168.0.143) | `dankclaw/` |
+| dankclaw | `dank@dankclaw` (`192.168.0.143`) | `/home/dank/serverAppConfigs` |
-| dankserver | dank@dankserver (192.168.0.235) | `dankserver/` |
+| dankserver | `dank@dankserver` (`192.168.0.235`) | `/home/dank/serverAppConfigs` |
 
-## Health Verification
+SSH uses the existing key pair. No password is stored or documented in this
-
+repository. If sudo is needed on a headless target, `FRANK_SUDO_PW` is read
-The deploy script polls the health endpoint of each service after restart. Service healthchecks are defined in each compose file. A service that fails its healthcheck is reported, not hidden.
+from the environment and is never written to the repository.
 
 ## Tests
 
-Run the test suite before pushing:
+Run the full suite before pushing:
 
-```
+```bash
 ./tests/run.sh
 ```
 
-The suite validates compose YAML, env consistency, Caddyfiles, docs completeness, and script syntax. See [[Testing Policy]].
+The deployment seams are in `tests/deployment_scoping.bats` and
+`tests/deployment_flow.bats`. They cover path isolation, compose narrowing,
+Caddy safety, inference isolation, per-service state, and legacy wrapper
+routing. See [[Testing Policy]].
 
 ## See Also
 
-- [[Storage Policy]]
 - [[Testing Policy]]
-- [LLM deployment docs](../LLM/deployment.md) — the LLM half of the same flow
+- [LLM deployment](../LLM/deployment.md)
+- [[Caddy]]
+- [[SearXNG]]

docs/Architecture/Testing Policy.md

@@ -11,7 +11,29 @@ This repository tests its configuration at defined seams. The tests run with `ba
 | Caddyfile validity | `tests/caddyfile.bats` | per-host Caddyfile, `.local` convention, balanced braces, basic auth present |
 | Docs completeness | `tests/docs_completeness.bats` | every compose service has a docs page; index links every page |
 | Script syntax | `tests/scripts_syntax.bats` | `bash -n` clean, executable bits, shellcheck when available |
-| Deployment flow | `tests/deployment_flow.bats` | deploy.sh exists, targets known, SSH reachable (skips offline) |
+| Deployment flow | `tests/deployment_flow.bats`, `tests/deployment_scoping.bats` | scoped dispatcher exists, targets are known, service plans isolate inference, Caddy ownership is guarded, and SSH reachability is checked (skips offline) |
+
+## Focused pre-commit testing
+
+For normal commits, use the focused wrapper:
+
+```bash
+./scripts/test-changed.sh
+```
+
+It reads staged paths with `git diff --cached` and runs only the mapped Bats
+files. The tracked `.githooks/pre-commit` hook invokes this wrapper. Enable the
+repository hook path once per checkout:
+
+```bash
+git config core.hooksPath .githooks
+```
+
+Use `--list` to inspect the selection without running tests, or `--paths FILE
+...` to preview a mapping. Shared test infrastructure and unmapped paths fail
+closed rather than silently skipping coverage. Run the full suite with
+`./tests/run.sh` when changing shared test helpers, the test runner, or when
+focused coverage is not sufficient.
 
 ## Red-Green
 

docs/Dankclaw/Dankclaw Stack.md

@@ -1,3 +1,15 @@
+---
+type: reference
+title: Dankclaw Stack
+description: Service inventory and routing overview for the Dankclaw AI, development, and observability host.
+tags:
+  - dankclaw
+  - services
+  - containers
+  - caddy
+timestamp: 2026-08-26
+---
+
 # Dankclaw Stack
 
 Dankclaw (192.168.0.143) runs the AI, search, and development stack. It
@@ -43,9 +55,9 @@ repo owns their scripts, units, and knowledge base (see
 
 ## Boot Persistence
 
-The Gitea trio, the SearXNG trio (gluetun, searxng, valkey), and open-webui run as quadlet units, so they start at boot. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent.
+The Gitea database and server, the SearXNG trio (gluetun, searxng, valkey), open-webui, Pi-hole, and Caddy run as Quadlet units, so they start at boot. Caddy left `compose.yaml` on 2026-08-26 after a reboot exposed that bare compose containers never come back; see [[Caddy]] for the unit and the incident. The former standalone Gitea Caddy unit is retired. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent — per AGENTS.md (Boot Persistence & Self-Healing) they each need a quadlet before they may be considered reliably deployed.
 
-Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw` and open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]).
+Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw`, open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`, and Pi-hole with `scripts/deploy-pihole-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]); Caddy is deployed through `scripts/deploy.sh dankclaw --service caddy`.
 
 ## Routing Rules
 

docs/Dankclaw/Gitea.md

@@ -1,8 +1,20 @@
+---
+type: configuration
+title: Gitea
+description: Gitea deployment on Dankclaw with its web UI reverse-proxied by the canonical Caddy service.
+tags:
+  - gitea
+  - dankclaw
+  - quadlet
+  - caddy
+timestamp: 2026-08-26
+---
+
 # Gitea
 
-Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On Dankclaw it runs as three Quadlet-managed containers in the rootless podman session of user `dank`.
+Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On Dankclaw it runs as two Quadlet-managed containers in the rootless Podman session of user `dank`. The canonical Caddy on the same host reverse-proxies its web UI at `gitea.franksdank.cloud`.
 
-**Host:** Dankclaw — **LAN access:** `http://dankclaw:3000` (or `http://192.168.0.143:3000`) — **Public domain:** `gitea.franksdank.cloud` (DNS pending) — **Docs:** https://docs.gitea.com
+**Host:** Dankclaw — **LAN access:** `http://dankclaw:3000` (or `http://192.168.0.143:3000`) — **Proxy domain:** `gitea.franksdank.cloud` — **Docs:** https://docs.gitea.com
 
 ## Containers
 
@@ -10,7 +22,6 @@ Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On
 |-----------|------|-------|---------|
 | `gitea-postgres` | `gitea-postgres.service` | `docker.io/library/postgres:16-alpine` | Database |
 | `gitea-server` | `gitea-server.service` | `docker.io/gitea/gitea:latest` | Web UI, API, git (HTTP :3000, SSH :2222) |
-| `gitea-caddy` | `gitea-caddy.service` | `docker.io/library/caddy:latest` | Reverse proxy for the public domain |
 
 The containers share the private bridge network `systemd-gitea`. The quadlet network file `gitea.network` enables DNS, so the containers resolve each other by name. The automatic `podman` default network has DNS disabled. Do not run the Gitea containers on it.
 
@@ -29,10 +40,10 @@ Run all commands as `dank` on Dankclaw.
 
 | Action | Command |
 |--------|---------|
-| Start all | `systemctl --user start gitea-postgres gitea-server gitea-caddy` |
+| Start all | `systemctl --user start gitea-postgres gitea-server` |
-| Stop all | `systemctl --user stop gitea-caddy gitea-server gitea-postgres` |
+| Stop all | `systemctl --user stop gitea-server gitea-postgres` |
 | Restart Gitea only | `systemctl --user restart gitea-server` |
-| Status | `systemctl --user status gitea-postgres gitea-server gitea-caddy` |
+| Status | `systemctl --user status gitea-postgres gitea-server` |
 | Container logs | `podman logs -f gitea-server` |
 | Unit logs | `journalctl --user -u gitea-server -f` |
 
@@ -42,14 +53,13 @@ The units start in dependency order:
 graph LR
     N[gitea-network.service] --> P[gitea-postgres.service]
     P --> S[gitea-server.service]
-    S --> C[gitea-caddy.service]
 ```
 
 ## Boot persistence
 
 User-level linger is enabled for `dank` (`loginctl enable-linger dank`). The units start at host boot without a login session. Each unit carries `[Install] WantedBy=default.target`. The podman generator applies it at boot; no symlink is needed.
 
-The containers use the quadlet fresh-start model. A stop removes the container (`podman run --rm --replace`). The next start recreates it. The bind-mounted data survives.
+The containers use the quadlet fresh-start model. A stop removes the container (`podman run --rm --replace`). The next start recreates it. The bind-mounted data survives. The canonical Caddy is managed separately by the Dankclaw stack.
 
 ## Secrets
 
@@ -61,11 +71,11 @@ Do not change the password in place. Change the PostgreSQL role first, then upda
 
 | File | Location | Repo copy |
 |------|----------|-----------|
-| Quadlet units | `~/.config/containers/systemd/gitea-{postgres,server,caddy}.container`, `gitea.network` | [`dankclaw/gitea/quadlet/`](../../dankclaw/gitea/quadlet/) |
+| Quadlet units | `~/.config/containers/systemd/gitea-{postgres,server}.container`, `gitea.network` | [`dankclaw/gitea/quadlet/`](../../dankclaw/gitea/quadlet/) |
-| Caddyfile | `/opt/gitea/data/caddy/Caddyfile` | host-only (not in repo) |
+| Caddyfile | `/home/dank/Containers/caddy/Caddyfile` | [`dankclaw/Caddyfile`](../../dankclaw/Caddyfile) |
 | Secrets | `/opt/gitea/.env` (0600) | [`dankclaw/gitea/.env.example`](../../dankclaw/gitea/.env.example) |
 
-After editing a repo copy, copy it to the target, then run `systemctl --user daemon-reload` and restart the affected units.
+After editing a repo copy, deploy through `scripts/deploy.sh dankclaw`. The Gitea database/server units are installed from this directory; the canonical Caddy and its rendered Caddyfile are deployed as the separate `caddy` service. Do not manually recreate the retired `gitea-caddy` unit.
 
 ## Data volumes
 
@@ -73,7 +83,6 @@ After editing a repo copy, copy it to the target, then run `systemctl --user dae
 |----------------|-----------|----------|
 | `/data` | `/opt/gitea/data/gitea-gitea` | Git repositories, app.ini, attachments |
 | `/var/lib/postgresql/data` | `/opt/gitea/data/gitea-postgres` | PostgreSQL data |
-| `/etc/caddy/Caddyfile`, `/root/.local/share/caddy` | `/opt/gitea/data/caddy/...` | Caddy config and TLS state |
 
 Back up `/opt/gitea/data/` (tar) or `pg_dump` the database. See the knowledge base page [[Gitea]].
 
@@ -83,8 +92,8 @@ Back up `/opt/gitea/data/` (tar) or `pg_dump` the database. See the knowledge ba
 - **The files in the data directory are owned by host UID 100999**, not `dank` (rootless ID mapping: container uid 1000 → host 100999). Edit them inside the container with `podman exec --user 1000 gitea-server ...`.
 - **The host auto-reboots.** `unattended-upgrades` reboots Dankclaw when a package needs it. The quadlet units make Gitea survive that. See the 2026-08-16 incident below.
 - **Port 3000 vs 3001.** The live host port is 3000. The full-stack [`dankclaw/compose.yaml`](../../dankclaw/compose.yaml) maps 3001. Align the two before deploying the full stack.
-- **Legacy compose retired.** `/opt/gitea/podman-compose.yml` is retired to `/opt/gitea/.deprecated/`. Do not run it. Its containers collide with the quadlet units on ports 3000, 2222, 80, and 443.
+- **Legacy compose retired.** `/opt/gitea/podman-compose.yml` is retired to `/opt/gitea/.deprecated/`. Do not run it. Its containers collide with the Quadlet units on ports 3000 and 2222.
-- **Rootless cannot bind ports below 1024 by default.** `gitea-caddy` publishes 80 and 443. Without a host sysctl, `rootlessport` fails with `permission denied` and the unit crash-loops under `Restart=always`. The fix is a one-time sysctl: `net.ipv4.ip_unprivileged_port_start=80`, persisted in `/etc/sysctl.d/99-rootless-ports.conf`. Applied 2026-08-18 after a 13,000-restart crash loop.
+- **Canonical proxy.** `gitea.franksdank.cloud` is served by the Caddy service in `dankclaw/compose.yaml`, which proxies to `192.168.0.143:3000`. Do not recreate a Gitea-specific Caddy unit or publish another proxy on ports 80/443. During migration, the deployer stages any installed legacy unit under `~/.AGENTTRASH/YYYYMMDD-gitea-caddy/` before starting canonical Caddy.
 
 ## Incident: 2026-08-16 — down after unattended reboot
 

docs/Dankclaw/Hermes-Dashboard.md

@@ -22,7 +22,7 @@ The gateway container runs `hermes gateway run` (the Signal platform). That entr
 
 The dashboard refuses to bind to a non-loopback address without an auth provider. There is no unauthenticated public bind. This repo uses basic auth:
 
-- `dashboard.basic_auth.username` and `dashboard.basic_auth.password_hash` live in the git-ignored `/home/dank/hermes-data/config.yaml`.
+- `dashboard.basic_auth.username` and `dashboard.basic_auth.password_hash` live in the live `/home/dank/hermes-data/config.yaml`; the tracked backup at `dankclaw/hermes/config.yaml.example` intentionally omits the password hash.
 - The stored value is a one-way scrypt hash. The plaintext password is never written to a file or the repo.
 - The hash is generated with `plugins.dashboard_auth.basic.hash_password`.
 
@@ -35,3 +35,7 @@ Open `http://192.168.0.143:9119` in a browser on the LAN. Log in with the config
 - The dashboard and the gateway share one data volume. A config change made in the dashboard takes effect when the gateway restarts.
 - If the shared password is reused elsewhere, treat it as exposed. Rotate it and regenerate the hash.
 - `--skip-build` uses the prebuilt web assets in the image. Do not remove it unless the assets are rebuilt.
+
+## See Also
+
+- [[Hermes-WebUI]] — the phone-facing API server (Hermex)

docs/Dankclaw/Hermes-WebUI.md

@@ -0,0 +1,114 @@
+---
+type: configuration
+title: Hermes-WebUI (Hermex server)
+description: The hermes-webui compose service on Dankclaw — the loopback-only API server that the Hermex iPhone app talks to over an SSH tunnel.
+tags:
+  - hermes
+  - hermes-webui
+  - hermex
+  - dankclaw
+  - ssh-tunnel
+timestamp: 2026-09-14
+---
+
+# Hermes-WebUI (Hermex server)
+
+`hermes-webui` is the [hermes-webui](https://github.com/nesquena/hermes-webui)
+server (MIT, third-party) that the **Hermex** iPhone app connects to. It runs
+the Hermes agent in-process from the shared `HERMES_HOME`, so it is the *same*
+agent — same memory, skills, and sessions — as the [[Hermes]] gateway and the
+[[Hermes-Dashboard]].
+
+**Host:** Dankclaw — **Port:** `127.0.0.1:8787` (loopback only) — **Ingress:** SSH tunnel
+
+## Facts
+
+| Item | Value |
+|------|-------|
+| Image | `docker.io/nousresearch/hermes-agent:latest` (same as the gateway) |
+| Command | `python3 /webui/bootstrap.py --foreground` |
+| Source checkout | `/home/dank/hermes-webui` on Dankclaw (mounted at `/webui`) |
+| Port | `8787`, published to `127.0.0.1` only |
+| Auth | `HERMES_WEBUI_PASSWORD` (host `.env`; Hermex signs in with it) |
+| Data | shares `/home/dank/hermes-data` with the gateway and dashboard |
+| Health | `GET /health` (compose healthcheck + deployer) |
+
+## Why a container, not a host process
+
+The gateway's state files are owned by the container-mapped uid (`109999`,
+from rootless Podman's `1 → 100000/65536` subuid range). A fresh rootless
+namespace on Dankclaw maps identically (verified with `podman unshare
+cat /proc/self/uid_map`), so a second container using the same image reads
+and writes the shared volume with exactly the ownership the gateway already
+uses. A host-side process running as `dank` (uid 1000) could not read the
+gateway's `0600` state files.
+
+## How it works
+
+- `bootstrap.py` creates `/webui/.venv` on first boot, pip-installs the WebUI
+  and agent requirements, then **execs** `server.py` as one foreground process
+  (`--foreground`). systemd-free: Podman supervises it via
+  `restart: unless-stopped`.
+- `HERMES_WEBUI_SKIP_ONBOARDING=1` keeps the first-run wizard off; a profile
+  already exists in `/data` from the gateway.
+- The healthcheck `start_period` is 600 s because the very first boot spends
+  minutes pip-installing into a fresh venv.
+
+## Remote access (Hermex on iOS, from anywhere)
+
+Only port `22` is forwarded to Dankclaw at the router, so the webui is
+reached through an SSH local forward — no extra router ports, no third-party
+relay (no Tailscale, no Cloudflare):
+
+1. On the iPhone, open **Termius** (or any SSH client with local port
+   forwarding) and connect to `<dankclaw-public>:22` with the usual key.
+2. On that connection, add a **local forward**: `127.0.0.1:8787` →
+   `127.0.0.1:8787`. The forward runs inside the encrypted SSH session;
+   nothing else on the phone routes through it.
+3. In Hermex, set the server URL to `http://localhost:8787` and sign in with
+   the webui password.
+
+The tunnel lives as long as the SSH session on the phone — iOS does not allow
+a true background daemon, so connect before you use Hermex.
+
+```mermaid
+graph LR
+    A[Hermex app] -->|http://localhost:8787| B[SSH local forward on iPhone]
+    B ==>|encrypted SSH to public IP:22| C[Dankclaw sshd]
+    C --> D["hermes-webui 127.0.0.1:8787"]
+    D --> E["/home/dank/hermes-data (shared agent state)"]
+```
+
+## Deployment
+
+```
+./scripts/deploy.sh dankclaw --service hermes-webui
+```
+
+The planner maps changes to the `hermes-webui` compose block (and only that
+block) to this service, so gateway and inference restarts stay isolated.
+Before first deploy, set a strong `HERMES_WEBUI_PASSWORD` in the live
+`dankclaw/.env`.
+
+## Gotchas
+
+- The `/webui` mount is **rw on purpose**: `bootstrap.py` creates `.venv` in
+  the repo root. Do not flip it to `:ro`.
+- Never publish `8787` beyond loopback without keeping the password set —
+  the webui can run agent commands.
+- The webui and the gateway share one data volume. A config change made on one
+  side takes effect when the other restarts (same caveat as the dashboard).
+- `bootstrap.py` auto-installs a Hermes Agent if it cannot find one; in this
+  image the agent is already present, so that path should never fire. If the
+  log shows an install attempt, the image changed — re-verify the entrypoint.
+
+# Citations
+
+- hermes-webui README (env vars, `--foreground`, `/health`): https://github.com/nesquena/hermes-webui
+- Hermex setup model (server URL + password; Cloudflare/Tailscale are optional, not required): https://hermexapp.com/
+
+## See Also
+
+- [[Hermes]] — the agent and its Signal gateway
+- [[Hermes-Dashboard]] — the LAN browser management UI
+- [[Dankclaw Stack]]

docs/Dankclaw/Hermes.md

@@ -4,6 +4,10 @@ Hermes is the self-improving AI agent from Nous Research. It runs in a terminal,
 
 **Host:** Dankclaw — **Hostname:** `hermes.dankclaw.local` — **Docs:** https://hermes-agent.nousresearch.com/docs
 
+**Persistent state:** `/home/dank/hermes-data` on Dankclaw, mounted as `/data` in
+both containers. The tracked, sanitized configuration baseline is
+[`dankclaw/hermes/config.yaml.example`](../../dankclaw/hermes/config.yaml.example).
+
 ## Facts (from research, primary source)
 
 | Item | Value |
@@ -17,15 +21,21 @@ Hermes is the self-improving AI agent from Nous Research. It runs in a terminal,
 
 ## Configuration
 
-- Point Hermes at the local model endpoint (`http://localhost:7770/v1`) or an external provider through its provider config.
+- Point Hermes at the local model endpoint or an external provider through its provider config.
 - The gateway process exposes one port for chat integration; the exact port is set in the Hermes config (`8090` convention in this repo).
 - The data volume persists skills, memory, and session history.
+- Hermes has two named local providers: `dankclaw-large` reaches
+  `http://192.168.0.143:7770/v1` and exposes `frank-default-large`;
+  `dankserver-small` reaches `http://192.168.0.235:7770/v1` and exposes
+  `frank-default-small`.
+- The live `config.yaml` contains secrets and user state and is not committed;
+  the tracked example is the sanitized backup of its non-secret settings.
 
 ## Volumes
 
 | Container path | Host path (convention) |
 |----------------|------------------------|
-| `/data` | `/opt/hermes/data` |
+| `/data` | `/home/dank/hermes-data` |
 
 ## Gotchas
 
@@ -35,6 +45,8 @@ Hermes is the self-improving AI agent from Nous Research. It runs in a terminal,
 
 ## See Also
 
+- [[Hermes-Dashboard]] — the LAN browser management UI
+- [[Hermes-WebUI]] — the Hermex server (phone-facing API, SSH-tunneled)
 - [[Dankclaw Stack]]
 - [[OpenWebUI]] — the chat UI that pairs with Hermes
 - Knowledge base: [[SearXNG]] (Hermes as the search agent)

docs/Dankclaw/SearXNG Providers.md

@@ -0,0 +1,400 @@
+# SearXNG Providers
+
+How Google and other upstream search engines get blocked behind the Gluetun
+VPN tunnel, and how to unblock them in SearXNG.
+
+**Host:** Dankclaw — **Docs:** https://docs.searxng.org
+
+SearXNG aggregates ~50 upstream engines (Google, Bing, DuckDuckGo, Brave,
+Wikipedia, …). When all of them are queried through **one** VPN exit IP — as on
+Dankclaw, where SearXNG reuses Gluetun's tunnel — the rate-sensitive engines
+start answering with reCAPTCHA or HTTP 429. SearXNG then *suspends* the offending
+engine for an hour to 15 days and silently drops it from results. That is the
+"Google (and sometimes a few others) went unavailable" symptom.
+
+This page is the fixbook. Read [[SearXNG]] for the stack; the sub-pages below
+are the detail if you need it.
+
+## Provider policy
+
+The default aggregation should maximize **independent, anonymous coverage**
+without turning one user query into a request storm. Disabled engines remain
+installed and selectable on demand; `inactive` is reserved for engines that
+should disappear completely.
+
+| Category | Default baseline | On-demand / deferred |
+|----------|------------------|----------------------|
+| General/web | Bing, DuckDuckGo, Wikipedia | Google CSE, Brave, Startpage, Qwant, Mojeek, Wiby, MWMBL after retesting |
+| Technical | GitHub, MDN, Stack Overflow, Ask Ubuntu, Superuser, Docker Hub, PyPI | Other technical indexes as needed |
+| Science | arXiv, PubMed, Semantic Scholar, OpenAlex-compatible sources | Google Scholar until independently tested |
+| News | Reuters, Bing News, DuckDuckGo News, Wikinews | Google News, Brave News, Startpage News |
+| Images | Openverse, Wikimedia Commons, Bing Images, Unsplash | Google/Brave/Startpage image engines |
+| Videos | YouTube, Vimeo, Dailymotion, Bing Videos | Brave video engines |
+
+This policy uses no API credentials and does not identify the operator to a
+search provider. It is deliberately conservative: a bundled engine is not
+necessarily healthy, anonymous, or suitable for every default query.
+
+## Root cause (one line)
+
+All SearXNG engine traffic shares **one ProtonVPN exit IP** with no rotation, so
+Google/DuckDuckGo see one datacenter IP spamming ~50 paginated engines and
+rate-limit it → CAPTCHA → engine **suspension** → empty results.
+
+## The suspension model (what "unavailable" actually is)
+
+An engine that returns 429 / reCAPTCHA / access-denied raises a named
+exception and is *suspended* (removed from result aggregation for a set time).
+Suspended engines are the usual cause of "gone":
+
+| Exception | Default suspension | Trigger |
+|-----------|--------------------|---------|
+| `SearxEngineTooManyRequests` | ~1 hour | HTTP 429 |
+| `SearxEngineAccessDenied` | ~1 day | 403 / access denied |
+| `SearxEngineCaptcha` | ~1 day | Google `/sorry/` or reCAPTCHA |
+| `recaptcha_SearxEngineCaptcha` | ~7 days | Google reCAPTCHA |
+| `cf_SearxEngineCaptcha` | ~15 days | Cloudflare edge CAPTCHA |
+| ordinary errors (timeout/parse) | 5–120 s | `ban_time_on_fail`/`max_ban_time_on_fail` |
+
+The values are `search.suspended_times` (+ `ban_time_on_fail`) in
+`settings.yml`. An engine that returns **0 results without an error** is *not*
+suspended — that is the separate "selector drift" problem (below).
+
+## Two failure modes
+
+| Mode | Symptom | Root cause | Fix family |
+|------|---------|-----------|-----------|
+| **CAPTCHA / 429 / access-denied** | engine flips `unresponsive`, then `suspended` for hours | blocked from the shared VPN exit IP | routing/proxies + request tuning (see below) |
+| **Selector drift** | engine returns 0 results, no error banner | Google renamed its minified CSS classes the engine's hardcoded XPath looks for | engine config (duplicate engine / JSON feed) |
+
+Most "Google is broken on a VPN host" cases are a **mix of both**.
+
+## Fix A — Engine config (settings.yml `engines:`)
+
+The current tracked policy enables Bing and DuckDuckGo for default general
+search, gives each weight 2, disables pagination, and allows 5 seconds for the
+VPN path. CAPTCHA/429-prone Google, Brave, and Startpage families are retained
+but disabled from default aggregation. This limits fan-out while preserving
+explicit/on-demand selection.
+
+Important: SearXNG's engines are configured in the **`engines:` list inside
+`settings.yml`**, not a separate `engines.yml` file. With
+`use_default_settings: true` (our setting) your `engines:` list is *merged* over
+the built-in engines by `name` — you do **not** set `use_default_settings: false`.
+
+To add, override, disable, or prune:
+
+```yaml
+# shared/searxng/settings.yml  (appended to the existing file)
+use_default_settings: true
+engines:
+  # prune a broken engine entirely
+  remove:
+    - # (example) wikipedia   # drop an engine that never recovers
+
+  # override + duplicate the Google engines so one survives the other's suspension
+  - name: google
+    weight: 3
+    disabled: false
+    timeout: 5.0                 # wait longer over the tunnel (global default is 2.0s)
+    retry_on_http_error: [403, 429]
+    display_error_messages: true
+
+  - name: google2                # fresh engine, same module, survives google's CAPTCHA ban
+    engine: google
+    categories: general
+    weight: 2
+    timeout: 5.0
+    disabled: false
+```
+
+Key per-engine keys (each overwrites the engine-module default; see
+`settings_engines.html`):
+
+| Key | Effect |
+|-----|--------|
+| `name` | merge key (lowercase, no underscore) |
+| `engine` | Python module (`google`, `json_engine`, `xpath`) |
+| `categories` | categories it serves (also a comma string) |
+| `timeout` | per-engine HTTP timeout, overrides `outgoing.request_timeout` |
+| `weight` | ranks results higher |
+| `disabled` | off by default, selectable in /preferences |
+| `inactive` | `disabled` **and** removed from the UI — use to fully drop a broken engine |
+| `retry_on_http_error` | retry on status, e.g. `true`, `403`, `[403, 429]` |
+| `display_error_messages` | show/hide the engine's error banner |
+| `proxies` | per-engine egress override (see Fix C) |
+| `language` | force region, e.g. `en` |
+
+**Best duplicate trick:** re-declare `google` under a new `name` in a
+*different* category. When the primary `google` gets CAPTCHA'd it suspends, but
+`google2` still returns results, and vice-versa.
+
+**Parser caveat (read before fiddling with `parsers`):** the modern `google`,
+`duckduckgo`, and `brave` engines ship a **custom `response()`** with hardcoded
+XPath and *ignore* the generic `parsers`/`p_parsers` table. So `parsers:
+[scrapy_html]` under `name: google` mostly does nothing. To make Google
+robust against selector drift, either update the upstream `google.py`
+selectors (upgrade SearXNG) or run a **`json_engine`** against a JSON feed.
+Verify `parsers` consumption on the pinned version before depending on it.
+
+**JSON-feed fallback:** decouple parsing from Google's HTML churn.
+
+```yaml
+engines:
+  - name: google-json
+    engine: json_engine
+    categories: general
+    paging: true
+    search_url: 'https://www.googleapis.com/customsearch/v1?key={api_key}&q={query}&cx={cx}'
+    results_query: results
+    title_query: title
+    url_query: link
+    content_query: snippet
+```
+
+## Fix B — Request tuning (settings.yml `outgoing:`)
+
+SearXNG already rotates a **random browser User-Agent per outgoing request**
+(`gen_useragent()` in `online.py`) — there is **no native `useragents:` mapping
+key**. The blogs that point at a `useragents:` table describe the old
+`asciimoo/searx`; current SearXNG rotates automatically. Keep relying on it.
+
+Tune the `outgoing` block and the suspend model to survive load:
+
+```yaml
+outgoing:
+  request_timeout: 4.0             # current Dankclaw value; modest VPN-latency allowance
+  max_request_timeout: 12.0
+  pool_connections: 100
+  pool_maxsize: 20
+  enable_http2: true
+  retries: 1                       # avoid amplifying 429/CAPTCHA responses from one exit IP
+  useragent_suffix: "ops@yourhomelab.example"   # so a per-engine blocker can reach you
+
+search:
+  ban_time_on_fail: 10
+  max_ban_time_on_fail: 180
+  suspended_times:
+    SearxEngineTooManyRequests: 300     # 429 → 5 min before it retries (tune down to recover faster)
+    SearxEngineCaptcha: 7200            # CAPTCHA → 2h before it retries
+```
+
+Notes:
+- **Auto-suspend is the real "rate limit".** The old per-engine `rate_limit` /
+  `time_between_requests` sleep field does **not** exist in current SearXNG —
+  that is a historical-searx concept. The functional equivalent is the suspend
+  schedule above (widen to back off before blocking, narrow to recover faster).
+- `request_timeout` interacts with the suspend model: too tight → engines look
+  "unresponsive"; too loose → slow pages. Dankclaw uses 4 seconds globally and
+  12 seconds as the cap; raise it only when measurements show real upstream
+  latency rather than 429/CAPTCHA responses.
+- Keep retries low: retrying a 429/CAPTCHA through the same VPN exit can make the
+  provider's rate limit worse. A retry count of 1 is a bounded compromise for a
+  transient network failure.
+- `safe_search: 0` when diagnosing — a too-strict global safe search can make an
+  engine return zero rows and *look* blocked.
+
+## Fix C — Routing & proxies (change which IP the upstreams exit as)
+
+Two independent levers: SearXNG's own upstream `proxies`, and the Gluetun VPN
+strategy. The decisive fact for our topology: a proxy that runs **inside the
+same Gluetun netns** exits from the *same* single IP, so round-robin among
+same-IP proxies **does not** unblock. The proxy must egress from a **different,
+clean IP**.
+
+```mermaid
+graph LR
+    SE[SearXNG] -->|default| GW[Gluetun #1 netns] -->|exit| IP1[ProtonVPN IP A — gets blocked]
+    SE -->|per-engine proxies| GW2[Gluetun #2 / dedicated IP] -->|exit| IP2[ProtonVPN IP B — clean]
+```
+
+**Option 1 — per-engine `proxies` (cheapest, settings.yml only).** Point the
+heaviest/blocked engines (Google and Google-hosted) at a proxy that exits from a
+clean IP:
+
+```yaml
+engines:
+  - name: google
+    proxies:
+      http:
+        - http://gluetun-upstream:8080
+      https:
+        - http://gluetun-upstream:8080
+        - socks5h://gluetun-upstream:8388
+  - name: google news
+    engine: google_news
+    proxies:
+      https:
+        - http://gluetun-upstream:8080
+```
+
+`using_tor_proxy` follows the same per-engine/global pattern.
+
+**Option 2 — dedicated second Gluetun for SearXNG's upstreams.** Run a second
+Gluetun on a different ProtonVPN server/geo, publish its built-in HTTP proxy
+(`8080`, `HTTPPROXY=on`), and point the Google engines at it. SearXNG inbound
+(still on the main host/Caddy) stays put; only *upstream* requests re-route.
+Gives a second isolated exit IP and reputation pool.
+- **Cons:** a second ProtenVPN device slot, a new container, port publishing
+  moves to Gluetun, and **split-brain DNS** (below).
+
+**Option 3 — dedicated ProtonVPN IP.** A static, exclusive exit IP that other
+subscribers can't pollute. Requires a **ProtonVPN Business plan** as of the
+sources reviewed (the long-requested *personal*-plan dedicated IP is still only
+on uservoice — flag: verify your plan). Wire it in Gluetun by pinning the node
+via `SERVER_HOSTNAMES`.
+
+**Option 4 — rotate the exit.** List `SERVER_COUNTRIES`/`SERVER_CITIES`/
+`SERVER_HOSTNAMES` so Gluetun picks from several physical exit servers. Caveat:
+rotating the **shared** Gluetun rotates the exit for *all* attached
+containers. To rotate only SearXNG's upstreams, use Option 1 → a dedicated
+gateway.
+
+> ⚠️ **Split-brain DNS:** a container sharing Gluetun's netns
+> (`Network=gluetun.container`) loses Docker's internal service-name DNS — it
+> can't resolve siblings by name. Use a static IP or `extra_hosts`/`~/.config/
+> containers/systemd/<unit>.container` `Environment` to inject names into the
+> tunnel's `/etc/hosts`.
+
+## Observed Dankclaw baseline (2026-08-25)
+
+The read-only audit found:
+
+- SearXNG image: `ghcr.io/searxng/searxng:latest`; embedded version `1.0.0`.
+- Gluetun uses `VPN_SERVICE_PROVIDER=custom` with a static WireGuard profile;
+  no `SERVER_COUNTRIES`, `SERVER_CITIES`, `SERVER_REGIONS`, or
+  `SERVER_HOSTNAMES` filters are configured.
+- The observed exit was `149.22.88.19` (Seattle, Washington, US).
+- Bing and DuckDuckGo returned results. Mojeek returned no results without an
+  upstream error.
+- `google cse` and Brave were suspended for too many requests; Startpage was
+  suspended for CAPTCHA; Qwant returned CAPTCHA. SearXNG logs also recorded
+  Google's "unusual traffic from your network" response.
+- `/healthz` returned `200 OK`; the SearXNG service and Gluetun service were
+  healthy. This isolates the primary failures to upstream provider handling,
+  not application availability.
+
+This deployment returns the legacy array form for `unresponsive_engines`:
+
+```json
+"unresponsive_engines": [
+  ["brave", "Suspended: too many requests"],
+  ["startpage", "Suspended: CAPTCHA"]
+]
+```
+
+Do not assume the newer object form (`engine`, `error_type`, `suspended`)
+until the live response has been checked. The direct published port is the
+useful diagnostic path because it exercises the ProtonVPN exit. The Caddy
+hostname resolved to the WAN address during the audit and timed out from the
+audit host, so it was not used as the baseline path.
+
+## Fix D — Verify an engine is unblocked
+
+Three layers, cheapest first. All queries should hit the path production uses
+(the **published Gluetun port → ProtonVPN exit**, not a direct host port).
+
+**1. Isolate one engine with `/json`.** Enable `json` in `search.formats`
+(otherwise `format=json` → 403), then query a single engine:
+
+```bash
+BASE="http://192.168.0.143:8081"   # published port (ProtonVPN exit)
+curl -s "$BASE/search?q=test&engines=google&format=json" \
+  | jq -r '.unresponsive_engines[]? | select(.engine=="google")'
+```
+
+Inspect:
+- `content[].engine` — which engine produced each result.
+- `unresponsive_engines[]` — inspect the live shape first. On Dankclaw it is
+  `[engine, message]`, for example `["google cse", "Suspended: too many requests"]`.
+  Newer builds may return objects containing `engine`, `error_type`, and
+  `suspended`.
+- `timings[].engine` — per-engine latency when the running build emits timings.
+
+Full status check (bash + jq):
+
+```bash
+BASE="http://192.168.0.143:8081"
+for e in google bing duckduckgo brave startpage; do
+  curl -s "$BASE/search?q=test&engines=$e&format=json" \
+    | jq -r --arg e "$e" '
+        (.unresponsive_engines // [])
+        | map(select(.engine==$e))
+        | if length==0 then "\($e): OK"
+          else .[] | "\($e): UNRESPONSIVE (\(.error_type), suspended=\(.suspended))" end'
+done
+```
+
+**2. `/config` REST endpoint** lists every engine + enabled/disabled +
+categories — an inventory of what is live. `?engines=<name>` isolates a single
+provider; `/debug` adds engine diagnostics.
+
+**3. `check.searx.space`** — the hosted official checker probes every engine of
+your instance with a **headless Firefox** (catches Cloudflare/CAPTCHA blocks a
+plain HTTP probe misses) and posts results on searx.space. Add your instance via
+a GitHub issue in `searxng/searx-instances`, then allow `check.searx.space` IPs
+to reach the SearXNG port.
+
+Local self-test CLI (`kvch/searx-checker`):
+
+```bash
+python3 checker/checker.py http://192.168.0.143:8081
+# google.OK  bing.OK  gitlab......ERROR
+```
+
+Verify through the **published Gluetun port** (realistic ProtonVPN-exit block
+signal), not a direct host port (which uses the host LAN IP and can be
+misleading).
+
+## Recommended path for Dankclaw (ranked)
+
+1. **Measure individual engines** — use `/json` + `?engines=<name>` and several
+   query classes before enabling a bundled engine in default aggregation.
+2. **Keep default fan-out small** — prefer independent anonymous providers and
+   disable pagination for rate-sensitive engines.
+3. **Promote by evidence** — raise `weight` only for engines that return useful
+   results consistently; weight is not a health check.
+4. **Avoid credentials** — do not add API-backed engines unless the provider is
+   genuinely free, anonymous, and compatible with our privacy requirements.
+5. **Verify first** — `/json` + `?engines=google` and the searx-checker to see
+   exactly which engines are down and why. Confirm the running SearXNG version
+   (check whether `parsers`/`rate_limit` exist in the pinned build).
+6. **Use conservative request tuning** — Dankclaw now allows 5 seconds per
+   upstream request, caps it at 15 seconds, and retries once. This addresses
+   modest VPN latency without multiplying 429/CAPTCHA traffic. Adjust only
+   after measuring the resulting error and latency profile.
+7. **If Google still re-blocks** — Fix C Option 1: point the Google engines at
+   a dedicated-IP proxy (Option 2 dedicated second Gluetun if reputation
+   isolation is needed). Avoid the same-netns-proxy trap.
+
+---
+
+## Sources
+
+- [SearXNG settings: engines](https://docs.searxng.org/admin/settings/settings_engines.html) — per-engine keys.
+- [SearXNG settings: outgoing](https://docs.searxng.org/admin/settings/settings_outgoing.html) — `request_timeout`, pool, `retries`, `proxies`, `source_ips`.
+- [SearXNG settings: search](https://docs.searxng.org/admin/settings/settings_search.html) — `suspended_times`, `ban_time_on_fail`.
+- [SearXNG settings (merge + remove/keep_only)](https://docs.searxng.org/admin/settings/settings.html) — `use_default_settings` semantics.
+- [Google engine docs](https://docs.searxng.org/dev/engines/online/google.html) — `detect_google_sorry`, WML/Nokia XML, `max_page`.
+- [json_engine](https://docs.searxng.org/dev/engines/json_engine.html) — `json` parser config.
+- [SearXNG exceptions](https://docs.searxng.org/src/searx.exceptions.html) — exception types and suspend settings.
+- [Google engine stops yielding results #5286](https://github.com/searxng/searxng/blob/master/searx/engines/google.py) — reproducible "Google unavailable".
+- [DuckDuckGo CAPTCHA #4824](https://github.com/searxng/searxng/issues/4824) — per-engine CAPTCHA signature.
+- [Google engine actively blocking IPs #2515](https://github.com/searxng/searxng/issues/2515)
+- [answer-captcha from server's IP](https://docs.searxng.org/admin/answer-captcha.html) — block is tied to the egress IP.
+- [SearXNG rate limits / 429 (SSD Nodes)](https://www.ssdnodes.com/learn/searxng-rate-limits-and-429-errors) — suspension table, `/stats/errors`.
+- [How search engines detect scrapers (Crawlbase)](https://crawlbase.com/blog/how-search-engines-detect-scrapers/) — IP reputation + rate + CAPTCHA mechanics.
+- [check.searx.space](https://github.com/searxng/searx-space) + [kvch/searx-checker](https://github.com/kvch/searx-checker) — the checkers.
+- [searx-instances — how to be checked](https://github.com/searxng/searx-instances) — add your instance + allow checker IPs.
+- [ProtonVPN — Dedicated IP addresses](https://proton.me/support/dedicated-ips) + [Benefits](https://proton.me/support/dedicated-ips) — IP-reputation reasoning; **Business plan** (flag: personal dedicated IP still on uservoice).
+- [Gluetun wiki — ProtonVPN provider](https://github.com/qdm12/gluetun-wiki/blob/main/setup/providers/protonvpn.md) — `SERVER_HOSTNAMES` narrowest-filter risk.
+- [Gluetun wiki — Built-in Proxies](https://deepwiki.com/qdm12/gluetun-wiki/4.5-built-in-proxies) — HTTP `8080` + Shadowsocks `8388`.
+- [Gluetun wiki — Container networking](https://deepwiki.com/qdm12/gluetun-wiki/6.5-container-networking) — netns sharing, port publishing on Gluetun.
+- [Gluetun inter-network DNS failures (Jacob Jangles)](https://jacobjangles.com/posts/fixing-gluetuns-inter-network-dns-failures-in-docker/) — split-brain DNS + fix.
+
+## See Also
+
+- [[SearXNG]] — the stack, routing, limiter, health
+- [[Gluetun]] — the VPN tunnel (where the exit IP comes from)
+- [[Dankclaw Stack]] — the service the SearXNG stack sits in
+- [[Service Research Brief]] — the research that produced these pages

docs/Dankclaw/SearXNG.md

@@ -61,9 +61,29 @@ valkey:
   url: "valkey://searxng-valkey:6379/0"
 
 outgoing:
-  request_timeout: 3.0
+  request_timeout: 5.0
-  max_request_timeout: 10.0
+  max_request_timeout: 15.0
-  retries: 3
+  retries: 1                     # avoid amplifying upstream 429/CAPTCHA responses
+
+# Anonymous, bounded default aggregation. Disabled engines remain configured
+# for explicit/on-demand selection; see SearXNG Providers for the policy.
+engines:
+  - name: bing
+    disabled: false
+    weight: 2
+    paging: false
+    timeout: 5.0
+  - name: duckduckgo
+    disabled: false
+    weight: 2
+    paging: false
+    timeout: 5.0
+  - name: google cse
+    disabled: true
+  - name: brave
+    disabled: true
+  - name: startpage
+    disabled: true
 ```
 
 ## Volumes
@@ -94,7 +114,28 @@ The rate limiter's JSON cap is `API_MAX = 4` requests per IP per hour. The const
 
 Reach SearXNG through Caddy (`search.dankclaw.local`) for browser use. A local subagent can hit the published port directly because its slirp source is now trusted. The searxng subnet 10.89.2.0/24 is in `trusted_proxies`, so Caddy's `X-Forwarded-For` is honored. The subnet is pinned in `searxng.network`, so it stays stable across reboots.
 
-## Gotchas
+## Unblocking Google & Search Providers
+
+The current request policy allows a modest amount of VPN latency (`4.0s` per
+request, `12.0s` maximum) while limiting retries to one. This can recover slow
+providers, but it cannot fix an upstream CAPTCHA or 429 caused by the shared
+VPN exit IP. SearXNG queries all ~50 upstream engines from one Gluetun VPN exit
+IP. The rate-sensitive engines (Google, DuckDuckGo) then answer with reCAPTCHA / 429.
+The default provider policy therefore favors anonymous Bing and DuckDuckGo,
+disables observed CAPTCHA/429 providers from normal aggregation, and disables
+pagination to reduce upstream request volume. SearXNG **suspends** offending
+engines for hours to days, which is what shows up as "Google went unavailable".
+This is upstream, not local.
+
+Fixes (cheapest first): tune the `outgoing` block (`request_timeout`,
+`retry_on_http_error`, `suspended_times`), duplicate the `google` engine under a
+fresh `name` so a CAPTCHA ban on one leaves the other returning results, and —
+if Google still re-blocks — route the Google engines at a clean-IP proxy via
+per-engine `proxies` in `settings.yml`. See [[SearXNG Providers]] for the full
+root cause, the suspend-model table, the `engines:` config, and the
+`/json` verification recipe.
+
+### Gotchas
 
 - The JSON API rate cap (4 per 3600s per IP) is hardcoded in Python, not settable in `limiter.toml`. The only bypass is `pass_ip`. Local subagents are covered because their slirp/loopback/LAN sources are in `pass_ip`.
 - The limiter stays armed for untrusted sources (a spoofed public IP still 429s). Do not disable it. Widen only `pass_ip` for sources you trust on this host.
@@ -106,4 +147,5 @@ Reach SearXNG through Caddy (`search.dankclaw.local`) for browser use. A local s
 
 - [[Dankclaw Stack]]
 - [[Gluetun]] — the VPN container
-- Knowledge base: [[Gluetun]], [[SearXNG]], [[Hermes]]
+- [[SearXNG Providers]] — unblocking Google & other upstream engines (VPN exit IP, engine config, proxies, verification)
+- Knowledge base: [[Gluetun]], [[SearXNG]], [[SearXNG Providers]], [[Hermes]]

docs/Dankserver/Entropy Volume.md

@@ -0,0 +1,43 @@
+# Entropy Volume
+
+The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at
+`/mnt/entropy`. The command-line helper reuses the host's desktop-integrated
+`udisks2`, `crypttab`, and `fstab` configuration; it does not store or handle
+the passphrase itself.
+
+## Command-line mount
+
+```bash
+/home/dank/serverAppConfigs/scripts/mount-entropy.sh
+```
+
+Run it as the normal desktop user (`dank`), not from a root shell. `udisksctl`
+will securely prompt for the LUKS passphrase. `sudo mount /mnt/entropy` then
+uses the existing `/etc/fstab` entry, preserving its options:
+
+```text
+nosuid,nodev
+```
+
+The existing mountpoint is owned by `dank:dankserver` with mode `0755`; the
+helper deliberately does not run `chmod` or `chown`, so it cannot silently
+change those permissions.
+
+## Device path
+
+The current storage chain is:
+
+```text
+/dev/md0 -> LUKS UUID cd86c728-459e-4f3a-bcc2-41ecf84685ec
+         -> ext4 label entropy
+         -> /mnt/entropy
+```
+
+The helper checks that `/dev/md0` exists and exits successfully if the volume
+is already mounted. It is intentionally specific to this host and volume:
+that avoids accidentally unlocking or mounting a different disk.
+
+## Related
+
+- [[Storage Policy]]
+- [[Dankserver Stack]]

docs/Infrastructure/Caddy.md

@@ -1,6 +1,6 @@
 # Caddy
 
-Caddy is the single reverse proxy for both hosts. It runs as a container on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth.
+Caddy is the single reverse proxy for both hosts. It runs as a boot-persistent quadlet container (`caddy.service`) on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth.
 
 **Docs:** https://caddyserver.com/docs
 
@@ -8,6 +8,7 @@ Caddy is the single reverse proxy for both hosts. It runs as a container on Dank
 
 | Item | Value |
 |------|-------|
+| Runtime | Quadlet unit `caddy.service` (`dankclaw/caddy/quadlet/caddy.container`) — boot-persistent, self-healing |
 | Image | `franksdank/caddy-porkbun:latest` (local build, `dankclaw/caddy/Containerfile`) |
 | Config | `/home/dank/Containers/caddy/Caddyfile` (bind mount, rendered by deploy) |
 | Data | `/home/dank/Containers/caddy/data` (certs) and `.../config` (ACME state) |
@@ -15,7 +16,19 @@ Caddy is the single reverse proxy for both hosts. It runs as a container on Dank
 | TLS | Let's Encrypt, DNS-01 via the Porkbun module (one cert per vhost) |
 | Runs on | Dankclaw only — Dankserver has no Caddy |
 
-The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before `up`. See [[Deployment Flow]].
+The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before start. See [[Deployment Flow]].
+
+## Quadlet (boot persistence)
+
+Caddy left `compose.yaml` on 2026-08-26. A reboot exposed that the bare compose container never comes back: rootless Podman has no boot-time auto-start for bare containers, and `restart: unless-stopped` does not survive the explicit stop that shutdown performs. The quadlet unit is now the only way Caddy starts (see AGENTS.md "Boot Persistence & Self-Healing").
+
+- **Install path on Dankclaw:** `~/.config/containers/systemd/caddy.container` (unit `caddy.service`)
+- **Network:** `dankclaw_default` (the compose bridge — the Caddyfile's container-name upstreams must keep resolving; if no compose service has created the network yet at boot, quadlet creates it first and compose reuses it by name later)
+- **Published ports:** 80/tcp, 443/tcp
+- **Env file:** `/home/dank/Containers/caddy/env` (`PORKBUN_API_KEY`, `PORKBUN_API_SECRET_KEY`) — the deploy writes it from the checkout's `.env`; secrets never enter the repo (placeholders in `dankclaw/caddy/env.example`)
+- **Restart:** `Restart=always`, `RestartSec=5`; the health check probes the bare-IP catch-all (302) inside the container
+- **Image policy:** `Pull=never` — the image is built on-host and exists nowhere else; a boot must not stall on a registry
+- **Deploy:** `./scripts/deploy.sh dankclaw --service caddy` — renders the Caddyfile, rebuilds the image if the Containerfile changed, retires the legacy split proxy, writes the env file, removes the old bare container (all state is in host volumes), then installs, enables, and starts the unit
 
 ## The porkbun image
 
@@ -34,7 +47,7 @@ acme_dns porkbun {
 }
 ```
 
-compose passes both variables from the host `.env`. Keep the real keys in `.env` only; `.env.example` holds placeholders. The Porkbun account holds only `franksdank.cloud`.
+The deploy writes both variables from the host `.env` into `/home/dank/Containers/caddy/env`, which the quadlet reads via `EnvironmentFile`. Keep the real keys in `.env` only; `.env.example` (and `dankclaw/caddy/env.example`) hold placeholders. The Porkbun account holds only `franksdank.cloud`.
 
 ## Vhost map
 
@@ -46,7 +59,7 @@ Every vhost uses the flat `franksdank.cloud` convention. The full list lives in
 | Dankclaw quadlet services | by published LAN port, e.g. `192.168.0.143:3000` (Gitea) |
 | Dankserver services | by published LAN port, e.g. `192.168.0.235:8085` (RomM) |
 
-The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/`. The legacy name `dankserver.franksdank.cloud` 302-redirects there too. `plex.franksdank.casa` is the only plain-HTTP vhost; it cannot get a DNS-01 cert because `franksdank.casa` is not in the Porkbun account. The canonical Plex name is `plex.franksdank.cloud`.
+The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/`. The legacy name `dankserver.franksdank.cloud` 302-redirects there too. `plex.franksdank.casa` is the only plain-HTTP vhost; it cannot get a DNS-01 cert because `franksdank.casa` is not in the Porkbun account. The canonical Plex name is `plex.franksdank.cloud`. Both Plex vhosts dial PMS **over TLS upstream** (`transport http { tls_insecure_skip_verify }` — in this Caddy build the flag is a bare token inside the transport block, not a `tls {}` sub-block): PMS 1.43.x silently closes plain-HTTP connections from non-loopback sources (diagnosed 2026-08-26 — it accepts, reads the request, and FINs without responding), which surfaced here as an empty-body 502 (blank page in browsers). `insecure_skip_verify` applies only to this pinned-LAN hop (PMS serves a self-signed cert); client-facing Let's Encrypt TLS is unchanged. The Plex web UI is at `/web/`; `/` returns the XML `MediaContainer` index.
 
 ## Basic auth
 
@@ -83,6 +96,7 @@ Certificates and ACME state live in the caddy data volume. Back it up with the s
 
 ## Gotchas
 
+- A bare compose container is **not** boot-persistent: after the 2026-08-26 reboot Caddy stayed down for hours because `restart: unless-stopped` does not survive shutdown's explicit stop. It now runs only from its quadlet unit — do not re-add it to `compose.yaml`.
 - `admin off` disables the config file watcher: rendered changes then sit on disk while Caddy serves its stale in-memory config until the container is recreated, and nothing logs an error (2026-08-22: a broken basicauth hash survived several deploys this way). `tests/caddyfile.bats` forbids `admin off`.
 - The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged.
 - A new vhost needs three things: a Pi-hole `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]].

docs/Infrastructure/LAN-DNS.md

@@ -42,7 +42,7 @@ The G36 has no NAT loopback (hairpin). A LAN client that resolves a name to the
 
 ## Pi-hole quadlet
 
-Pi-hole FTL is a dnsmasq fork. It serves the same `addn-hosts` file the retired dnsmasq quadlet used, mounted into `/etc/dnsmasq.d/` read-only. It retired the dnsmasq quadlet on 2026-08-24.
+Pi-hole FTL is a dnsmasq fork. It serves the same `addn-hosts` file the retired dnsmasq quadlet used, mounted into `/etc/pihole/hosts/` read-only. It retired the dnsmasq quadlet on 2026-08-24.
 
 - **Location:** `dankclaw/pihole/` — `quadlet/pihole.container`, `env.example`, `addn-hosts`
 - **Ports:** 53/udp + 53/tcp (DNS) and 8083/tcp (web UI) on 192.168.0.143
@@ -91,6 +91,34 @@ The Dankclaw host itself resolves the flat names through Pi-hole:
 
 Without this, the host would resolve the names through the router to the WAN IP and time out (no hairpin).
 
+## Dankware host DNS
+
+`dankware` is a LAN client, not a Pi-hole server. Its flat names resolve through Pi-hole (`192.168.0.143`), but its resolver setup differs from Dankclaw's because it carries two `+DefaultRoute` links.
+
+- **Two default links:** `enp47s0` (ethernet, metric 100) and `wlan0` (wifi, metric 600), each getting the same DHCP DNS list `[192.168.0.143, 1.1.1.1, …]`.
+- **The flap:** `systemd-resolved` picks a different "current" resolver per link — `enp47s0` settled on the Comcast v6 upstream (→ WAN IP, times out on the LAN), `wlan0` on Pi-hole (→ LAN IP). Flat-name resolution flapped between the two on every query.
+- **The fix:** pin `192.168.0.143` as the static IPv4 resolver on both NetworkManager connections. `resolved.conf`'s own `DNS=` is a per-link fallback, so the link itself must be pinned:
+
+```bash
+for c in "Wired connection 1" FRANKFI-5G; do
+  nmcli connection modify "$c" ipv4.method auto \
+    ipv4.dns 192.168.0.143 \
+    ipv4.ignore-auto-dns yes \
+    ipv4.ignore-auto-routes no
+done
+nmcli connection up "Wired connection 1"
+nmcli connection up FRANKFI-5G
+systemctl restart systemd-resolved
+```
+
+`method:auto` keeps DHCP for the address/gateway/route (dual-default traffic path preserved); `ignore-auto-dns=yes` uses only the static resolver; `ignore-auto-routes=no` keeps the DHCP routes. Verify with a cache-clear loop — `romm.franksdank.cloud` should return `192.168.0.143` on every run:
+
+```bash
+for i in $(seq 1 10); do resolvectl flush-caches; dig romm.franksdank.cloud +short; done
+```
+
+The apex and unknown subdomains still fall through to Porkbun (→ `71.227.227.172`), so WAN clients and a bare `dig franksdank.cloud` keep working.
+
 ## WAN records (Porkbun)
 
 The Porkbun zone `franksdank.cloud` carries an A record for every vhost plus the apex, all pointing at `71.227.227.172`. TTL is 300. The zone has no wildcard record: unknown subdomains return NXDOMAIN. Caddy's DNS-01 challenges add and remove `_acme-challenge.*` TXT records through the same API during issuance. See [[Caddy]] for the TLS flow.
@@ -103,6 +131,7 @@ The Porkbun zone `franksdank.cloud` carries an A record for every vhost plus the
 |---------|-------------|
 | Name does not resolve on a new client | `resolv.conf` shows 192.168.0.1, not .143 — the G36 hand-off step is missing |
 | Name resolves to 71.227.x and times out on the LAN | Client used the router DNS — same root cause |
+| Flat name flaps between 192.168.0.143 and 71.227.x on one client | Two `+DefaultRoute` links; resolved picks a different "current" resolver per link. Pin 192.168.0.143 as static DNS on both NM connections — see [[#Dankware host DNS]] |
 | `dig @192.168.0.143` fails entirely | Pi-hole quadlet down: `systemctl --user status pihole` |
 | Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`). FTL serves it. Use the FQDN, or an `/etc/hosts` entry on the client |
 | HTTPS works from the host but not the WAN | No port forward on the G36 for that service (manual router step) |

docs/Infrastructure/Pi-hole.md

@@ -9,6 +9,7 @@ Pi-hole is the LAN DNS server for the homelab. It runs as a quadlet container on
 | `pihole` quadlet | `dankclaw/pihole/quadlet/pihole.container` | Boot-persistent container unit |
 | FTL | `docker.io/pihole/pihole:latest` | DNS engine (a dnsmasq fork) + ad blocking |
 | `addn-hosts` | `dankclaw/pihole/addn-hosts` | One A record per Caddy vhost |
+| sysctl drop-in | `dankclaw/pihole/sysctl/99-dnsmasq.conf` (repo) → `/etc/sysctl.d/` | Rootless port-53 threshold, applied at boot |
 | env file | `/home/dank/Containers/pihole/env` (host) | `ServerIP`, `TZ`, `DNS1`, `DNS2`, `WEBPASSWORD` |
 | Web UI | `pihole.franksdank.cloud` (Caddy → 8083) | Dashboard + admin, family login |
 
@@ -49,7 +50,7 @@ The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `D
 
 ## Host prerequisites
 
-1. `sysctl net.ipv4.ip_unprivileged_port_start=53` (persisted in `/etc/sysctl.d/99-dnsmasq.conf`) — rootless Podman may then publish port 53.
+1. `net.ipv4.ip_unprivileged_port_start=53`, persisted in `/etc/sysctl.d/99-dnsmasq.conf` — rootless Podman may then publish port 53. The repo file `dankclaw/pihole/sysctl/99-dnsmasq.conf` is the single source of truth: `deploy_pihole` installs it, stages any later-sorted drop-in that resets the value in the home trash, and applies it with `sysctl --system` (the same code path boot uses via systemd-sysctl). **Incident (2026-08-26):** a second drop-in (`99-rootless-ports.conf`, sorted after) reset the value to 80. The runtime override from an earlier manual `sysctl -w` died with a reboot, and `pihole.service` crash-looped on `bind: permission denied` for port 53 (restart counter 1645) until the conflict was removed.
 2. `DNSStubListener=no` in `/etc/systemd/resolved.conf` — the resolved stub would otherwise hold port 53 inside the container netns.
 
 Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
@@ -62,6 +63,7 @@ Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
 | A name does not resolve for a new client | The client's `resolv.conf` does not show `192.168.0.143` — the G36 hand-off is missing. See [[LAN-DNS]] |
 | The web UI asks for a password loop | `WEBPASSWORD` in the env file is the placeholder `changeme` — fill the real value, restart the unit |
 | Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong |
+| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: exactly one file under `/etc/sysctl.d/` may set `ip_unprivileged_port_start`, and it must say 53. Redeploy pihole — `deploy_pihole` reinstalls the drop-in and neutralizes conflicts |
 | Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
 
 ## See Also

docs/LLM/Dankclaw Improvements.md

@@ -21,20 +21,24 @@ marked "verify first". Re-verified 2026-08-15/16; corrections marked below.
 - **KV cache**: production runs `q8_0` K **and** `q8_0` V (both quantized).
 - **Spec**: DFlash/DSpark are retired. Production is `--spec-default`
   (ngram-mod) with draft-mtp first. See [[qwen3.8-spec-matrix]].
-- **TTM**: still 15.24 GiB as of 2026-08-16 — the modprobe fix is still
+- **TTM**: raised to 114 GiB on 2026-08-25, set live (no reboot) and
-  pending a reboot.
+  persisted in `/etc/modprobe.d/ttm.conf`. The effective VRAM ceiling that
+  once held the service at ~15 GiB is gone.
 
 ## Verified Memory Picture (measured 2026-08-11)
 
 | Figure | Value | Source |
 | ------ | ----- | ------ |
 | OS physical RAM | 30.5 GiB | `/proc/meminfo` |
-| TTM pages_limit | 15.24 GiB (3,995,281 x 4KiB) | `/sys/module/ttm/parameters/pages_limit` |
+| TTM pages_limit | 114 GiB (29,883,536 x 4KiB) — raised 2026-08-25 | `/sys/module/ttm/parameters/pages_limit` |
 | ROCm VRAM total | 96.0 GiB | `rocm-smi --showmeminfo vram` |
 | ROCm VRAM used | 76.2 GiB | `rocm-smi` while serving 3x200700 |
 
-The OS accounts only ~30.5 GiB physical RAM, and the TTM GPU-addressable pool
+The OS accounts only ~30.5 GiB physical RAM. The TTM GPU-addressable pool
-is capped at ~15.24 GiB. ROCm/SMI still reports a 96 GiB VRAM carve-out (VGM).
+was capped at ~15.24 GiB (the real pre-2026-08-25 ceiling); that cap is now
+lifted to 114 GiB. ROCm/SMI still reports a 96 GiB VRAM carve-out (VGM) — a
+different, driver-reported figure that has always been larger than the kernel
+cap.
 This split is the Strix Halo GTT/VGM design. It drives the batch/ubatch memory
 limits documented in [[frank-default-large-benchmark]].
 
@@ -77,7 +81,7 @@ All require root and a reboot for the modprobe change.
 
 | Setting | Action | Effect | Priority |
 | ------- | ------ | ------ | -------- |
-| TTM GPU pool | Set `amd-ttm --set 96` (write `/etc/modprobe.d/ttm.conf`), reboot | Lifts the 15 GiB TTM cap toward full RAM; more headroom for GPU buffers | High |
+| TTM GPU pool | `/etc/modprobe.d/ttm.conf` → `options ttm pages_limit=29883536 page_pool_size=29883536` (applied 2026-08-25, set live then persisted) | Lifts the 15 GiB cap to 114 GiB; the full 262k KV + Q8 AGENT payload fit in one slot | Done |
 | Power profile | **N/A on this machine** (2026-08-15): no `power_profile` sysfs; GPU DPM already `performance` | none | ~~Medium~~ |
 | NVMe scheduler | Already `none` (`[none]` confirmed) | No change | Done |
 | BIOS VGM | Set Variable Graphics Memory / UMA to "High" in the EVO-X2 BIOS | Max dedicated VRAM for the GPU carve-out; verify in the live BIOS | Verify |

docs/LLM/ciru-activefpx-promptforge.md

@@ -253,7 +253,7 @@ ngram draft costs its own forward pass / occupancy without adding enough
 accepted tokens on top of MTP. Baseline (no ngram) remains the production
 config (`--spec-type draft-mtp` only). Keep ngram-mod off.
 
-The harness lives at [scripts/bench-spec-stack.sh](../../scripts/bench-spec-stack.sh);
+The historical harness was moved to `.AGENTTRASH/bench-scripts-20260901/` and is retained for audit only.
 it boots the v3 stack on a scratch port and reads authoritative
 `print_timing` lines. It requires the production service to be stopped.
 

docs/LLM/deployment.md

@@ -1,229 +1,124 @@
+---
+type: guide
+title: LLM Deployment
+description: Scoped deployment and update process for the two inference services.
+tags:
+  - llm
+  - deployment
+  - systemd
+timestamp: 2026-09-03
+---
+
 # Deployment and Update Process
 
-This page describes how to deploy and update the LLM servers on Dankserver
+This page describes how to deploy and update the LLM servers on Dankserver and
-and Dankclaw. The process uses git as the single source of truth. All script
+Dankclaw. The process uses Git as the single source of truth. A change to
-edits flow through this repository.
+SearXNG, Caddy, or another container service does **not** restart inference.
 
-> 2026-08-22: the former `vllmScripts` repository is merged into this repo.
+## Fleet
-> Both LLM services now run from the `/home/dank/serverAppConfigs` checkout
-> on each host. The retired checkouts (`/home/dank/vLLM` on Dankserver,
-> `/home/dank/vllmScripts` on Dankclaw) are archived on the hosts.
 
-## Overview
+| Target | Host | Service | Port |
+|---|---|---|---:|
+| Dankserver | `dank@dankserver` | `frank-default-small` | 7770 |
+| Dankclaw | `dank@dankclaw` | `frank-default-large` | 7770 |
 
-Two servers run LLM inference services. Each server pulls its scripts from
+## Scoped workflow
-this repository. The deployment process moves a committed change from the
-repository to a running service.
-
-| Target | Host | Runtime | Service | Port |
-|--------|------|---------|---------|------|
-| Dankserver | dank@dankserver | upstream llama.cpp (Ornith-1.5-35B-A3B-CRACK Q6_K, native MTP draft, selected MoE experts offloaded via --override-tensor) | frank-default-small | 7770 |
-| Dankclaw | dank@dankclaw | CIRU ROCmFPX llama.cpp (Qwen3.8-27B-Uncensored-ROCmFP4 Q6 AGENT, standalone MTP Q4_0 head, 262K ctx / 1 slot) | frank-default-large | 7770 |
-
-## Workflow
-
-The deployment workflow has seven steps.
 
 ```mermaid
 graph LR
-    A[Edit scripts locally] --> B[Commit]
+    A[Edit LLM files] --> B[Run tests]
-    B --> C[Push to origin<br>Gitea on Dankclaw]
+    B --> C[Commit main]
-    C --> D[Pull on target<br>git pull]
+    C --> D[deploy.sh selects matching service]
-    D --> E[Restart podman stack]
+    D --> E[Target pulls ff-only]
-    E --> F[Relink + restart LLM unit]
+    E --> F[Relink one unit]
-    F --> G[Verify health endpoints]
+    F --> G[Restart one inference service]
+    G --> H[Wait for that service health]
 ```
 
-1. Fetch the current script from the target machine.
+Run the unified deployer from the repository root:
-2. Commit the fetched version in this repository.
-3. Edit the script locally in the working tree.
-4. Run the tests (`./tests/run.sh` must pass).
-5. Commit and push the change to origin.
-6. Pull the change on the target machine.
-7. Restart the services and verify that they are healthy.
 
-## Deploy Script
+```bash
-
+./scripts/deploy.sh dankserver --service frank-default-small
-The `scripts/deploy.sh` script is the unified deployer. One run per target
+./scripts/deploy.sh dankclaw --service frank-default-large
-covers both workloads on that host: the podman stack and the LLM service.
+./scripts/deploy.sh dankclaw --dry-run
-Run it from the repository root.
-
-Run this command to deploy to Dankserver:
-
-```
-./scripts/deploy.sh dankserver
 ```
 
-Run this command to deploy to Dankclaw:
+Without `--service`, `scripts/deploy-plan.py` maps changed files to the service
+on the selected host. The following files select the corresponding service:
 
-```
+- `scripts/frank-default-small.env`, `frank-default-small.service`, and
-./scripts/deploy.sh dankclaw
+  `serve-frank-small.sh` select `frank-default-small` on Dankserver.
+- `scripts/frank-default-large.env`, `frank-default-large.service`, and
+  `serve-frank-large.sh` select `frank-default-large` on Dankclaw.
+- `scripts/chat-template.jinja` selects the LLM on the selected host only.
+- Documentation, tests, deployment tooling, and unrelated container changes
+  do not select either inference service.
+
+The target-side `scripts/deploy-remote.sh` relinks the selected unit into
+`/etc/systemd/system/`, reloads systemd, enables it, restarts it, and waits for
+`http://127.0.0.1:7770/health`. The wait budget is 20 minutes because cold model
+loads can be slow. A failed handler does not advance its per-service state.
+
+## Unit and artifact paths
+
+Both units point into the stable target checkout:
+
+```text
+/home/dank/serverAppConfigs/scripts/frank-default-small.service
+/home/dank/serverAppConfigs/scripts/frank-default-large.service
 ```
 
-Add the `--dry-run` flag to preview without making changes:
+The unit files and serve scripts validate their model/runtime artifacts before
+starting. Keep `REMOTE_DIR` stable; moving the checkout requires updating both
+the unit absolute paths and the deployment tests in the same change.
 
-```
+## NPU unit
-./scripts/deploy.sh dankserver --dry-run
+
+`frank-default-npu` is an inactive, historical Dankclaw endpoint. It is
+relinked only when explicitly selected with `--service frank-default-npu`; the
+large-model handler never restarts it. This is intentional: the NPU is not part
+of the active inference path and should not be disturbed by a large-model
+change.
+
+## Isolation guarantee
+
+The remote runner has one explicit service dispatch. It does not run a
+host-wide LLM loop, `podman-compose up` for unrelated services, or a blanket
+systemd restart. Each successful service is recorded under the target user's
+state directory outside Git:
+
+```text
+~/.local/state/serverAppConfigs/deploy-state/
 ```
 
-The script does these actions:
+On a later ordinary deployment, a service remains pending only while its
+recorded revision differs from the checkout and the planner still finds a
+change in that service's inputs.
 
-1. Verifies that the local working tree has no uncommitted changes.
+## Manual recovery
-2. Pushes the `main` branch to origin.
-3. Pulls the repository on the target machine.
-4. Restarts the podman stack (Dankclaw only. The Dankserver media stack
-   runs as bare containers).
-5. Relinks the LLM systemd units, reloads systemd, enables, and restarts
-   the active LLM service.
-6. Waits for the Caddy homepage and the LLM health endpoint (over ssh).
 
-Set `FRANK_SUDO_PW` when running headless on a host without passwordless
+If the LLM handler fails after the target pull:
-sudo, so that remote sudo reads the password from stdin (`sudo -S`). The
-password is never stored in this repo. Both fleet hosts currently allow
-passwordless sudo for the deploy user (verified 2026-08-22). The
-passwordless restart helpers from the retired topology are obsolete and are
-not used.
 
-Health endpoints:
+1. Inspect `journalctl -u frank-default-small -n 200` or
+   `journalctl -u frank-default-large -n 200`.
+2. Confirm the model files, custom runtime binary, and chat template are
+   readable by `dank`.
+3. Retry only the affected service:
 
-- Dankserver: `http://127.0.0.1:7770/health`
+   ```bash
-- Dankclaw: `http://127.0.0.1:7770/health`
+   ./scripts/deploy.sh dankserver --service frank-default-small
+   # or
+   ./scripts/deploy.sh dankclaw --service frank-default-large
+   ```
 
-## Manual Update
+4. Verify the matching health endpoint before declaring recovery.
 
-### Update Dankserver (frank-default-small)
+Do not stop the other host's inference service during recovery. For benchmark
-
+work, follow the separate stop/benchmark/restart procedure in [[Testing Policy]]
-Run these commands on Dankserver:
+and the repository rules.
-
-```
-cd /home/dank/serverAppConfigs
-git pull origin main
-sudo ln -sf /home/dank/serverAppConfigs/scripts/frank-default-small.service /etc/systemd/system/frank-default-small.service
-sudo systemctl daemon-reload
-sudo systemctl restart frank-default-small
-curl -sf http://127.0.0.1:7770/health
-```
-
-### Update Dankclaw (frank-default-large)
-
-Run these commands on Dankclaw:
-
-```
-cd /home/dank/serverAppConfigs
-git pull origin main
-sudo ln -sf /home/dank/serverAppConfigs/scripts/frank-default-large.service /etc/systemd/system/frank-default-large.service
-sudo systemctl daemon-reload
-sudo systemctl restart frank-default-large
-curl -sf http://127.0.0.1:7770/health
-```
-
-## Service Units
-
-### Dankserver — frank-default-small
-
-The active service on Dankserver is upstream llama.cpp (b10121) serving
-`Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf` under the alias `frank-default-small`
-on port 7770. It runs the stock `/home/dank/llama.cpp/build/bin/llama-server`
-binary with expert tensors selected by an explicit `--override-tensor`
-regex. Expert tensors from every full-attention block and blocks 35 through 40
-stay on CUDA; other expert tensors are CPU-resident. The service uses 7 CPU
-threads, ngram-mod speculative decoding only, f16 KV, context 491520 (3 slots
-of 163840 each), batch
-4096, ubatch 1024, and `--no-mmap`. The unit is
-`scripts/frank-default-small.service`. It launches through
-`scripts/serve-frank-small.sh`, which reads tunables from
-`scripts/frank-default-small.env`. It uses `ubatch=1024` and the chat template
-from `peculiar-ragdoll/Qwen-Sharp-Chat-Templates`. See [[dealign-ornith-crack]].
-
-### Dankclaw — frank-default-large
-
-The active service on Dankclaw is the CIRU ROCmFPX llama.cpp runtime serving
-`kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP` (Q6_0 AGENT, dense)
-under the alias `frank-default-large` on port 7770. It runs
-`/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server` (ROCm
-7.15 SDK in the retired ling3-ciru venv) with a standalone MTP Q4_0 draft
-head (`--model-draft` and `--spec-type draft-mtp`), f16 KV, full 262144
-context (1 slot. Concurrent requests queue; the server never preempts a
-running generation), batch 4096 / ubatch 4096, 16 context checkpoints,
-8 GiB prompt cache, and reasoning in deepseek format. The unit file is
-`scripts/frank-default-large.service`. It launches through
-`scripts/serve-frank-large.sh`, which reads tunables from
-`scripts/frank-default-large.env`, applying the Qwen-Sharp chat template.
-See [[kingjones-strix-mtp]].
-
-> The kingjones Q6 AGENT model has served the slot since 2026-08-24; the
-> v3 PromptForge payload is retired on this slot.
-> Restored 2026-08-21. The intermediate DFlash2 Vulkan deployment was
-> retired the same day. See [[dflash2-large-vulkan]] for the archived runbook.
-
-## Validation
-
-The deploy script checks the health endpoint after a restart. For deeper
-checks, run a completion request against the endpoint:
-
-```
-curl -s http://127.0.0.1:7770/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{"model":"frank-default-small","messages":[{"role":"user","content":"Say OK"}]}'
-```
-
-## Troubleshooting
-
-If the health check fails after a deploy:
-
-1. Read the service logs with `journalctl -u <service> -n 200`.
-2. Confirm that the model files exist in the cache.
-3. Confirm that the GPU has free memory (`nvidia-smi` on Dankserver, `rocm-smi` on Dankclaw).
-4. Check the service status with `systemctl status <service>`.
-
-## Incident: HIP Runtime Crash From the `TEMP` Env Variable (2026-08-13)
-
-On 2026-08-13 the old `frank-default-large` service crash-looped for hours
-on Dankclaw. Root cause: the env files exported `TEMP=1.0` (a sampling
-default). The ROCm HIP runtime reads `TEMP` as its JIT kernel-cache
-directory, used the path `1.0`, failed, and segfaulted on a NULL object.
-Fix: rename the sampling variable to `TEMPERATURE` (commit `227ce01`).
-
-If a service segfaults at load with no obvious cause, compare the exported
-env against the HIP runtime's env reads. Do not name any tunable `TEMP`,
-`HOST`, or `GPU_MAX_HEAP_SIZE`.
-
-## Incident: Jobs "Released Prematurely" Under Parallel Load (2026-08-13)
-
-Historical: on the retired topology, parallel jobs on the Dankserver 35B
-service died with a cuBLAS crash under researcher load. The affected
-service is retired. A client abort logs `W srv stop: cancel task,
-id_task = N`. A natural stop logs only `slot release` with
-`truncated = 0`. The server never preempts a running generation.
-
-## Incident: frank-default-small Crash Loop After Distro Upgrade (2026-08-21)
-
-On 2026-08-21 a release upgrade moved Dankserver from Ubuntu 24.04 to
-Ubuntu 26.04 (Resolute Raccoon). The upgrade removed the CUDA 12 runtime
-packages: `libcudart12`, `libcublas12`, `libcublaslt12`, and
-`nvidia-cuda-toolkit`. The `llama-server` binary links
-`libcudart.so.12` and `libcublas.so.12`. On every start the binary exited
-with code 127. systemd restart-looped the service continuously. The health
-endpoint on port 7770 stayed down.
-
-Fix: reinstall the runtime packages. The same versions remain in the 26.04
-repository.
-
-    sudo apt-get install -y libcudart12 libcublas12 libcublaslt12
-
-Verify that the binary resolves all libraries:
-
-    ldd /home/dank/llama.cpp/build/bin/llama-server | grep "not found"
-
-Start the service and check the health endpoint on port 7770.
-
-After any release upgrade on Dankserver, run the `ldd` check above. A
-missing CUDA 12 library means the packages need a reinstall.
 
 ## See Also
 
-- [AGENTS.md](../../AGENTS.md) — the git workflow rules
+- [Deployment Flow](../Architecture/Deployment%20Flow.md)
-- [README.md](../../README.md) — repository overview
+- [[index]]
-- [index.md](index.md) — LLM fleet knowledge base index
+- [Dankclaw deployment architecture](../Architecture/Network%20Architecture.md)
-- [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) — the full redo runbook
-- [docs/index.md](../index.md) — the project knowledge base (podman stacks)

docs/LLM/eemin-rocmi4-mtp.md

@@ -0,0 +1,186 @@
+---
+type: reference
+title: caf-eemin Qwen3.8-27B ROCmI4 embedded MTP on Dankclaw
+description: caf-eemin/Qwen3.8-27B-ROCmI4-MTP-GGUF (Q4_0_ROCMI4, embedded NextN/MTP head) served by the charlie12345/ROCmFPX llama.cpp runtime (ROCmI4 W4A4) on the Strix Halo iGPU. frank-default-large.
+tags:
+  - llama.cpp
+  - rocm
+  - dankclaw
+  - gfx1151
+  - rocmi4
+  - mtp
+  - iu4
+timestamp: 2026-08-25
+---
+
+# caf-eemin Qwen3.8-27B ROCmI4 embedded MTP on Dankclaw
+
+> **Active 2026-08-25.** `frank-default-large` serves this model. It replaced
+> the kingjones777 ROCmFP4 STRIX MTP payload on the same port and the same
+> Strix Halo iGPU, running the new charlie12345/ROCmFPX ROCmI4 W4A4 build. See
+> [[kingjones-strix-mtp]] for the retired page.
+
+`frank-default-large` (port 7770) serves
+`cafonz/Qwen3.8-27B-ROCmI4-MTP-GGUF` (published under the public
+`eemin` namespace) from the **charlie12345/ROCmFPX** llama.cpp fork built with
+the opt-in `ROCmI4 W4A4` path. The model is `Q4_0_ROCMI4`: signed four-bit
+weights packed two per byte, quantized from the BF16 Qwen3.8-27B and carrying
+its **embedded NextN/MTP head** in the main GGUF — so there is no separate
+draft file. The native gfx1151 `v_wmma_i32_16x16x16_iu4` matrix instruction
+batch-verifies the MTP proposals for higher decode throughput.
+
+## What This Is
+
+**ROCmI4** is the GGUF weight format (signed 4-bit codes). **IU4** is the AMD
+WMMA instruction spelling that consumes them. **W4A4** means four-bit weights
+and four-bit activations during the accelerated matrix multiply. Ordinary
+one-token decode stays memory-bandwidth-bound (MMVQ); batched prompt processing
+and batched MTP target verification use the IU4 MMQ path. W4A4 is a lossy
+activation-quant speed path — treat it as a speed/quality tradeoff.
+
+The model is a conversion/quantization of `Qwen/Qwen3.8-27B` (Unsloth lineage,
+Apache-2.0), not a retrained model. It reports architecture `qwen35`
+(M-RoPE), ~27.32B params, native 262144 context, 866 tensors, ftype 108.
+
+## Model Files
+
+| File | Size | SHA-256 |
+|------|------|---------|
+| `Qwen3.8-27B-Q4_0_ROCMI4.gguf` | 13.50 GiB (14,534,384,384 B) | `ad51ecbee55930ababc629c377943a7bbe50f8aafcf32340c7cdae6fa6b7bb3c` |
+| `mmproj-BF16.gguf` (vision, optional) | 0.87 GiB (931,146,432 B) | `83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53` |
+
+Store under `/home/dank/models/eemin-Qwen3.8-27B-ROCMI4-MTP/`. The vision
+projector is optional for text-only use.
+
+## Runtime Build
+
+The stock llama.cpp build does not understand `Q4_0_ROCMI4` or the qualified
+IU4/MTP path. Use charlie12345/ROCmFPX at the pinned commit, built HIP-only
+(so the combined HIP/Vulkan scheduler cannot route the ROCmI4 tensors to
+Vulkan, which has no shader for this type) with the W4A4 option on.
+
+| Artifact | Path |
+|----------|------|
+| Fork + pinned commit | `/home/dank/rocmi4-qwen38/ROCmFPX` @ `c49ebdbd5c9f01ec242369f9e7f7967855f80cba` |
+| Build | `cmake -S . -B build-rocmi4-w4a4 -G Ninja -DGGML_HIP=ON -DGGML_VULKAN=OFF -DCMAKE_HIP_ARCHITECTURES=gfx1151 -DGGML_HIP_ROCMI4_W4A4=ON -DLLAMA_BUILD_SERVER=ON -DLLAMA_BUILD_WEBUI=OFF` |
+| Binary | `/home/dank/rocmi4-qwen38/ROCmFPX/build-rocmi4-w4a4/bin/llama-server` (version 257, `c49ebdb`) |
+| ROCm | system `/opt/rocm-7.2.4`, amdclang 22 |
+
+Build a separate directory so the exact-int8 path (the default branch) stays
+available. Rollback: disable `-DGGML_HIP_ROCMI4_W4A4=ON` and rebuild — no model
+conversion or data change needed. The build prints
+`ROCmI4 W4A4: enabled for device 0` when the accelerated path is active.
+
+## Model Download
+
+Downloaded from the public
+`eemin/Qwen3.8-27B-ROCmI4-MTP-GGUF` repo (the card's download section still
+names the still-private `cafonz/…`; use `eemin`). Verify against the repo's
+`checksums.sha256` after download:
+
+```
+cd /home/dank/models/eemin-Qwen3.8-27B-ROCMI4-MTP
+sha256sum -c checksums.sha256
+```
+
+## Launch Profile
+
+The qualified MTP profile (from the model card and PR #94, verified on the
+live service): strict Qwen MTP, backend sampling, `n_max 4`, `p_min 0.60`.
+
+```mermaid
+graph TD
+    A[llama-server charlie12345 ROCmFPX c49ebdb<br/>ROCmI4 W4A4 enabled] --> B[load Q4_0_ROCMI4 GGUF + mmproj<br/>-ngl 999 -dev ROCm0]
+    B --> C[draft-mtp on ROCm0<br/>--spec-draft-ngl all]
+    C --> D[strict Qwen MTP<br/>n_max 4 / p_min 0.60<br/>backend sampling]
+    D --> E[Sharp chat template<br/>reasoning deepseek]
+```
+
+Key flags (see `scripts/serve-frank-large.sh`):
+
+```
+--spec-type draft-mtp --spec-mtp-strict-qwen
+--spec-draft-device ROCm0 --spec-draft-ngl all
+--spec-draft-type-k f16 --spec-draft-type-v f16
+--spec-draft-n-max 4 --spec-draft-n-min 0
+--spec-draft-p-min 0.60 --spec-draft-backend-sampling
+```
+
+Because the MTP head is **embedded**, no `--model-draft` is passed and the
+draft uses all layers on ROCm0. `--spec-mtp-strict-qwen` preserves the target
+path's greedy decisions; it does not remove the W4A4 activation approximation.
+
+### Serving settings
+
+- 65536 context (192K below the native 262144 maximum), 1 slot, f16 KV,
+  context checkpoints, prompt cache. The lower limit leaves headroom for the
+  model plus KV cache in the host's unified-memory budget. Concurrent requests
+  queue; the server never preempts a running generation.
+- `-t 16 -tb 32`, batch 512 / ubatch 256 (the qualified W4A4 profile).
+- `--temp 0.7 --top-p 0.95`, reasoning on (`deepseek` puts `<think>` into
+  `message.reasoning_content`).
+- Qwen-Sharp chat template (`--chat-template-file`), `enable_thinking` +
+  `preserve_thinking` on, preserving the peg-native + deepseek shape the Pi
+  agent expects.
+- Strix Halo unified memory + native gfx1151 detection: `HSA_OVERRIDE_GFX_VERSION=11.5.1`,
+  `GGML_HIP_ENABLE_UNIFIED_MEMORY=1`.
+
+### Service
+
+- Unit: `frank-default-large.service`, port 7770
+- Script: `scripts/serve-frank-large.sh`, env `scripts/frank-default-large.env`
+
+## Tuning MTP
+
+MTP gains are content-dependent: predictable code, structured data, and lists
+accept longer drafts than creative prose. The active conservative profile is
+`n4 / p0.60`; the previously qualified starting point was `n16 / p0.60`.
+
+## ubatch Benchmark
+
+Decode throughput depends on the ubatch (per-eval token batch). Run the sweep
+on 8k contexts, 512 → 4096 in steps of 512, with a fixed batch:
+
+```
+sudo systemctl stop frank-default-large
+./scripts/bench-ubatch.sh
+sudo systemctl start frank-default-large
+```
+
+The script stops `frank-default-large` for a clean GPU, boots one server
+instance per ubatch on a scratch port, reads authoritative `print_timing` tok/s,
+and restarts the service. See [[frank-default-large-benchmark]].
+
+## Known Issues
+
+1. **gfx1151 only** for the W4A4 path. Other GPUs fall back to exact int8 MMQ.
+2. **Memory pressure:** the model is 13.5 GiB. Under `--no-mmap` with full
+   offload it needs the whole tensor set at once; leave room in the 64 GiB
+   unified pool (rocm-smi figure). Do not run a bench with the live service up
+   or it OOMs at load.
+3. **`Q4_0_ROCMI4` + MTP flags need the ROCmFPX fork**; stock llama.cpp may
+   reject the file or drop the optimized path.
+4. **W4A4 is lossy** (+~5.4% perplexity vs exact int8). Opt-in speed path.
+
+## Measured Results (qualification)
+
+From the model card, gfx1151, pinned build, W4A4 + strict MTP-16:
+
+| Result | Value |
+|--------|-------|
+| HumanEval pilot, W4A4 IU4 (mean) | 49.40 tok/s |
+| Full HumanEval, W4A4 (mean / median) | 44.39 / 45.23 tok/s |
+| HumanEval / HumanEval+ pass@1 | 94.5% / 91.5% |
+| Non-speculative tg128 | ~13.8 tok/s |
+
+Local smoke test 2026-08-25 on the iGPU: model loaded, strict-MTP active,
+draft acceptance 0.74 (mean 7.67), generation ~25 tok/s at a small output.
+The ubatch sweep above gives the production ubatch choice.
+
+## See Also
+
+- [[deployment]] — deploy and update process
+- [[kingjones-strix-mtp]] — the retired ROCmFP4 predecessor on this slot
+- [[ciru-activefpx-promptforge]] — the retired v3 CIRU PromptForge page
+- [[Dankclaw System Specs]] — the box
+- [[frank-default-large-benchmark]] — backend/spec benchmark history

docs/LLM/ikllama-ornith-dankserver.md

@@ -0,0 +1,262 @@
+---
+type: reference
+title: ik_llama.cpp on Dankserver (Ornith Q8_0)
+description: ikawrakow/ik_llama.cpp replaces upstream llama.cpp for frank-default-small on the RTX 3090. Q8_0 model, KV K default f16 / V q6_0 with Hadamard on V only (iq5_k is not a KV-cache type in this build; q8_KV crashes on this arch), split-mode none, speculative decoding disabled, 3 parallel slots x 163840 ctx, the Qwen-Sharp template rendered with --jinja, and explicit CPU-expert offload. Includes the mandatory CUDA/IQK FA_ALL_QUANTS build flags.
+tags:
+  - ikllama
+  - llamacpp
+  - cuda
+  - dankserver
+  - moe
+  - ornith
+  - kv-cache
+  - hadamard
+timestamp: 2026-08-26
+---
+
+# ik_llama.cpp on Dankserver (Ornith-1.5-35B-A3B CRACK)
+
+> **Active 2026-08-26.** `frank-default-small` switched from upstream
+> llama.cpp (b10121) to `ikawrakow/ik_llama.cpp` (commit `b166e269`) on the
+> RTX 3090. This page documents the build, the mandatory FA build flags, and
+> the MoE / CPU-offload runtime arguments.
+
+The model is `dealignai/Ornith-1.5-35B-A3B-UNCENSORED` (CRACK **Q8_0**), 35B-total /
+3B-active MoE-dense hybrid with an MTP head in the GGUF (mmproj f16 vision
+encoder). Served as **three parallel 160000-context slots** (`PARALLEL=3`).
+Speculative decoding is **disabled**. See [[dealign-ornith-crack]] for the
+model runbook.
+
+## Why ik_llama.cpp
+
+ik_llama.cpp is ikawrakow's llama.cpp fork with heavy CPU + CUDA
+optimizations, IQK/AVX-512 kernels, `--fit` auto-offload, and fast q8_KV
+cache. Dankserver is a CUDA (RTX 3090) + CPU-MoE-offload box, which is
+exactly the supported hybrid the fork targets. Only CPU and CUDA backends
+are fully functional; ROCm / Vulkan / Metal are unsupported upstream.
+
+## Build recipe (CUDA 13.2 via staged pip toolkit)
+
+The system CUDA toolkit on Dankserver is **removed** (`nvidia-cuda-toolkit`
+is in `rc` state; no `/usr/local/cuda*`, no `/usr/bin/nvcc`). The CUDA
+13.2 toolkit is installed via **pip** inside the vllm venv at
+`/home/dank/vllm/.venv/lib/python3.12/site-packages/nvidia/cu13/`. A staged,
+self-contained toolchain at `/home/dank/cuda13-stage` adds the unversioned
+`.so` aliases and static libs CMake needs.
+
+```bash
+# 1. Clone a pinned ik_llama.cpp
+git clone https://github.com/ikawrakow/ik_llama.cpp.git /home/dank/ik_llama.cpp
+cd /home/dank/ik_llama.cpp
+git checkout b166e269   # the version this deployment was built from
+
+# 2. Configure (stage already symlinks nvcc/include/lib for cu13)
+cmake -B build \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DGGML_CUDA=ON \
+  -DGGML_CUDA_FA_ALL_QUANTS=ON \
+  -DGGML_IQK_FA_ALL_QUANTS=ON \
+  -DCMAKE_CUDA_ARCHITECTURES=86 \
+  -DCMAKE_CUDA_COMPILER=/home/dank/cuda13-stage/bin/nvcc \
+  -DCUDAToolkit_ROOT=/home/dank/cuda13-stage \
+  -DGGML_NATIVE=ON
+
+# 3. Build
+cmake --build build --config Release -j 8
+```
+
+### ⚠️ MANDATORY: always include the FA all-quants flags
+
+> **Rule.** Any ik_llama.cpp build that serves a quantized KV cache
+> (q8_KV / q8_0 / Hadamard / any non-f16 KV) **MUST** be compiled with
+> BOTH:
+> - `-DGGML_CUDA_FA_ALL_QUANTS=ON` — compiles every K/V cache quant combo
+>   into the **CUDA** FlashAttention kernels. Without it, CUDA FA only ships
+>   a subset and **errors at runtime** telling you to recompile when a combo
+>   (e.g. q8_KV + Hadamard V-cache) isn't present. The RTX 3090 keeps KV on
+>   the GPU, so this is the flag that matters here.
+> - `-DGGML_IQK_FA_ALL_QUANTS=ON` — compiles all quant types (Q4_0/Q4_1/
+>   IQ4_NL on top of the always-on F16/Q8_0/Q6_0/BF16) into the **CPU/IQK**
+>   FA kernels. Relevant because the MoE experts run on the CPU; the KV
+>   cache that backs CPU-side attention uses the IQK FA path.
+>
+> These are the old "asymmetrical quants" compile-time gates. On a CUDA
+> host, omitting `GGML_CUDA_FA_ALL_QUANTS` is the #1 way to get a
+> "recompile with GGML_CUDA_FA_ALL_QUANTS" crash at load time.
+
+`GGML_NATIVE=ON` lets GCC target the host (the Ryzen 7 7700X is Zen4, so
+`-march=native` enables the AVX512 / VNNI IQK kernels automatically —
+verifiable with `system_info: AVX512_VNNI = 1` in the boot log).
+
+### CUDA stage note
+
+`/home/dank/cuda13-stage` is the pip `cu13` toolkit plus the symlinks CMake's
+(CUDAToolkit) finder needs (`libcudart.so`, `libcublas.so`, `libcublasLt.so`
+and static `libcudadevrt.a`/`libcudart_static.a`). It also works around the
+CCCL CUDA version skew by defining
+`-DCCCL_DISABLE_CTK_COMPATIBILITY_CHECK` in `CMAKE_CUDA_FLAGS` (the pip
+package ships nvcc 13.2.78 but reports `CUDART_VERSION 13000`, which trips
+a strict header lint). Do not re-run configure without that flag or you get
+a hard compile error.
+
+## Runtime configuration
+
+Files: `scripts/serve-frank-small.sh`, `scripts/frank-default-small.env`,
+`scripts/frank-default-small.service` in the repo. The service just calls
+the serve script; the env file holds all tunables.
+
+### KV cache: K default f16 / V q6_0 + Hadamard (V only)
+
+- `CACHE_TYPE_K=f16` — K cache at the default full-precision f16
+  (passes `--cache-type-k f16`). No Hadamard on K.
+- `CACHE_TYPE_V=q6_0` — V cache at 6-bit k-quant
+  (passes `--cache-type-v q6_0`).
+- `V_CACHE_HADAMARD=1` — `--v-cache-hadamard` on V only. The Hadamard
+  transform preserves V quality at 6-bit; K is full-precision f16 and does
+  not need it (`K_CACHE_HADAMARD=0`). Implemented for both CUDA
+  (`ggml-cuda/hadamard.cu`) and CPU (IQK) paths. Flash-attention is **on
+  by default**.
+
+> **KV cache type history on this build (`b166e269`):**
+> - `iq5_k` is **not** a KV-cache type — the cache whitelist is
+>   `f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/q5_1/q6_0/q8_KV`; `iq5_*` only
+>   exist as weight quants. `--cache-type-k iq5_k` throws `Invalid cache
+>   type`.
+> - `q8_KV` **crashes** with a `ggml view` size assert
+>   (`ggml.c:5427: data_size + view_offs > ggml_nbytes(view_src)`) on this
+>   QWEN3MOE/MTP + multimodal arch — independent of PARALLEL/speculation.
+> - The deployment settled on K at default **f16** (full precision, no
+>   Hadamard) with V at **q6_0 + Hadamard**.
+
+With 3 parallel slots at 163840 context the KV cache is large; keeping K at
+full precision f16 while quantizing only V (with Hadamard) is the chosen
+budget split. The GPU-side expert set stays reduced to fit (see the expert
+section).
+
+### Split mode: none (single GPU)
+
+- `SPLIT_MODE=none` → `--split-mode none` (the default). Dankserver has a
+  single RTX 3090 yet ik_llama rejects `--split-mode graph` on a box with a
+  single device:
+
+  > `llama error loading model: It is not possible to use split mode 'graph'
+  > with less than 2 devices`
+
+  So `graph` is **not** a harmless no-op on one GPU — it is a hard load
+  error (this bit us on the first deploy and loop the service). It also still
+  works with the `--override-tensor` expert pins.
+
+> **Lesson.** ik_llama `--split-mode graph` requires ≥2 devices. On a
+> single-GPU host you must use `none` (or `layer`, which also needs >1).
+> Do not assume `--split-mode graph` degrades gracefully on one GPU.
+
+### Expert offload (MoE: CPU with a partial GPU slice)
+
+The expert-offload pattern from upstream llama.cpp carries over: the
+`--override-tensor` regex pins the sparse expert (`ffn_*_exps`) tensors to
+the CPU buft, leaving the output-adjacent blocks 35-40 on CUDA. On the
+Q8_0 model each ffn expert tensor is **272 MiB** (~4 tensor/block ≈ 1088
+MiB/block). With `PARALLEL=3` the KV cache tripled, so most expert blocks
+are pinned to CPU; only blocks **35-40** (output-adjacent, latency- and
+quality-relevant) stay on CUDA. Full-attention blocks 3,7,11,15,19,23 have
+no exps and are untouched by the regex.
+
+- `OVERRIDE_EXPERT` — the layer-bounded regex listing the blocks whose
+  experts stay on CPU: `(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|
+  24|25|26|27|28|29|30|31|32|33|34)`. Blocks 35-40 (and the always-on
+  full-attention set) are left off the list so their experts load to CUDA.
+- `OVERRIDE_BUFT=CPU`.
+- `N_GPU_LAYERS=999` — ik_llama `-ngl` / `--gpu-layers` only accepts an
+  integer (the literal `all` is rejected). 999 means "load as many tensor
+  layers as possible", while the expert override still pins the listed exps
+  to CPU.
+
+> **Fused MoE** (`fused_moe`, `fused_up_gate`, `fused_mmad`) is **on by
+> default** in ik_llama — there is no positive flag; only `-no-fmoe`
+> disables it. `only_active_experts` is also on by default. Confirmed in the
+> boot log for the running service.
+
+To move more experts back to the GPU later, drop more block numbers from
+`OVERRIDE_EXPERT` (each block is ~1088 MiB on Q8_0; verify `nvidia-smi`
+headroom first and keep ~2-3 GiB margin for KV/compute). With the 3-slot KV
+cache there is little headroom, so the default keeps only 35-40 on GPU.
+
+### Qwen-Sharp chat template via --jinja
+
+`--chat-template-file` alone only stores the template text; ik_llama **does
+not** render a custom template as real Jinja unless `--jinja` is also given
+(`inputs.use_jinja ? common_chat_templates_apply_jinja : <non-jinja
+fallback>`). Without `--jinja` it warns it only supports "commonly used
+templates" and silently uses built-in handlers, dropping the Qwen-Sharp
+template's thinking / image / tool formatting. So the serve script passes
+both `--chat-template-file` and `--jinja` (gated on `USE_JINJA`, set to `1`).
+
+### Speculative decoding: disabled (3 parallel slots)
+
+`SPEC_TYPE=none` (passed as `--spec-type none`). Speculative decoding is
+intentionally disabled because ik_llama **limits any speculative stage to a
+a single server slot**: with `--parallel 3` a configured spec stage
+hard-exits at load with
+
+> `ERR [load_model] Speculative decoding is currently limited to a single
+> server slot (-np 1).` / `n_parallel=3`
+
+This service keeps **3 parallel slots** (one per user family via
+`PARALLEL=3`), so speculative decoding cannot be used here. MTP stays
+disabled. The model is a low-parameter MoE (3B active of 35B) and does not
+need the speculation it would cost a slot for.
+
+> **Lesson.** Do not combine speculative decoding with `--parallel > 1` in
+> ik_llama. Either drop to `-np 1` (one slot) and keep a spec stage, or keep
+> multiple slots and set `SPEC_TYPE=none`. For a multi-user service the
+> extra slots win.
+
+### Flash-attention + sampling
+
+Flash-attention is **on by default** in ik_llama (`flash_attn=true`), so no
+`-fa`/`--flash-attn` flag is needed. Sampling, thread, batch, context, and
+checkpoint tunables (`TEMPERATURE`, `TOP_K`, `TOP_P`, `MIN_P`, thread,
+`BATCH_SIZE`, `UBATCH_SIZE`, `CTX_SIZE`, `PARALLEL`, `CTX_CHECKPOINTS`,
+`NO_MMAP`) are unchanged and pass straight through.
+
+## MoE / hybrid-arg reference (not in upstream llama.cpp)
+
+Several ik_llama knobs matter for a CPU-MoE offload host:
+
+| flag | meaning | our use |
+|------|---------|---------|
+| `--split-mode graph` | split tensors+graph across devices; strong for MoE; **requires 2+ devices** | single 3090 → use `none`
+| `-ooae` / `--offload-only-active-experts` | default ON: copy **only active** experts RAM→VRAM | keep ON (huge for A3B: few experts active) |
+| `-grt q8_0` | reduce type for inter-GPU data exchange | not needed (single GPU) |
+| `--fit` | auto-offload to fit VRAM | conflicts with `--override-tensor`; do not use together (runtime error) |
+| `-n-cpu-moe N` / `--cpu-moe` | keep MoE weights of first N / all layers on CPU | also conflicts with `--fit`; our `--override-tensor` is finer-grained |
+| `--fit-margin` | VRAM safety margin for `--fit` | only relevant with `--fit` |
+
+The current deployment uses `--override-tensor` (not `--fit` or
+`--n-cpu-moe`) for the expert offload, so the `--fit`-conflict rules do not
+apply.
+
+## Rollback
+
+The previous runtime `upstream llama.cpp b10121` and its binary
+(`/home/dank/llama.cpp/build/bin/llama-server`) are **left untouched**. To
+rollback to the old stack, change `LLAMA_SERVER=` in
+`scripts/frank-default-small.env` back to that path and
+`CACHE_TYPE_K/V` back to `f16` with Hadamard off, and re-enable the ngram
+spec / `--spec-type` flags. The model file is not changed either way.
+
+## See Also
+
+- [[dealign-ornith-crack]] — the Ornith Q6_K model run
+- [[Dankserver System Specs]] — the host
+- [[deployment]] — service-scoped git deploy
+
+## Citations
+
+- [ikawrakow/ik_llama.cpp README](https://github.com/ikawrakow/ik_llama.cpp) —
+  MoE offload guidance and the `--split-mode graph` + CPU-offload warning
+- [ik_llama.cpp docs/build.md](https://github.com/ikawrakow/ik_llama.cpp/blob/main/docs/build.md) —
+  build flags incl. `GGML_CUDA_FA_ALL_QUANTS` and `GGML_IQK_FA_ALL_QUANTS`
+- [ik_llama.cpp docs/parameters.md](https://github.com/ikawrakow/ik_llama.cpp/blob/main/docs/parameters.md) —
+  q8_KV (PR 208), K/V Hadamard options, `-cuda graphs=0` workaround, and
+  `--split-mode graph` MoE note (PR 1080)

docs/LLM/index.md

@@ -1,10 +1,10 @@
 # LLM Fleet — Knowledge Base
 
 Documentation for the LLM inference fleet on two local machines
-(2026-08-24): upstream llama.cpp on Dankserver (frank-default-small,
+(2026-08-26): ik_llama.cpp on Dankserver (frank-default-small,
-Ornith-1.5-35B-A3B CRACK) and CIRU ROCmFPX llama.cpp on Dankclaw
+Ornith-1.5-35B-A3B CRACK) and ROCmFPX llama.cpp on Dankclaw
-(frank-default-large, kingjones Qwen3.8-27B ROCmFP4). Both services
+(frank-default-large, kingjones Qwen3.8 Uncensored Q6 AGENT). Both
-run the Qwen-Sharp chat template.
+services run the Qwen-Sharp chat template.
 
 > 2026-08-22: the former `vllmScripts` repository is merged into the
 > serverAppConfigs repository. This knowledge base moved from the old repo's
@@ -17,24 +17,24 @@ run the Qwen-Sharp chat template.
 graph LR
     subgraph Dankserver[NVIDIA Dankserver - Ubuntu 26.04]
         A[RTX 3090 24GB + 62GB RAM]
-        B[upstream llama.cpp b10121
+        B2[ik_llama.cpp build b166e269
 frank-default-small, port 7770]
         C[Ornith-1.5-35B-A3B-CRACK Q6_K
-ngram-mod, selected experts offloaded via override, 491520 ctx]
+no speculation, q8_0 + Hadamard cache, 491520 ctx]
-        A --> B --> C
+        A --> B2 --> C
     end
 
     subgraph Dankclaw[AMD Strix Halo - Ubuntu 26.04]
-        D[gfx1151 iGPU 64 GiB unified memory]
+        E2[gfx1151 iGPU 64 GiB unified memory]
-        E[CIRU ROCmFPX llama.cpp PromptForge v3
+        E[charlie12345/ROCmFPX build-kairic (v81)
 frank-default-large, port 7770]
-        F[Qwen3.8-27B-Uncensored-ROCmFP4 Q6_0 AGENT
+        F[Qwen3.8 Uncensored Q6 ROCmFPX AGENT
-MTP draft + Sharp template, 262K ctx / 1 slot]
+MTP draft + Turbo4 V cache, 262K ctx / 1 slot]
-        D --> E --> F
+        E2 --> E --> F
     end
 
-    I[LLM Clients / API] -->|HTTP :7770| B
+    CLI[LLM Clients / API] -->|HTTP :7770| B2
-    I -->|HTTP :7770| E
+    CLI -->|HTTP :7770| E
 ```
 
 ## Servers
@@ -48,15 +48,24 @@ MTP draft + Sharp template, 262K ctx / 1 slot]
 
 - [[deployment]] — deploy and update process for both servers
 - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change
-- [[kingjones-strix-mtp]] — the kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model
+- [kingjones Qwen3.8 Uncensored ROCmFP4 STRIX MTP](https://huggingface.co/kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF) — active frank-default-large model on Dankclaw; Q6 AGENT main model with standalone MTP
-  on Dankclaw (frank-default-large, active 2026-08-24)
+- [[kingjones-strix-mtp]] — the active Q6 AGENT runbook: serving profile, standalone MTP, reasoning + preserve_thinking, 114 GiB TTM/GTT
+- [[ubatch-benchmark-large]] — active-model llama-bench ubatch sweep plan and results
+- [[kingjones-q8agent-rocmfpx]] — the retired Q8_0-AGENT investigation and `set_rows` failure
+- [[eemin-rocmi4-mtp]] — the former caf-eemin Qwen3.8-27B embedded-MTP model
+  on Dankclaw (retired)
 - [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on
   Dankserver (frank-default-small, active 2026-08-24)
+- [[ikllama-ornith-dankserver]] — the ik_llama.cpp runtime for
+  frank-default-small (2026-08-26): q8_0 + Hadamard KV cache,
+  split-mode none (single-GPU), mandatory FA_ALL_QUANTS build flags
+- [[kingjones-strix-mtp]] — the active kingjones Qwen3.8-27B ROCmFP4 STRIX MTP
+  model on Dankclaw (restored 2026-09-01 after the Q8_0 AGENT experiment)
 - [[ciru-activefpx-promptforge]] — the RETIRED v3 CIRU PromptForge runtime
   on Dankclaw (superseded 2026-08-24 by the kingjones ROCmFP4 model)
 - [[kairic-edge]] — the retired Kairic Edge IU4 experiment
   (deployed and reverted 2026-08-22)
-- [[Dankserver System Specs]] — the upstream llama.cpp box (frank-default-small)
+- [[Dankserver System Specs]] — the RTX 3090 box (frank-default-small)
 - [[Dankclaw System Specs]] — the CIRU ROCmFPX box (frank-default-large)
 
 ## CIRU Reference (Dankclaw)
@@ -101,10 +110,16 @@ MTP draft + Sharp template, 262K ctx / 1 slot]
 - [[Pipeline Parallelism/index]] — index of multi-node inference research. The PP
   pair is retired. These pages are research history.
 
+## Runtime Research (llama.cpp family)
+
+- [[Llama Cpp Runtimes]] — mainline vs charlie12345/ROCmFPX vs ik_llama.cpp: positioning, quantization, server fit
+- [[Llama Cpp Code Diff]] — code-level quantization diff of the three runtimes + server recommendations
+- [[SGLang]] — the structured-generation serving engine; when it fits Dankserver (3090) vs Dankclaw (Strix Halo)
+
 ## Related
 
 - [Project knowledge base](../index.md) — the Podman app stacks on the same
   two hosts. OpenWebUI consumes the `:7770` endpoint on Dankserver. The
   Gitea instance on Dankclaw hosts this repository.
-- [Deployment Flow](../Architecture/Deployment%20Flow.md) — the unified
+- [Deployment Flow](../Architecture/Deployment%20Flow.md) — the service-scoped
-  git-based deploy for both workloads
+  git-based deploy with inference isolation

docs/LLM/kingjones-q8agent-rocmfpx.md

@@ -0,0 +1,225 @@
+# frank-default-large: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT
+
+**Status:** Deployed on Dankclaw (since 2026-08-25). Serving is **BLOCKED by
+the `set_rows` build bug** — the model loads but crashes in
+`ggml_compute_forward_set_rows` before it can serve. All config goals are met;
+a build fix is required for the model to run.
+
+**Alias:** `frank-default-large`, port **7770**.
+
+The large service serves the kingjones `Qwen3.8-27B-ROCmFPX-Q8_0-AGENT` GGUF
+from `/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT/` with a
+charlie12345/ROCmFPX `llama-server` (build `build-kairic`). It runs a
+**standalone MTP draft head** (loaded separately, on ROCm0), **TurboQuant** V
+KV cache, the **Sharp** chat template with **reasoning on**, and a **full
+262144 context / 1 slot** profile. The whole payload plus the 262k KV cache
+fit in the lifted **114 GiB TTM/GTT** budget (see the 114 GiB change in
+[[Dankclaw Improvements]] and [[qwen3.8-strix-optimizations]]).
+
+| Figure | Value |
+| ------ | ----- |
+| Model | kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT (`Q8_0_ROCMFPX`) |
+| HF repo | `kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF` (public) |
+| Runtime | charlie12345/ROCmFPX `llama-server`, `build-kairic` |
+| Draft head | standalone `mtp-Qwen3.8-27B-Q4_0.gguf` on ROCm0 |
+| KV cache | f16 K, TurboQuant (`turbo4`) V, full 262144 context, 1 slot |
+| Reasoning | on, `deepseek`, budget `-1`, `--reasoning-format none` |
+| Chat template | Sharp (repo-tracked), `enable_thinking` + `preserve_thinking` on |
+| Host path | `/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT/` |
+| Tunables | `scripts/frank-default-large.env` |
+| Serve script | `scripts/serve-frank-large.sh` |
+
+## Serving status: the `set_rows` build bug (blocks serving)
+
+As of 2026-08-25 the Q8_0 AGENT model will not serve on Dankclaw. It loads,
+reaches `initializing slots, n_slots = 1`, but never prints `start serving`.
+Health stays `503` until the process exits. The crash is in
+`ggml_compute_forward_set_rows` in `libggml-cpu.so.0`, a NULL function-pointer
+jump (address `0x00000000`).
+
+The backtrace (gdb). The main thread and one OpenMP worker crash in the same
+path:
+
+```mermaid
+graph TD
+    A[llama_context::graph_compute] --> B[ggml_backend_cpu_graph_compute]
+    B --> C[ggml_graph_compute]
+    C --> D[GOMP_parallel]
+    D --> E[ggml_compute_forward_set_rows]
+    E --> F[NULL dispatch -> address 0x0]
+    D --> G[Thread 23 worker: same path]
+```
+
+Test matrix. Five ROCmFPX builds and the all-CPU path were tested on a scratch
+port (`/tmp/g80bench/run.sh`), `gl=45`, `ctx=131072`. All crash in `set_rows`.
+
+| Build | Binary | Version | Result |
+| ----- | ------ | ------- | ------ |
+| build-kairic | `ciru-qwen38/ROCmFPX/build-kairic/bin/llama-server` | v81 (205a3e5f4) | segfault in set_rows |
+| build-promptforge | `ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server` | PromptForge/ActiveFPX lineage | segfault in set_rows |
+| build-rocmi4-w4a4 | `rocmi4-qwen38/ROCmFPX/build-rocmi4-w4a4/bin/llama-server` | v257 (c49ebdb) | segfault in set_rows |
+| build-strix-rocmfp4 | `GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-server` | v209 (00d5452) | segfault in set_rows |
+| all-CPU | build-kairic, `-ngl 0` | v81 | segfault in set_rows |
+
+The crash is not GPU-specific (the all-CPU path fails too), not MTP-specific
+(`--spec-type none` fails), and not fixed by `LLAMA_SET_ROWS=0`. So it is a
+build bug: the Q8_0 AGENT GGUF graph needs a `set_rows` dispatch the ROCmFPX
+backends do not register. The older caf-eemin ROCmI4 and Kairic Edge models
+served fine on related ROCmFPX builds. The caf-eemin payload was about 14.4
+GiB; the separate
+PromptForge/ActiveFPX deployment used a much larger multi-file payload of about
+36.7 GiB. These are different deployments. The Q8_0 AGENT graph is the trigger
+for the `set_rows` failure.
+
+**Fix path:** rebuild charlie12345/ROCmFPX from source with a working
+`set_rows` for the ROCmFPX backend, or find a build where it is registered.
+This unblocks the model. See the context budget reality below once `set_rows`
+is fixed.
+
+## Model files
+
+Three files, from the HF repo above:
+
+| File | Size | Purpose |
+| ---- | ---- | ------- |
+| `Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf` | 28.21 GB | main weights |
+| `mtp-Qwen3.8-27B-Q4_0.gguf` | 1.68 GB | standalone MTP draft head (loaded by the script) |
+| `mmproj-Qwen3.8-27B-BF16.gguf` | 0.93 GB | vision projector (`--mmproj`) |
+
+The repo also has a `mtp-Qwen3.8-27B-Q8_0.gguf` (3.16 GB) draft; the env uses
+the Q4_0 draft. Download them from the resolve URLs below (they are large —
+use the token in `~/.cache/huggingface/token`).
+
+## Download
+
+```bash
+# on Dankclaw
+TOKEN=$(cat ~/.cache/huggingface/token)
+REPO=kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF
+DIR=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT
+mkdir -p "$DIR"; cd "$DIR"
+dl(){ curl -fsSL -C - -o "$1" --retry 5 --retry-delay 3 --retry-max-time 3600 \
+  -H "Authorization: Bearer $TOKEN" \
+  "https://huggingface.co/$REPO/resolve/main/$1"; echo "done: $1"; }
+dl Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf
+dl mmproj-Qwen3.8-27B-BF16.gguf
+dl mtp-Qwen3.8-27B-Q4_0.gguf
+```
+
+The serve script checks each artifact is present and readable before launch
+(`scripts/frank-default-large.env` -> the four `MODEL_*` files plus the chat
+template). A missing file fails the artifact check with `missing or
+unreadable required artifact`.
+
+## Serving profile
+
+`scripts/serve-frank-large.sh` builds the `llama-server` call from the env:
+
+```
+/dev/llama-server -m <main> --alias frank-default-large
+  --host 0.0.0.0 --port 7770 --jinja
+  -dev ROCm0 -ngl 999 -np 1 -c 262144
+  -b 512 -ub 512 -fa on
+  -ctk f16 -ctv turbo4 -t 16 -tb 32
+  -ctxcp 32 --cache-ram 8192 --cache-prompt --cache-idle-slots --metrics
+  --spec-type draft-mtp --model-draft <mtp-Q4_0>
+  --mmproj <mmproj-BF16>
+  --spec-draft-device ROCm0 --spec-draft-ngl 999
+  --spec-draft-type-k f16 --spec-draft-type-v turbo4
+  --spec-draft-n-max 4 --spec-draft-n-min 0
+  --spec-draft-p-min 0.0 --spec-draft-p-split 0.10
+  --spec-draft-backend-sampling
+  --temp 0.7 --top-p 0.8 --top-k 20 --min-p 0.0
+  --presence-penalty 1.5
+  --reasoning on --reasoning-format none --reasoning-budget -1
+```
+
+Key choices:
+
+- **Standalone MTP draft** (`--model-draft`, `--spec-type draft-mtp`). The
+  AGENT GGUF has no embedded draft head; the draft head is a separate file on
+  ROCm0. This matches the Qwen3.8-27B AGENT shape (`--spec-draft-type-k
+  f16 --spec-draft-type-v turbo4`).
+- **f16 K + TurboQuant V** cache. f16 K is the safe long-ctx default; TurboQuant
+  halves the V cache at 262k while keeping quality (see the ubatch/threads-batch
+  notes in [[qwen3.8-strix-optimizations]]).
+- **262144 context, 1 slot — the target.** The whole context KV cache plus the
+  ~30 GB payload (main + draft + mmproj) are meant to fit in the 114 GiB
+  budget. Measured reality: `gl=999` (all layers on ROCm0) at 262144 **OOMs**
+  on device-local (`cudaMalloc failed`, the KV cache needs ~10-18 GiB that does
+  not fit once the payload is on the card). `gl=45` (partial offload) fits a
+  131072 context. The env ships `CTX_SIZE=262144`; the proven serving context
+  today is **131072 at `gl=45`** (before the `set_rows` fix). `deploy.sh` waits
+  up to 20 minutes (see `LLM_WAIT`).
+- **Reasoning on.** The Sharp template drives thinking (medium effort);
+  `--reasoning-format none` lets the template project the reasoning content into
+  `reasoning_content`. `--reasoning-budget -1` = no budget cap.
+
+## Reasoning + chat template
+
+The Sharp template (`scripts/chat-template.jinja`, repo-tracked) is a
+Qwen-AGENT shape. Env `CHAT_TEMPLATE_KWARGS` keeps the reasoning trail in the
+conversation:
+
+```json
+{"enable_thinking": true, "preserve_thinking": true, "preserve_reasoning": true}
+```
+
+`preserve_thinking: true` keeps the model's prior thinking in the next prompt
+(the user's "preserve thinking" request). `preserve_reasoning: true` keeps the
+model's prior reasoning trace across requests too (OpenWebUI sends
+`preserve_reasoning`/`preserve_thinking: true` when the conversation needs the
+full trail). For multi-turn reasoning, the template stays in the reasoning
+shape and `enable_thinking: true` per request.
+
+## Memory: the 114 GiB TTM/GTT requirement
+
+The 262k f16 KV cache is large (~a few GB) and the payload is ~30 GB. Both live
+in the Strix Halo unified-memory GTT. The TTM `pages_limit` was raised to
+114 GiB on 2026-08-25 (written live to
+`/sys/module/ttm/parameters/pages_limit`, then persisted in
+`/etc/modprobe.d/ttm.conf`). Without the raise the effective ceiling is ~15 GiB
+and a 28 GB + 262k-KV load stalls. The GTT page-limit change is per the
+`qwen3.8-strix-optimizations` table item 5.
+
+## Health + troubleshooting
+
+- Health: `curl http://192.168.0.143:7770/health` → `{"status":"ok"}`.
+- `/api/models` reports `n_ctx:262144, n_ctx_train:262144, n_params:27.3B`.
+- Model load is slow (a few minutes cold for the ~30 GB payload with a fresh
+  KV cache). Health returns 503 while loading. Do not kill the process.
+  `deploy.sh` waits up to 20 minutes (see `LLM_WAIT`).
+- Kernel spam about `amdgpu: SVM mapping failed` is the known documented
+  symptom, not root cause (see the crash-loop procedure in the project
+  AGENTS.md). Investigate other causes before assuming memory pressure.
+- A client abort logs `W srv stop: cancel task, id_task = N`; a natural stop
+  logs only `slot release` with `truncated = 0`.
+
+## Env tunables
+
+`scripts/frank-default-large.env`. Serving: `CTX_SIZE=262144`,
+`PARALLEL=1`, `BATCH_SIZE=512`, `UBATCH_SIZE=512`, `NUM_THREADS=16`,
+`THREAD_BATCH=32`, `CACHE_TYPE_K=f16`, `CACHE_TYPE_V=turbo4`,
+`CTX_CHECKPOINTS=32`, `CACHE_RAM=8192`. Reasoning: `REASONING=on`,
+`REASONING_FORMAT=none`. Sampling: `TEMPERATURE=0.7`, `TOP_K=20`,
+`TOP_P=0.8`. MTP: `MTP_N_MAX=4`, `MTP_N_MIN=0`, `MTP_P_MIN=0.0`,
+`MTP_P_SPLIT=0.10`. Strix Halo: `HSA_OVERRIDE_GFX_VERSION=11.5.1`,
+`GGML_HIP_ENABLE_UNIFIED_MEMORY=1`.
+
+**Never name a tunable `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`** — the ROCm HIP
+runtime reads those bare vars (see the 2026-08-13 TEMP incident). llama.cpp
+reads `--host/--port/--temp` as CLI flags, never as env vars.
+
+## History
+
+- **2026-08-25 (today):** switched the large service to the kingjones
+  Q8_0-AGENT model with 262144 context and reasoning on. Downloaded the three
+  payload files from the HF repo, raised the TTM/GTT to 114 GiB for the full
+  context, and persisted the env/serve script in this repo. Measured that the
+  model loads but crashes in `ggml_compute_forward_set_rows` on all five
+  available ROCmFPX builds; serving is BLOCKED pending a rebuild. The config
+  goals (GTT 114 GiB, reasoning on, preserve_thinking/preserve_reasoning on,
+  262144 env context) are in place. Proven serving context today is 131072 at
+  `gl=45`. See the `set_rows` build bug section above.
+- Preceding model: Kairic Edge Qwen3.8-27B IU4 (deployed 2026-08-25 ~05:42,
+  retired the same day for the AGENT model). See [[kairic-edge]].

docs/LLM/kingjones-strix-mtp.md

@@ -15,10 +15,12 @@ timestamp: 2026-08-24
 
 # kingjones Qwen3.8-27B ROCmFP4 STRIX MTP on Dankclaw
 
-> **Active 2026-08-24.** `frank-default-large` serves this model. It
+> **Active again 2026-09-01.** The kingjones ROCmFP4 STRIX MTP payload is
-> replaced the jcbtc v3 CIRU ActiveFPX PromptForge payload on the same
+> restored as `frank-default-large` after the Q8_0 AGENT experiment. The Q8_0
-> ROCmFPX runtime and the same CIRU llama.cpp binary. See
+> model was retired because it crashed before serving. The Q6 AGENT model,
-> [[ciru-activefpx-promptforge]] for the retired v3 page.
+> standalone MTP draft, Q8 projector, and PromptForge runtime are now active.
+
+> This page documents the active model and its runbook.
 
 `frank-default-large` (port 7770) serves
 `Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP` from `kingjones777` on Hugging
@@ -31,7 +33,7 @@ here; the head is loaded with `--model-draft`.
 
 The card ships six text files plus the vision projector. The service uses
 the Q6 AGENT file (requested for the fleet), the separate MTP draft head
-and the Q8 vision projector.
+and the Q8 vision projector (`mmproj-Qwen3.8-27B-Q8_0.gguf`).
 
 | File | Size |
 |------|------|
@@ -68,7 +70,9 @@ llama-server \
   -m Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf \
   --mmproj mmproj-Qwen3.8-27B-Q8_0.gguf \
   --chat-template-file chat-template.jinja \
-  -ngl 999 -dev ROCm0 -c 262144 -np 1 \
+  --chat-template-kwargs '{"enable_thinking":true,"preserve_thinking":true}' \
+  -ngl 999 -dev ROCm0 -c 196608 -np 1 \
+  -b 8192 -ub 8192 -ctxcp 32 \
   --spec-type draft-mtp --model-draft mtp-Qwen3.8-27B-Q4_0.gguf \
   --spec-draft-ngl 999 --spec-draft-n-max 4 \
   -fa on --no-mmap --cache-type-k f16 --cache-type-v f16
@@ -82,6 +86,10 @@ llama-server \
    `enable_thinking: false` instead.
 4. Small `max_tokens` with thinking on returns empty `content`. The Sharp
    template drives thinking on by default (effort medium).
+5. `preserve_thinking:true` (in `chat_template_kwargs`) keeps `<think>`
+   blocks in recorded assistant messages. `preserve_thinking:false` strips
+   them. See [[deployment]] for the chat_format/reasoning_format pairing
+   (`peg-native` + `deepseek`) the Pi agent expects from this endpoint.
 
 ## See Also
 

docs/LLM/qwen3.8-strix-optimizations.md

@@ -19,7 +19,7 @@ Related pages:
 | 2 | MTP `n_max` explicit 4–5 + Q4_0 draft head | +5–15% on the spec share | restart, 1.56 GB file | test 08-16 |
 | 3 | Plain `Q8_0` A/B vs UD-Q8_K_XL (same 8 bits) | dequant ALUs may differ | restart, 29 GB file | test 08-16 |
 | 4 | Prefill path: ROCm backend + tuned ubatch/threads-batch | 2–3× prefill for agentic turns | per-workload backend choice | partial data |
-| 5 | TTM `pages_limit` raise (15.2 GiB → 90+) | removes a hard VRAM ceiling, stable 96 GiB budget | reboot | pending |
+| 5 | TTM `pages_limit` raised to 114 GiB | removes the hard VRAM ceiling; the full 262k f16 KV plus the Q8 AGENT payload, MTP draft, and mmproj fit in one slot | modprobe, set live 2026-08-25 | applied (29,883,536 pages) |
 | 6 | Vulkan ICD pinning + loader-layer disable | robustness; small | service env vars | test 08-16 |
 
 Everything else (quantization downgrade, NVFP4, larger drafts) is documented
@@ -166,11 +166,13 @@ hybrid state, upstream). Levers:
 
 ### 5. OS level (from the Improvements doc, re-verified 2026-08-15)
 
-- **TTM `pages_limit` is still 3,995,281 pages ≈ 15.24 GiB** — verified
+- **TTM `pages_limit` is now 114 GiB (29,883,536 pages)** — applied 2026-08-25.
-  today, unchanged. The 96 GiB GTT budget only materializes if the TTM cap
+  Written live to `/sys/module/ttm/parameters/pages_limit` (no reboot required)
-  is raised (kernel parameter, needs reboot). Until then the effective
+  and persisted in `/etc/modprobe.d/ttm.conf`. The full 262k f16 KV cache plus
-  ceiling is lower and the 28 GiB load-stall risk remains. **Highest-value
+  the ~28 GB Q8 AGENT payload, MTP draft, and mmproj now fit in one slot. The
-  single OS change; schedule with the next reboot.**
+  load-stall risk from the old 15 GiB cap is resolved. Note: 96 GiB is the
+  driver-reported VRAM carve-out (VGM/SVM pool); the TTM pages_limit is the
+  kernel page map that was the real ceiling.
 - `iommu=pt` — verified present on the kernel cmdline. Done.
 - **Vulkan ICD pinning** — `radeon_icd.json` exists; no `VK_ICD_FILENAMES`
   or loader-layer env is set. Pinning the radeon ICD and disabling loader

docs/LLM/ubatch-benchmark-large.md

@@ -0,0 +1,56 @@
+---
+type: benchmark plan
+title: frank-default-large ubatch sweep
+description: Standardized ubatch sweep for the active kingjones Qwen3.8 ROCmFPX deployment.
+tags:
+  - llama.cpp
+  - rocm
+  - benchmark
+  - ubatch
+timestamp: 2026-09-01
+---
+
+# frank-default-large ubatch sweep
+
+This benchmark uses the active `frank-default-large` model on Dankclaw with
+`llama-bench`. It does not use `llama-server`, HTTP requests, or server-log
+scraping. Stop the production service before running it and start the service
+again afterward.
+
+The old custom harnesses are retained in `.AGENTTRASH/bench-scripts-20260901/`
+for auditability only. Their results are invalid and must not be used.
+
+## Sweep
+
+The requested points are `256, 512, 768, 1024, 1280, 1536, 1792, 2048,
+3072, 4096` at 8K context, with `batch` equal to `ubatch` at every point.
+Each point runs three 128-token repetitions by default.
+
+Run on Dankclaw from the repository checkout:
+
+```bash
+sudo systemctl stop frank-default-large
+./scripts/bench-ubatch.sh
+sudo systemctl start frank-default-large
+```
+
+The script uses the active Q6 AGENT model, an 8K prompt workload, equal
+`batch` and `ubatch` values, and `llama-bench` JSONL output. The sweep is:
+```text
+256, 512, 768, 1024, 1280, 1536, 1792, 2048, 3072, 4096
+```
+
+The GPU is shared with live workloads; do not run with the production service
+up.
+
+## Results
+
+No valid results are recorded here. All results from the retired custom
+harnesses are explicitly discarded. Record the raw `llama-bench` JSONL output
+from the new harness here after a clean run.
+
+## See also
+
+- [LLM fleet index](index.md)
+- [Deployment](deployment.md)
+- [Active kingjones runbook](kingjones-strix-mtp.md)

docs/Research/Llama Cpp Code Diff.md

@@ -0,0 +1,222 @@
+---
+type: research
+title: Llama.cpp Code Diff
+description: Code-level diff of mainline llama.cpp, charlie12345/ROCmFPX, and ik_llama.cpp with a focus on quantization code, ggml type numbers, and the differences that matter for Dankserver (RTX 3090) and Dankclaw (Strix Halo).
+tags:
+  - llama.cpp
+  - quantization
+  - ggml
+  - ROCmFPX
+  - ik_llama
+  - code-diff
+timestamp: 2026-09-01
+---
+
+# Llama.cpp Code Diff
+
+Code-level diff of the three runtimes, focused on **quantization code** and the
+differences that affect our two servers. Positioning overview is in
+[[Llama Cpp Runtimes]].
+
+## 1. Quantization file map
+
+Where quant lives in each runtime (paths relative to the repo root):
+
+| File / path | Mainline | ROCmFPX | ik_llama.cpp |
+|-------------|----------|---------|--------------|
+| `ggml/include/ggml.h` (`enum ggml_type`) | FP16/FP32/BF16, legacy `Q*_0`, K-quant types, `IQ*_` | **adds** `Q4_0_ROCMFP4` … `Q2_0_ROCMFPX`, `Q4_0_ROCMI4`, `Turbo3_0`, `Turbo4_0` (enums **100–108**) | IQK non-linear (`IQ4_NL` etc.), Trellis `IQ*_KT`, MXFP4, Bitnet, `Q8_KV` |
+| `ggml/src/ggml-quants.*` | K/I/legacy dequant, row-interleaved 3-bit | `block_rocmfp4` unpack + E4M3 scaling, IU4 WMMA dispatch | `iqk/*` fused quant-gemms, IQK row-interleave, Trellis decode |
+| `tools/quantize/quantize.c` | all mainline quants + `--imatrix` | ROCmFPX quantizer (`STRIX`/`FAST`/`_AGENT` presets) | IQK/Trellis/MXFP4 packers, `--cache-type-k/v` + Hadamard |
+| `gguf.cpp` / `gguf-io.c` | standard ftype table | **custom ftypes** 100–106 (FP4), 110–113 (FPX), 118 (I4) | IQK/Trellis ftype tags |
+| KV cache | `f32/f16/bf16/q8_0/q4_0/q4_1/iq4_nl/q5_0/q5_1` (needs FlashAttention) | **TurboQuant** `Turbo3_0`/`Turbo4_0`/`V` | `Q8_KV`, IQK/Trellis/MXFP4 reused in `-ctk`, `--k-cache-hadamard`, `--v-cache-hadamard` |
+
+Two number spaces in ROCmFPX (authoritative `ggml/include/ggml.h`, sha
+`c49ebdbd`):
+- **`ggml_type` enum** (tensor storage): `Q4_0_ROCMFP4`=100, `Q4_0_ROCMFP4_FAST`=101,
+  `Q6_0_ROCMFPX`=102, `Q8_0_ROCMFPX`=103, `Q3_0_ROCMFPX`=104, `Turbo3_0`=105,
+  `Turbo4_0`=106, `Q2_0_ROCMFPX`=107, `Q4_0_ROCMI4`=108, count 109.
+- **`ggml_ftype`** (GGUF model-file): ROCmFP4 family `100–106` (`STRIX`=105,
+  `STRIX_LEAN`=106), ROCmFPX family `110–113` (FP6=110, FP8=111, FP3=112,
+  FP2=113), `ROCmI4`=118.
+
+## 2. Comparison table
+
+| Runtime | Quant formats | Granularity | Custom ggml types | MoE | Attention accel | Hybrid GPU/CPU offload | Build backends |
+|---------|---------------|-------------|-------------------|-----|-----------------|------------------------|----------------|
+| Mainline | K/I/legacy/TQ, imatrix, row-RI 3-bit | grouped/per-block + double-quant | none | native `--cpu-moe` | FlashAttention (multi-backend) | `--n-gpu-layers`, partial offload | CPU/CUDA/Vulkan/Metal/SYCL/HIP-ROCm/CANN |
+| charlie12345/ROCmFPX | ROCmFP2/3/4/6/8, ROCmI4, TurboQuant KV | block + E4M3 codebook | **yes (100–108)** | MTP draft head (bundled) | **MTEA** + TurboQuant FA (custom) | Strix-tuned MTP profile | HIP/ROCm + Vulkan (**CUDA off**), CPU ref |
+| ik_llama.cpp | IQ* non-linear, improved K, IQK row-RI, Trellis, MXFP4, Q8_KV, Bitnet | grouped + non-linear + row-interleaved | IQK/Trellis/MXFP4/Bitnet tags | **fused MoE**, `-ooae`, `--fit`, `-ot`, GDN | **FlashMLA** (DeepSeek MLA) | **finest-grained** override-tensor / cpu-moe / fit | CPU (AVX-512/VNNI), CUDA (Turing+), Vulkan, ROCm(weak) |
+
+## 3. Code-difference narrative
+
+**What each fork ADDED that mainline lacks:**
+- **ROCmFPX** — ~10 new `ggml_type` entries, the `block_rocmfp4` packed-nibble
+  format, `*_ROCMFPX`/`*_ROCMI4` ftypes, tensor-aware presets (`*_AGENT`,
+  `*_COHERENT`, `*_STRIX`), `block_rocmfp4` dequant + dispatch kernels,
+  TurboQuant KV types, MTEA attention kernel. Mainline does not know these
+  tensor types. It is a *fork* that adds a new backend: custom tensor
+  registration + HIP/ROCm + Vulkan kernels + CPU reference. **CUDA is disabled.**
+- **ik_llama.cpp** — new code under `ggml/src/iqk/` (IQK GEMM + row-interleave),
+  CUDA IQK flash-attn kernels (`GGML_IQK_FA_ALL_QUANTS=ON`), fused MoE
+  (`fnn_up`+`fnn_gate`), fused delta-net (GDN) SSM ops, `flash_fwd_mla` for
+  DeepSeek MLA/DSA, and hybrid-offload controls (`-ot/--override-tensor`,
+  `--cpu-moe`, `-ooae/--offload-only-active-experts`, `-ser/--smart-expert-reduction`,
+  `--fit`, `-op/--offload-policy`, split-mode graph). Run-time quant repack
+  (`-rtr`). **Additive** — GGUF remains compatible.
+
+**What mainline has that the forks lag on:**
+- **ik_llama.cpp**: architecture baseline is **Aug-2024** — newest
+  architectures arrive late (PRs move fast, but the core is ~1.5y behind).
+- **ROCmFPX**: **AMD-first, experimental**; narrower, must rebuild/re-quant per
+  fork line, CUDA ecosystem (the fleet's 3090) is unreachable.
+
+**The one caveat that changes a build (ik_llama.cpp + `-rtr`):**
+K-quants have **no CUDA row-interleaved implementation**. With `-rtr`, every
+CPU-resident K-quant tensor is repacked to row-interleaved and then **run on CPU**
+even where GPU would be faster → lower prompt-processing speed. For a MoE that
+keeps experts on CPU (`--cpu-moe`), this is a real hit. Prefer IQK/R4/Trellis/
+MXFP4 variants, or skip `-rtr` and place tensors with `--fit`/`-ot`.
+
+## 4. Server-specific benefits
+
+### Dankserver — NVIDIA RTX 3090 (24 GB), 62 GB RAM — 35B MoE (Ornith-1.5-35B-A3B CRACK)
+- The winning lever for a **low-VRAM CUDA MoE** is **expert-aware offload + fused
+  MoE**, which is ik_llama.cpp's core strength:
+  - `-ooae/--offload-only-active-experts` — with ~8 active experts this cuts
+    RAM→VRAM traffic dramatically vs shipping all experts.
+  - `-fmoe/--fused-moe`, GDN fusion (the CRACK model is a hybrid GatedDeltaNet +
+    MoE), `--fit`, `-ot/--override-tensor`, `--cpu-moe`.
+- **Quant pick for the 3090:** `Q4_K_M`/`Q5_K_M` on mainline, or an **IQK/R4 /
+  Trellis / MXFP4** variant on ik_llama.cpp that has a **CUDA** path. Avoid raw
+  K-quant + `-rtr`. Enable FlashAttention; `GGML_CUDA_FORCE_MMQ=1`;
+  `ngram-mod` drafts (MoEs want long drafts).
+- ⚠️ **Expert-count discrepancy:** the CRACK GGUF README says **256 experts / 8
+  active**; the fleet brief cites ~36. Verify `n_expert` / `n_expert_used` from
+  the GGUF before finalizing the offload recipe — both point at expert-aware
+  offload being the right tool.
+- **Mainline only?** Fine: `--cpu-moe` + `Q4_K_M` + FlashAttention + `ngram-mod`.
+- **ROCmFPX?** No — ROCm-only formats, CUDA disabled on the builds.
+
+### Dankclaw — AMD gfx1151 Strix Halo (64 GiB unified), ROCm 7.2.4 — Qwen3.8-27B ROCmFP4 STRIX MTP
+- **ROCmFPX is the natural fit** and stays the choice: native IU4 (ROCmI4),
+  ROCmFP4 weight formats, MTP draft head, TurboQuant KV, 262K / 1 slot. The
+  production `build-promptforge` `llama-server` already runs there.
+- **ik_llama.cpp on Strix?** ROCm is its **weakest** backend (gfx1151 not
+  validated, k-quant-no-hip-row-interleave), and we'd lose the bespoke MTP
+  draft-head speedup. Higher-risk change; only with a small soak-test first.
+- **Mainline on Strix** works for correctness with standard quants but loses FPX,
+  IU4, TurboQuant, and MTEA.
+
+## 6. KV cache quant table
+
+Bit-density-ordered comparison of everything usable in the **KV cache** (`-ctk`
+key / `-ctv` value, and draft `-ctkd`/`-ctvd`). Order is high → low bits-per-
+value. Columns mark **origin** (`*` = mainline-provided) and whether the type is
+**KV-native** (designed for the cache) or **weight-family reused for KV** (a
+model-weight quant the runtime also accepts in `-ctk`/`-ctv`).
+
+**How bpw is read here.** `bpw = ggml_type_size × 8 / QK`, computed from the
+exact block structs in `ggml.h` / `ggml-common.h` (`QK_K = 256`, fp16 scale).
+This is the true in-RAM cost per value, so it is what sets KV-cache size and
+quantize/dequant throughput. A footnote marks the few types whose **nominal**
+figure (older 1-byte-scale layout / data-bits-only) diverges from this exact
+number; the table uses the exact number throughout.
+
+### 6a. All KV-usable types, by bit density (interleaved)
+
+| bpw | Type | Origin | KV-native | Notes |
+|-----|------|--------|-----------|-------|
+| 32.0 | f32 | mainline * | yes | full precision |
+| 16.0 | f16 | mainline * | yes | default |
+| 16.0 | bf16 | mainline * | yes | |
+| 9.25 | q8_K | mainline * | weight | rare in KV; per-16-block sums; ik Q8_KV is the fast KV version |
+| 8.13 | **q8_0** | mainline * | yes | |
+| 8.06 | **q8_KV** | ik * | yes | the dedicated **fast KV cache** type (`-ctk q8_KV`); Q8_0-shaped, 1 shared scale |
+| 6.63 | iq6_K | ik | weight→KV | IQK family |
+| 6.56 | q6_K | mainline * | weight→KV | |
+| 6.00 | **q5_1** | mainline * | yes | (nominal 5.06) |
+| 5.50 | **q5_0** | mainline * | yes | (nominal 5.06) |
+| 5.50 | q5_K | mainline * | weight→KV | |
+| 5.50 | iq5_K | ik | weight→KV | IQK family |
+| 4.50 | **q4_0** | mainline * | yes | (nominal 4.25) |
+| 4.50 | **q4_1** | mainline * | yes | (nominal 4.25) |
+| 4.50 | **iq4_nl** | mainline * | yes | (nominal 4.0) |
+| 4.50 | iq4_K | ik | weight→KV | IQK family |
+| 4.5 | **Turbo4_0** | ROCmFPX * | yes | TurboQuant; ~4.5 |
+| 4.25 | iq4_ks | ik | weight→KV | IQK subsample |
+| 4.25 | iq4_xs | mainline * | weight→KV | popular *weight* format, also usable in KV |
+| 4.25 | mxfp4 | ik * | weight→KV | |
+| 3.5 | **Turbo3_0** | ROCmFPX * | yes | TurboQuant; ~3.5 |
+| 3.44 | iq3_K | ik | weight→KV | IQK family |
+| 3.44 | iq3_s | mainline * | weight→KV | |
+| 2.69 | iq2_kl | ik | weight→KV | IQK subsample |
+| 2.38 | iq2_K | ik | weight→KV | IQK family |
+| 2.19 | iq2_ks | ik | weight→KV | IQK subsample |
+| 2.13 | iq2_kt | ik * | weight→KV | **Trellis** (lattice) |
+| 1.75 | iq1_kt | ik * | weight→KV | **Trellis** (lattice) |
+
+### 6b. Origin legend
+
+- **Mainline KV-native** (`-ctk`/`-ctv` = exactly these 9, from `common/arg.cpp`
+  `kv_cache_types`): `f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1`. Marked
+  with `*` and `yes` in the KV-native column.
+- **ROCmFPX KV-native**: `turbo3_0, turbo4_0` — TurboQuant's dedicated KV formats
+  (add a third `V` type for values only). Marked `*` here.
+- **ik additions**: `q8_KV` (fast KV, dedicated), plus the weight-family quants
+  ik lets you reuse in `-ctk`/`-ctv` — the **IQK** family (`iq*_K`, plus the
+  `IKS`/`IKL` subsamples) and the **Trellis** lattice family (`iq*_KT`), and
+  `mxfp4`. ik's `-ctk` accepts a broader set than mainline's.
+
+### 6c. Trellis, IQK, MXFP4 — where they sit
+
+- **Trellis (`IQ*_KT`)** — ik's lattice-code quants, mirror the `IQ1..IQ4` bit
+  ladder: `iq1_kt` 1.75 → `iq2_kt` 2.13 → `iq3_kt` 3.13 → `iq4_kt` 4.00. Highest
+  quality per bit of the family; lowest-density entries in the whole table.
+- **IQK (`IQ*_K`)** — ik's improved K-quants: `iq2_k` 2.38, `iq3_k` 3.44,
+  `iq4_k` 4.50, `iq5_k` 5.50, `iq6_k` 6.63, plus subsamples `iq2_kl` 2.69 /
+  `iq2_ks` 2.19 / `iq4_ks` 4.25. They sit alongside the mainline K-family
+  (iq2_k ≈ iq2_s, iq4_k ≈ q4_K) but with ik's row-interleave + fused GEMM.
+- **MXFP4** (`mxfp4`, 4.25) — the mixed-precision FP4 cache quant; ik's entry.
+
+### 6d. Hadamard is a transform, not a row
+
+`--k-cache-hadamard` / `--v-cache-hadamard` (ik) is a **structured transform
+applied on top of a base KV quant**, not its own bit-density row. It spreads
+quantization error, improving accuracy for the underlying type; ik's guidance is
+**best below Q6_0**. Row-interleaving (the `R4`/`R8` variants) is the same idea —
+a packing transform on the K-family — and is likewise a modifier, not a quant.
+
+### 6e. Server notes
+
+- **Dankclaw (production, ROCmFP4 STRIX MTP):** KV stays on `turbo4_0` /
+  `turbo3_0` (TurboQuant) — the runtime's own formats. ik's IQK/Trellis path on
+  Strix is higher-risk (see §4).
+- **Dankserver (3090 MoE):** `-ctk q8_KV` or `q4_K`/`q5_K` (mainline) with
+  FlashAttention + KV-quant; on ik, `iq4_k`/`iq4_kt`/`iq2_k` if a CUDA path is
+  confirmed for that type. Keep FlashAttention and the KV quant version in lockstep
+  — they are coupled (a quant change that drops FA off the path changes KV math).
+
+## 5. Recommendations for our fleet
+
+- **Dankclaw:** keep **charlie12345/ROCmFPX** (already production-tuned for
+  gfx1151). If you want more quant variety on Strix, explore the ROCmFP2 (`Q2_0_ROCMFPX`,
+  2.5 bpw, S40 codebook) preview — but do not port to ik_llama.cpp as primary.
+- **Dankserver:** **try ik_llama.cpp** (CUDA-90 + AVX-512 build) for fused-MoE +
+  `offload-only-active-experts`, staying off raw K-quant `-rtr`. If that is too
+  much surface, stay on **mainline** with `--cpu-moe` + `Q4_K_M` + FlashAttention
+  + `ngram-mod`. Neither box needs ROCmFP4.
+- **Common:** pin a commit per fleet line; re-test after any quant/KV/context
+  change (FlashAttention + KV-quant are coupled).
+
+## See Also
+- [[Llama Cpp Runtimes]] — positioning + quantization overview
+- [[dealign-ornith-crack]] — the CRACK MoE on Dankserver
+- [[kingjones-strix-mtp]] — the ROCmFP4 model on Dankclaw
+- [[Quantization by Hardware Architecture]] — quant vs hardware
+
+## Citations
+- Mainline quant + backends — https://github.com/ggml-org/llama.cpp ·
+  tools/quantize/README.md · docs/speculative.md
+- ROCmFPX enum (ggml_type 100–108) — https://github.com/charlie12345/ROCmFPX/blob/main/ggml/include/ggml.h
+- ik_llama.cpp quant/MoE/FlashMLA/backends — https://github.com/ikawrakow/ik_llama.cpp (README, docs/parameters.md, docs/build.md) · DeepWiki · discussions#562
+- ik_llama.cpp `-rtr` K-quant trap & backend-bandwidth caveat — https://github.com/ikawrakow/ik_llama.cpp/blob/main/README.md

docs/Research/Llama Cpp Runtimes.md

@@ -0,0 +1,141 @@
+---
+type: research
+title: Llama.cpp Runtimes
+description: Compared mainline llama.cpp, charlie12345/ROCmFPX, and ik_llama.cpp — positioning, quantization formats, code-level differences, and fit for Dankserver (RTX 3090) and Dankclaw (Strix Halo).
+tags:
+  - llama.cpp
+  - quantization
+  - ROCmFPX
+  - ik_llama
+  - inference
+  - amd
+  - nvidia
+timestamp: 2026-09-01
+---
+
+# Llama.cpp Runtimes
+
+Comparison of three `llama.cpp`-derived inference runtimes used (or usable) on
+the two homelab boxes: **mainline** `ggml-org/llama.cpp`,
+**charlie12345/ROCmFPX** (what Dankclaw runs), and **ik_llama.cpp**
+(ikawrakow). The goal is to know, for each server, which quant formats are on
+the table and which runtime best fits the hardware.
+
+Full code-level quantification is in [[Llama Cpp Code Diff]]. This page is the
+positioning + quantization summary.
+
+## Which runtime is which
+
+| Runtime | Repo | What it is | Primary backends | For our boxes |
+|---------|------|-----------|------------------|---------------|
+| Mainline | [`ggml-org/llama.cpp`](https://github.com/ggml-org/llama.cpp) | Canonical, general-purpose, one codebase for every backend | CPU, CUDA, Vulkan, Metal, SYCL, HIP/ROCm, CANN, OpenVINO | Baseline both boxes; the portable default |
+| charlie12345/ROCmFPX | [`charlie12345/ROCmFPX`](https://github.com/charlie12345/ROCmFPX) (+ reference fork [`rocmfp4-llama`](https://github.com/charlie12345/rocmfp4-llama), clean pkg [`rocmfp4`](https://github.com/charlie12345/rocmfp4)) | Experimental AMD-focused GGUF *weight* quant family layered on llama.cpp | HIP/ROCm + Vulkan (CUDA **disabled**), CPU reference | **Dankclaw** (native Strix Halo) |
+| ik_llama.cpp | [`ikawrakow/ik_llama.cpp`](https://github.com/ikawrakow/ik_llama.cpp) | Fork focused on SOTA quants + CPU / hybrid GPU-CPU performance | CPU (AVX-512/VNNI), CUDA (Turing+), Vulkan, ROCm (weak) | **Dankserver** candidate (3090 + MoE) |
+
+Key lineage notes:
+- Mainline moved from `ggerganov/llama.cpp` to the `ggml-org` org. Rolling
+  commit hashes (the fleet pins `b10121`) plus newer `vX.Y.Z` tags. Pin a
+  commit for reproducibility.
+- ROCmFPX is an *experimental feature family* on the canonical `main` branch;
+  three GitHub surfaces exist (reference fork, clean patch, pre-patched tree).
+  It is **AMD-only** — CUDA is compiled out.
+- ik_llama.cpp's core is an **August-2024 llama.cpp snapshot** maintained with
+  a rolling set of PRs (MLA, fused MoE, row-interleaved quants, MTP, DFlash all
+  appeared here first, later partially merged upstream). The feature surface is
+  current; the architecture baseline is ~1.5 years behind mainline.
+
+## Quantization summary
+
+### Mainline (what the fleet already ships)
+- **Base:** FP32 / FP16 / BF16.
+- **K-quants** (modern default, two-level block + double-quant): `Q2_K` …
+  `Q6_K`, with `_S`/`_M`/`_L` precision mixes. `Q4_K_M` is the popular default;
+  `Q6_K` is "almost lossless."
+- **i-quants** (importance-matrix, non-linear, best quality/size at low bits but
+  slower decode): `IQ4_NL`, `IQ4_XS`, `IQ3_*`, `IQ2_*`. Row-interleaved 3-bit
+  family (IQ3_XXS/S/XS) reduces the quality penalty of whole-row quant.
+- **Legacy:** `Q*_0`/`Q*_1` (fast, simple), ternary `TQ1_0`.
+- **KV cache quant:** `f32/f16/bf16/q8_0/q4_0/iq4_nl/q5_0/q5_1`; quantizing KV
+  requires FlashAttention (`-fa on`).
+- Speculative decoding native (MTP, EAGLE3, DFlash, DSpark, `ngram-mod`).
+
+### ROCmFPX (Dankclaw's AMD weight formats)
+- Integer GGUF **weight** formats only this fork reads: **ROCmFP2/3/4/6/8** and
+  **ROCmI4 (W4A4 IU4 WMMA)**. These are custom `ggml` tensor types
+  (**`ggml_type` 100–108**) with matching GGUF **`ftype`** (ROCmFP4 family
+  `100–106`; ROCmFPX family `110–113`; `ROCmI4` = `118`). Stock llama.cpp,
+  Ollama, and LM Studio **cannot load them**.
+- Flagship **ROCmFP4** (`block_rocmfp4`): 32 weights/block, two E4M3 scale
+  bytes, ~18 bytes/block, ~4.5 bpw, E2M1 nibbles + E4M3 codebook up to 10
+  units. Variants: `STRIX` / `STRIX_LEAN`, `FAST`, `COHERENT`, plus `*_AGENT`
+  / `*_STRIX_MTP` draft-head presets.
+- **TurboQuant KV:** `Turbo3_0` (3.5 bpw) / `Turbo4_0` (4.5 bpw) + `TurboQuant V`
+  (f16 K) on the production model.
+- **ROCmI4 (W4A4):** the gfx1151 native IU4 4-bit WMMA path (`-DGGML_HIP_ROCMI4_W4A4=ON`);
+  lossy activation quant, exact int8 MMQ.
+
+### ik_llama.cpp (SOTA quants + CPU/hybrid)
+- **IQ\*** non-linear family; `IQ4_NL` gives lower perplexity than mainline
+  `Q4_K_M` at the same size. Improved **K-quants** and the signature
+  **row-interleaved IQK / R4 / R8** packing (activates AVX-512/VNNI/NEON
+  matmul paths that vanilla llama.cpp leaves cold).
+- **Trellis** quants (`IQ*_KT`), **MXFP4**, **Q8_KV** fast KV type, first-class
+  **Bitnet**.
+- ⚠️ **K-quant / `-rtr` trap (critical for the 3090):** K-quants have **no CUDA
+  row-interleaved path**; `-rtr` (run-time repack) forces CPU-resident K-quant
+  tensors to CPU matmul even when GPU would be faster. Prefer IQK/R4/Trellis/
+  MXFP4 variants, or skip `-rtr` and let `--fit`/`-ot` place experts.
+
+## Fit for our servers
+
+### Dankserver — NVIDIA RTX 3090 (24 GB VRAM), 62 GB RAM, CUDA
+Serves **Ornith-1.5-35B-A3B CRACK**, a 35B **MoE** (~36 experts; the CRACK
+GGUF README lists 256/8-active — verify `n_expert` from the GGUF).
+- **Mainline** is the natural baseline: `Q4_K_M`/`Q5_K_M`, FlashAttention to
+  shrink KV, `GGML_CUDA_FORCE_MMQ=1` (default on RTX 30xx, lower VRAM),
+  `--cpu-moe` to spill experts to the 62 GB RAM, and `ngram-mod` drafts (MoEs
+  want long drafts).
+- **ik_llama.cpp** is the more compelling *upgrade* here: fused MoE (`-fmoe`),
+  **offload-only-active-experts** (`-ooae`) which — with only ~8 active experts —
+  sharply cuts RAM→VRAM traffic, `--fit`, smart-expert-reduction (`-ser`), GDN
+  fusion for the SSM path, and MTP. Requires the CUDA-90 + AVX-512 build and care
+  with the `-rtr` K-quant trap.
+- **ROCmFPX** does **not** help a CUDA box (ROCm-only formats, CUDA disabled).
+  The 3090's lever is CUDA quant + FastMTP, not ROCmFPX.
+
+### Dankclaw — AMD gfx1151 Strix Halo (64 GiB unified VRAM), ROCm 7.2.4
+Serves **kingjones Qwen3.8-27B ROCmFP4 STRIX MTP** (the production build).
+- **ROCmFPX is the native fit**: IU4/MTP/FPX, HIP+Vulkan kernels, TurboQuant KV,
+  262K ctx / 1 slot. Measured 8K MTP ~30.3 tok/s (2.83× vs `Q4_K_M`).
+- **ik_llama.cpp** — ROCm is its **weakest** backend (gfx1151 not validated) and
+  the k-quant-no-hip-row-interleave issue bites there too. Keep ROCmFPX.
+- **Mainline** ROCm build works for correctness (standard quants) but loses FPX,
+  IU4, TurboQuant, MTEA, and agent presets.
+
+## When to reach for which
+- **Stay mainline** if you want one portable upstream for both boxes, the widest
+  GGUF ecosystem, and standard FlashAttention/MTP. Best fit for **Dankserver**
+  if you don't want the ik_llama.cpp build surface.
+- **Use ik_llama.cpp** for Dankserver when the MoE offload and fused-MoE wins
+  justify an extra build (watch `-rtr` + K-quants).
+- **Use ROCmFPX** for Dankclaw — it is purpose-built for gfx1151.
+- **ROCmFPX** is for the AMD fleet; **CUDA/FastMTP** for the NVIDIA box. Do not
+  port ROCmFP4 to the 3090 via Vulkan as a primary path.
+
+## See Also
+- [[Llama Cpp Code Diff]] — the code-level quantization diff + comparison table
+- [[dealign-ornith-crack]] — the CRACK model on Dankserver
+- [[kingjones-strix-mtp]] — the ROCmFP4 model on Dankclaw
+- [[Quantization by Hardware Architecture]] — which quant fits which GPU/CPU
+- [[SGLang]] — the structured-generation serving engine (the other lane)
+
+## Citations
+- Mainline — https://github.com/ggml-org/llama.cpp
+- charlie12345/ROCmFPX — https://github.com/charlie12345/ROCmFPX ·
+  rocmfp4-llama — https://github.com/charlie12345/rocmfp4-llama · rocmfp4 —
+  https://github.com/charlie12345/rocmfp4
+- ik_llama.cpp — https://github.com/ikawrakow/ik_llama.cpp · docs ·
+  https://ikawrakow-ik_llama-cpp.mintlify.app/
+- ROCmFPX quant/enum/serve detail —
+  https://github.com/charlie12345/ROCmFPX/blob/main/ggml/include/ggml.h
+- SGLang — https://github.com/sgl-project/sglang · https://docs.sglang.io/

docs/Research/SGLang.md

@@ -0,0 +1,154 @@
+---
+type: research
+title: SGLang
+description: What SGLang is, how it works (RadixAttention, paged KV, overlap scheduler, structured output), how it compares to llama.cpp and vLLM, and whether it fits Dankserver (RTX 3090) and Dankclaw (Strix Halo).
+tags:
+  - sglang
+  - serving
+  - inference
+  - structured-output
+  - radix-attention
+  - llama.cpp
+timestamp: 2026-09-01
+---
+
+# SGLang
+
+SGLang (for **S**tructured **G**eneration **Lang**uage) is a high-throughput
+serving engine for large (and vision-language) models, from the LMSYS / UC
+Berkeley team, published at NeurIPS 2024. Today it is one of the two dominant
+open-source serving engines, alongside vLLM. People talk about it because it
+makes "call the model, branch, force the output into JSON" programs *fast*.
+
+## What it is — in one paragraph
+
+You serve an LLM by writing a small program: call the model, then based on its
+output call it again, branch into parallel attempts, and force the answer into a
+JSON schema. SGLang co-designs a Python-embedded language with a runtime that can
+*see* the program's structure, then exploits it: it caches and reuses the model's
+"working memory" (the KV cache) across requests that share a prompt prefix,
+batches new requests into running generations without waiting for a batch to
+drain, and forces tightly-structured output (JSON / regex / tool tags) by masking
+illegal tokens one step at a time — or several at once. Reported payoff:
+**up to 6.4× higher throughput** on structured / agentic / RAG workloads vs vLLM,
+TGI, and similar.
+
+## Core architecture
+
+Two halves that can also run independently:
+- **Frontend** — a Python DSL (`@sgl.function`, `gen`/`select`/`fork`/`join`/
+  `extend`) for multi-call LLM programs with branching, parallel sampling, tool
+  calls, and structured output.
+- **Backend runtime** — a high-throughput server (`python -m sglang.launch_server`)
+  or an in-process `Engine`. OpenAI-compatible `/v1`, so RadixAttention and
+  structured decoding work with a vanilla OpenAI client.
+
+Building blocks:
+- **z-normalization** — pre-compile per-request sampling metadata into contiguous
+  tensors so the GPU sampler does one bulk copy per batch per step (kills the
+  CPU scheduler bottleneck).
+- **PagedTensor** — the KV cache is paged (fixed-size pages) and laid out
+  non-contiguously in a global pool; mirrors vLLM's PagedAttention.
+- **RadixAttention** — a radix tree of shared token prefixes; on a new request it
+  walks to the longest cached prefix, reuses those KV blocks, and only prefills
+  the divergent suffix. Payoff is compute + throughput for long shared prefixes
+  (RAG, agents), not raw memory.
+- **Overlap scheduler** — runs the CPU prep for step t+1 *under* the GPU forward
+  of step t, so kernels launch back-to-back with no idle bubble (plus CUDA
+  graphs + torch.compile).
+- **Structured / output-guided generation** — compiles the constraint (JSON
+  schema / regex / EBNF) into a finite-state machine over the vocabulary and masks
+  disallowed logits; **jump-forward** decoding emits forced token runs in one step.
+  Backends: XGrammar (default), Outlines.
+
+## SGLang vs llama.cpp vs vLLM
+
+| Dimension | SGLang | llama.cpp |
+|-----------|--------|-----------|
+| Runtime | Python + CUDA/C++ kernels | C/C++, no Python required |
+| GPU requirement | GPU required, **no CPU path** | CPU-only / GPU-only / hybrid offload |
+| Model format | HF safetensors; AWQ/GPTQ/FP8 on the fly; **GGUF via llama.cpp backend** | **GGUF only** (convert first) |
+| KV reuse | RadixAttention (always-on radix tree) | basic prompt caching, no radix sharing |
+| Structured output | first-class (XGrammar, jump-forward) | narrower |
+| Concurrency | continuous batching + overlap scheduler | single-node, lower concurrency |
+| Multi-GPU | native tensor parallelism | layer-split, more manual |
+| Setup | `pip install sglang[all]`, venv, CUDA match | static binary / Docker, minutes |
+| License | Apache 2.0 | MIT |
+
+**Pick SGLang** when you need many concurrent users, structured / JSON / tool-call
+output, or repeated long prefixes (RAG, agents) where RadixAttention pays off, and
+you own a GPU + Python/CUDA dependency chain. **Pick llama.cpp** for the fastest
+download-to-endpoint path, CPU/edge, GGUF libraries, and quantization breadth.
+**Pick vLLM** for the broadest model-hardware coverage and mature multi-GPU.
+
+The paper's headline 6.4× / 5× numbers are vs **vLLM v0.1** (no prefix caching);
+the gap narrows vs modern vLLM — benchmark on your own traffic.
+
+## Deployment model & gotchas
+- **Pluggable backends:** default vLLM engine, SGLang's own `sglang` engine, or a
+  recent beta **`llama.cpp` backend** that serves GGUF and reuses `llama-server`.
+- **VRAM headroom runs higher** than llama.cpp (pre-allocated KV pool + radix
+  tree). Size with `--mem-fraction-static`, `--context-length`,
+  `--chunked-prefill-size`. OOM → lower context / fraction or a smaller quant.
+- **Model-format lock-in:** wants safetensors + AWQ/GPTQ/FP8 (or GGUF via the
+  llama.cpp backend). The current CPU-Q6_K_P and ROCmFP4-MTP models are **GGUF**.
+- **AMD / ROCm: yes, officially.** Prebuilt `lmsysorg/sglang` ROCm Docker image or
+  `pip`. Quant on AMD: FP8 (Aiter/Triton), AWQ (Triton), MXFP4 (CDNA3/4),
+  `petit_nvfp4`. Marlin-based methods are **out** on AMD. Kernel work targets
+  **datacenter Instinct (CDNA2/3/4)** — consumer/client parts like **Strix Halo
+  (gfx1151)** are supported at the ROCm level but are not the kernel-tuning target.
+- **Prefix diversity kills RadixAttention** (per-user/personalized prompts → tree
+  degenerates). Single-replica trees need prefix-aware routing across a fleet
+  (`sglang-router --policy cache_aware`). Fast-moving API — pin versions.
+
+## Relevance to our servers
+
+### Dankserver — NVIDIA RTX 3090 (24 GB) + 62 GB RAM — 35B MoE
+A **genuine fit candidate (with a migration step)**. The 3090 is CUDA-native.
+A 35B MoE (~3B active) is ideal for continuous batching + structured generation,
+and 24 GB can hold a quantized 35B MoE (FP8/AWQ safetensors ~18–20 GB) with room
+for KV. MoE expert parallelism is supported.
+- **What changes:** the current model is a **CPU GGUF (Q6_K_P)**; SGLang wants
+  safetensors (FP8/AWQ) for CUDA, OR use SGLang's **`llama.cpp` backend** to serve
+  the existing GGUF under SGLang's frontend. Check for an AWQ/FP8 variant of the
+  HauhauCS checkpoint on Hugging Face.
+- **Where it wins:** repeated structured/JSON/tool calls or a long shared system
+  prompt (RAG) → RadixAttention + jump-forward give big gains vs the current
+  sequential CPU pipeline. Simple low-concurrency chat → the operational weight
+  may not pay off.
+
+### Dankclaw — AMD gfx1151 Strix Halo (64 GiB unified) — Qwen3.8-27B ROCmFP4 MTP
+A **niche / experimental fit, not a drop-in replacement**. 64 GiB is generous and
+SGLang-on-ROCm is supported (Docker/pip), but:
+- Our model is a **custom ROCmFP4/MTP GGUF**. SGLang's primary path wants
+  safetensors + an AMD quant (FP8/AWQ); MXFP4 needs CDNA3/4 and Strix Halo (CDNA
+  3.5 *client*) is **not** an SGLang/AMD kernel target. Quant availability for this
+  exact MTP model is unverified, and we'd lose the bespoke MTP draft-head speedup.
+- **Best-fit path:** SGLang's **`llama.cpp` backend** on top of our existing ROCm
+  llama.cpp / GGUF — gain the structured-output DSL, RadixAttention, and scheduler
+  without rebuilding the kernel stack. Or test the native `lmsysorg/sglang` ROCm
+  image as a proof-of-concept. Treat it as an enhancement/experiment, not a
+  production swap, until a quant for the MTP model exists and perf is measured on
+  gfx1151.
+
+## Tradeoffs at a glance
+- Higher baseline VRAM than llama.cpp (KV pool + radix tree).
+- GPU/accelerator required — no pure-CPU path (llama.cpp is the only option of the
+  trio for a CPU-only box).
+- Model-format lock-in (safetensors/AWQ/GPTQ/FP8, or GGUF via llama.cpp backend).
+- Marlin-based quant out on AMD; Strix Halo is not an MXFP4 target.
+- Prefix diversity can nullify RadixAttention; pin versions; watch cache-report.
+
+## See Also
+- [[Llama Cpp Runtimes]] / [[Llama Cpp Code Diff]] — the llama.cpp side of the trio
+- [[dealign-ornith-crack]] — the CRACK MoE on Dankserver
+- [[kingjones-strix-mtp]] — the ROCmFP4 MTP model on Dankclaw
+- [[Quantization by Hardware Architecture]]
+
+## Citations
+- SGLang paper (arXiv:2312.07104) — https://arxiv.org/abs/2312.07104 · NeurIPS 2024
+- Repo — https://github.com/sgl-project/sglang · docs — https://docs.sglang.io/
+- AMD / ROCm — https://docs.sglang.io/docs/hardware-platforms/amd_gpu ·
+  Structured Outputs — https://docs.sglang.io/docs/advanced_features/structured_outputs
+- lmsysorg/sglang ROCm images — https://hub.docker.com/r/lmsysorg/sglang
+- SGLang vs llama.cpp — https://markaicode.com/vs/sglang-vs-llamacpp/

docs/Research/index.md

@@ -0,0 +1,24 @@
+# Research
+
+Subagent research briefs for the two hosts. Each page is a focused research
+findings doc; the project root index ([docs/index.md](../index.md)) links back.
+
+## LLM Inference
+
+- [[Llama Cpp Runtimes]] — mainline vs charlie12345/ROCmFPX vs ik_llama.cpp: positioning, quantization, server fit
+- [[Llama Cpp Code Diff]] — code-level quantization diff + comparison table + server recommendations
+- [[SGLang]] — the structured-generation serving engine (llama.cpp/vLLM comparison, our- servers fit)
+
+## Image / Media
+
+- [[ComfyUI ROCm and Vulkan]] — live-verified GPU path for ComfyUI on Strix Halo
+- [[Lemonade and sd.cpp]] — the ggml-family image-generation alternative
+- [[Jellyfin]] — media server research brief
+
+## Backups
+
+- [[Backblaze Integration]] — off-site backup plan (future)
+
+## Legacy
+
+- [[Service Research Brief]] — consolidated subagent research behind these pages

docs/index.md

@@ -27,9 +27,13 @@ covers both workloads).
 
 - [[Network Architecture]] — Caddy reverse proxy, hostname routing, Gluetun VPN path
 - [[Storage Policy]] — bind mounts, data locations, ownership
-- [[Deployment Flow]] — git-based deploy, target map, health checks
+- [[Deployment Flow]] — service-scoped git deploy, target map, and health checks
 - [[Testing Policy]] — the test seams and how to run them
 
+## Dankserver Storage
+
+- [[Entropy Volume]] — command-line unlock and mount helper
+
 ## Dankserver Services
 
 - [[RomM]] — retro game manager
@@ -51,12 +55,14 @@ covers both workloads).
 ## Dankclaw Services
 
 - [[SearXNG]] — privacy metasearch (VPN-routed)
+- [[SearXNG Providers]] — unblocking Google & other upstream engines behind the VPN
 - [[Gluetun]] — the VPN tunnel container
 - [[Valkey]] — SearXNG cache and limiter store
 - [[ComfyUI]] — node-based AI image generation (CPU)
 - [[OpenWebUI]] — chat UI for local LLMs
 - [[Hermes]] — the Nous Research agent
 - [[Hermes-Dashboard]] — the Hermes web management UI (config, keys, sessions)
+- [[Hermes-WebUI]] — the Hermex server (loopback API for the iPhone app, SSH-tunneled)
 - [[Gitea]] — Git hosting (migrated)
 - [[Postgres]] — Gitea database backend
 
@@ -83,6 +89,9 @@ covers both workloads).
 - [[Service Research Brief]] — consolidated subagent research behind these pages
 - [[ComfyUI ROCm and Vulkan]] — live-verified GPU path for ComfyUI on Strix Halo
 - [[Lemonade and sd.cpp]] — the ggml-family image-generation alternative
+- [[Llama Cpp Runtimes]] — mainline vs charlie12345/ROCmFPX vs ik_llama.cpp: quantization + server fit
+- [[Llama Cpp Code Diff]] — code-level quantization diff of the three llama.cpp runtimes
+- [[SGLang]] — the structured-generation serving engine (llama.cpp/vLLM comparison + our fit)
 - [[Backblaze Integration]] — off-site backup plan (future)
 - [[Jellyfin]] — media server research brief
 - [[ARR-Stack]] — automated download stack research brief
@@ -96,18 +105,23 @@ The LLM inference fleet on the two hosts. Full index:
 
 - [[deployment]] — unified deploy and update process for both servers
 - [[frank-redo-2026-08-17]] — the 2026-08-17 topology change
+- [[kingjones-strix-mtp]] — the active kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model on Dankclaw (restored 2026-09-01)
+- [[kingjones-q8agent-rocmfpx]] — the retired Q8_0-AGENT investigation and `set_rows` failure
+- [[eemin-rocmi4-mtp]] — the former caf-eemin Qwen3.8-27B ROCmI4 embedded-MTP model on Dankclaw
 - [[ciru-activefpx-promptforge]] — the RETIRED v3 CIRU PromptForge runtime on Dankclaw (superseded 2026-08-24)
 - [[kingjones-strix-mtp]] — the kingjones Qwen3.8-27B ROCmFP4 STRIX MTP model on Dankclaw (active)
 - [[dealign-ornith-crack]] — the dealign Ornith-1.5-35B-A3B CRACK model on Dankserver (active)
+- [[ikllama-ornith-dankserver]] — the ik_llama.cpp runtime for frank-default-small (q8_0 + Hadamard KV cache, split-mode none; 2026-08-26)
 - [[kairic-edge]] — the retired Kairic Edge IU4 experiment (2026-08-22)
 - [[dflash2-large-vulkan]] — the retired DFlash2 Vulkan deployment (2026-08-21)
-- [[Dankserver System Specs]] — the upstream llama.cpp box hardware
+- [[Dankserver System Specs]] — the RTX 3090 box hardware
 - [[Dankclaw System Specs]] — the CIRU ROCmFPX box hardware
 - [[Dankclaw Improvements]] — host-level changes on the CIRU box
 
 ### Benchmarks and Sizing
 
 - [[frank-default-large-benchmark]] — backend/spec benchmark behind the old head
+- [[ubatch-benchmark-large]] — active-model llama-bench ubatch sweep plan and results
 - [[frank-default-small-benchmark]] — the small-model benchmark suite
 - [[Running Qwen3.6 35B A3B FP8 on Dankserver]] — Qwen3.6-35B-A3B hardware notes
 - [[Running Qwen3.6 27B FP8 on Dankclaw]] — Qwen3.6-27B sizing on the iGPU
@@ -146,7 +160,12 @@ The LLM inference fleet on the two hosts. Full index:
 - [[Sample Script Set]] — reference scripts for the PP pair
 - [[vLLM Pipeline Parallelism]] — vLLM's PP implementation
 
+## Update Log
+
+- [[log]] — chronological record of knowledge-base changes
+
 ## See Also
 
 - [AGENTS.md](../AGENTS.md) — repository rules
 - [docs/LLM/index.md](LLM/index.md) — the LLM fleet knowledge base
+- [dankware Configuration Knowledgebase](~/.pi/agent/docs/index.md) — the "root" level system knowlegebase

docs/log.md

@@ -0,0 +1,39 @@
+---
+type: reference
+title: Bundle Change Log
+description: Chronological record of meaningful changes to the serverAppConfigs knowledge base.
+tags:
+  - changelog
+  - documentation
+  - maintenance
+timestamp: 2026-09-14
+---
+
+# Bundle Change Log
+
+## 2026-09-14
+- **Creation**: [[Hermes-WebUI]] — new `hermes-webui` compose service on Dankclaw (the Hermex server): runs `hermes-webui`'s `bootstrap.py --foreground` in the agent image, shares `/home/dank/hermes-data` with the gateway (identical rootless Podman uid mapping), publishes `127.0.0.1:8787` only, password-gated via `HERMES_WEBUI_PASSWORD` in the host `.env`.
+- `/dankclaw/compose.yaml`, `/dankclaw/.env.example` — service definition and password placeholder; container-internal bind is `0.0.0.0` (Podman's port forwarder dials the bridge IP) while the host-side publish stays loopback so the SSH tunnel is the sole ingress.
+- `/scripts/deploy-plan.py`, `/scripts/deploy.sh`, `/scripts/deploy-remote.sh` — `hermes-webui` registered as an isolated Dankclaw service in all three inventories; planner narrows edits to its compose block only.
+- `/tests/hermes_webui.bats` — new seam: foreground entrypoint, loopback publish + auth, shared volume, `/health` healthcheck, env placeholder, planner narrowing (hermetic fixture repo), deployer dispatch.
+- Remote access pattern documented: iPhone Termius local forward `127.0.0.1:8787 -> 127.0.0.1:8787` over the existing router port forward to Dankclaw `:22`; Hermex points at `http://localhost:8787`. No Tailscale, no Cloudflare.
+
+## 2026-08-26
+- `/dankclaw/Caddyfile` — Both Plex vhosts now proxy to PMS over a TLS upstream (`insecure_skip_verify`); PMS drops plain-HTTP connections from non-loopback sources, which produced empty-body 502s.
+- `/docs/Infrastructure/Caddy.md` — Documented the Plex TLS-upstream requirement and the `/web/` vs `/` distinction.
+- `/tests/caddyfile.bats` — Added a test pinning the TLS upstream transport on both Plex vhosts.
+- `/docs/Dankclaw/Gitea.md` — Documented migration of Gitea web proxy ownership to canonical Caddy and retirement of the standalone proxy.
+- `/docs/Dankclaw/Dankclaw Stack.md` — Updated boot-persistence and Caddy deployment descriptions for the two-unit Gitea stack.
+- `/docs/Architecture/Deployment Flow.md` — Replaced the split-proxy safety description with the canonical Caddy ownership model.
+- `/dankclaw/gitea/quadlet/gitea-caddy.container` — Retired the standalone Gitea Caddy Quadlet; deployment stages any live copy into home trash.
+- `/scripts/deploy-plan.py` — Routes retirement of the legacy proxy to the canonical Caddy and removes it from the service inventory.
+- `/scripts/deploy.sh` and `/scripts/deploy-remote.sh` — Removed standalone proxy dispatch and added idempotent retirement before canonical Caddy deployment.
+- `/tests/caddyfile.bats`, `/tests/deployment_scoping.bats`, `/tests/deployment_flow.bats`, `/tests/frank_large_config.bats` — Added migration coverage and corrected the existing ubatch harness assertion.
+- `/docs/index.md` — Bundle index remains the root navigation entry for the updated pages.
+- **Boot-persistence audit (post-reboot, 2026-08-26 evening):** after Frank's reboot, Caddy and Pi-hole were both down. Caddy ran as a bare compose container (`restart: unless-stopped` does not survive shutdown's explicit stop; rootless Podman has no boot-time auto-start for bare containers). Pi-hole's quadlet existed but crash-looped (restart counter 1645): a later-sorted sysctl drop-in (`99-rootless-ports.conf`) reset `net.ipv4.ip_unprivileged_port_start` to 80, so the rootless port-53 publish failed with `bind: permission denied`.
+- `/AGENTS.md` — new MANDATORY section "Boot Persistence & Self-Healing": every deployed, proven-working container must have a self-healing, boot-persistent quadlet unit (`WantedBy=default.target` + enabled + linger, `Restart=always`, repo-provenance `.container`), verified before a service is declared deployed.
+- `/dankclaw/caddy/quadlet/caddy.container`, `/dankclaw/caddy/env.example` — Caddy is now a boot-persistent quadlet (`caddy.service`): joins `dankclaw_default` so container-name upstreams keep resolving, `Pull=never` (local-only image), Porkbun credentials via `/home/dank/Containers/caddy/env` (deploy writes it from `.env`), bare-IP-catch-all health check. The caddy service was removed from `/dankclaw/compose.yaml`.
+- `/dankclaw/pihole/sysctl/99-dnsmasq.conf` — repo source of truth for `ip_unprivileged_port_start=53`; `deploy_pihole` installs it to `/etc/sysctl.d/`, stages conflicting later-sorted drop-ins in the home trash, and applies `sysctl --system` (the boot code path).
+- `/scripts/deploy-remote.sh` — `deploy_caddy` installs/enables/starts the quadlet (retiring the bare container; all state is in host volumes); `deploy_pihole` enforces the sysctl prerequisite.
+- `/tests/caddy_quadlet.bats` (new seam), `/tests/caddyfile.bats`, `/tests/deployment_flow.bats`, `/tests/pihole.bats` — quadlet boot-persistence, self-healing, network, env, and sysctl coverage; compose assertions repointed from the removed caddy service to the quadlet.
+- `/docs/Infrastructure/Caddy.md`, `/docs/Infrastructure/Pi-hole.md`, `/docs/Dankclaw/Dankclaw Stack.md` — quadlet runtime facts, the 2026-08-26 incident, and the sysctl-conflict failure mode.

scripts/bench-fast-greedy.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-# Bench the Kairic Edge "fast greedy (default)" profile.
-# Usage: ./bench-fast-greedy.sh
-#
-# Config: reasoning off, argmax fastpath=1, deterministic sampler.
-# Sends a coding prompt, captures full stream, computes decode tokens and time.
-set -euo pipefail
-
-HOST="127.0.0.1"
-PORT=7770
-ALIAS="frank-default-large"
-
-# A coding prompt representative of the card's 164-task suite.
-PROMPT='### Instruction
-Write a Python class that implements a priority queue using a binary heap. Include methods for enqueue, dequeue, peek, and is_empty. Add type hints, docstrings, and a usage example. The heap should support arbitrary comparable elements.
-### Response'
-
-DECODE_TOKENS=0
-START_SEC=0
-END_SEC=0
-
-echo "=== Fast-greedy benchmark (FASTGREEDY=1) ==="
-echo "Model: $ALIAS on $HOST:$PORT"
-echo "Prompt tokens: $(echo "$PROMPT" | wc -c) chars"
-echo ""
-
-# Use /v1/completions with streaming, capture tokens+time from usage.
-RESPONSE=$(curl -sf http://$HOST:$PORT/v1/completions \
-  -H 'Content-Type: application/json' \
-  -d "{
-    \"model\": \"$ALIAS\",
-    \"prompt\": $(python3 -c "import json,sys; print(json.dumps(sys.stdin.read()))" <<< "$PROMPT"),
-    \"max_tokens\": 512,
-    \"temperature\": 0,
-    \"top_p\": 1.0,
-    \"top_k\": 0,
-    \"min_p\": 0.0,
-    \"n\": 1,
-    \"stream\": true
-  }" 2>/dev/null)
-
-# Extract final usage from the last JSON line (finish_reason=stop or length).
-# Also count output text tokens.
-LAST_LINE=$(echo "$RESPONSE" | grep 'finish_reason' | tail -1)
-
-# Parse timing from the last chunk (has full usage)
-if echo "$LAST_LINE" | grep -q 'timings'; then
-    COMP_TOKENS=$(echo "$LAST_LINE" | python3 -c "
-import sys, json
-data = json.loads(sys.stdin.read())
-print(data.get('usage', {}).get('completion_tokens', 0))
-print(data.get('timings', {}).get('predicted_per_second', 0))
-print(data.get('timings', {}).get('prompt_per_second', 0))
-" 2>/dev/null)
-    echo "$COMP_TOKENS" | {
-        read CT
-        read TPS
-        read PPS
-        echo "Completion tokens: $CT"
-        echo "Predicted (decode) tokens/sec: $TPS"
-        echo "Prompt tokens/sec: $PPS"
-    }
-else
-    echo "WARNING: No timing data found in response"
-    echo "$LAST_LINE" | head -5
-fi
-
-# Count the actual text output
-OUTPUT_TEXT=$(echo "$RESPONSE" | grep '"text"' | head -1 | sed 's/.*"text":"//;s/","index.*//')
-OUTPUT_TOKENS=$(echo -n "$OUTPUT_TEXT" | wc -w)
-echo "Output text (rough token count): ~$OUTPUT_TOKENS words"
-echo ""
-
-# Check journal for draft acceptance and final timing
-echo "=== Draft acceptance & final timing ==="
-sleep 2
-ssh -p 2222 dank@dankclaw "journalctl -u frank-default-large --since '1 minute ago' --no-pager 2>/dev/null | grep -E 'draft acceptance|print_timing.*n_decoded|statistics.*draft-mtp' | tail -4" 2>/dev/null || echo "(could not read journal)"

scripts/bench-kairic-ab.sh

@@ -1,153 +0,0 @@
-#!/usr/bin/env bash
-# bench-kairic-ab.sh — A/B perf isolation for the Kairic Edge deployment on
-# Dankclaw. Run this ON DANKCLAW (paths are absolute target paths).
-#
-# Configs:
-#   runner        pinned runner profile (b2048/ub512, reasoning off, target
-#                 greedy argmax fast path ON)
-#   nommap        runner + --no-mmap
-#   nofast        runner + fast path OFF
-#   prod          production profile (b4096/ub4096, --no-mmap, reasoning on/deepseek,
-#                 fast path OFF, preserve_thinking)
-#   prod2048      prod with the pinned runner's batch sizes (b2048/ub512)
-#   prodnommap    (obsolete: identical to prod since ubatch 4096 + --no-mmap
-#                 moved into the production profile)
-#
-# Usage: ./scripts/bench-kairic-ab.sh <cfg> [ctx] [prompt_chars]
-# Example: ./scripts/bench-kairic-ab.sh prod 16384 53000
-#
-# The server runs on scratch port 7893. STOP frank-default-large before
-# benchmarking (see AGENTS.md). The server is killed on exit.
-set -uo pipefail
-CFG="${1:?runner|nofast|prod}"
-CTX="${2:-32768}"
-PCHARS="${3:-11000}"
-PORT=7893
-BIN=/home/dank/ciru-qwen38/ROCmFPX-kairic/build-kairic/bin/llama-server
-MDIR=/home/dank/models/Kairic-Qwen3.8-27B-IU4-Edge
-SDK=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages
-LOG=/tmp/kairic-ab-$CFG.log
-: > "$LOG"
-
-export ROCM_PATH="$SDK/_rocm_sdk_devel"
-export LD_LIBRARY_PATH="$(dirname "$BIN"):$SDK/_rocm_sdk_libraries/lib:$SDK/_rocm_sdk_core/lib:$SDK/_rocm_sdk_devel/lib"
-export HIP_VISIBLE_DEVICES=0
-export HSA_OVERRIDE_GFX_VERSION=11.5.1
-export GGML_CUDA_GRAPH_OPT=0
-export LLAMA_MTP_CPU_ARGMAX_FASTPATH=1
-export PROMPTFORGE_TARGET_ONLY=0
-export PROMPTFORGE_MODE=iu4_ffn
-export PROMPTFORGE_IU4_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
-export PROMPTFORGE_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-FFN.pfs"
-export PROMPTFORGE_IU4_HADAMARD=1
-export PROMPTFORGE_IU4_SEGMENTED=0
-export PROMPTFORGE_ENABLE_FFN_KEEPERS=0
-export PROMPTFORGE_FFN_KEEPERS=late6
-export PROMPTFORGE_ENABLE_GDN=1
-export PROMPTFORGE_ENABLE_GDN_W8=0
-export PROMPTFORGE_GDN_SIDECAR_OVERRIDE="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
-export PROMPTFORGE_GDN_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN.pfs"
-export PROMPTFORGE_GDN_IU4_HADAMARD=1
-export PROMPTFORGE_ENABLE_GDN_KEEPERS=0
-export PROMPTFORGE_ENABLE_GDN_OUTPUT=1
-export PROMPTFORGE_GDN_OUTPUT_SIDECAR="$MDIR/Qwen3.8-27B-Kairic-IU4-GDN-Output.pfs"
-export PROMPTFORGE_GDN_OUTPUT_KEEPERS=v3_lateq6
-export PROMPTFORGE_ENABLE_SMALLM_IU4=1
-export PROMPTFORGE_ENABLE_SMALLM_GDN_IU4=1
-export PROMPTFORGE_ENABLE_SMALLM_GDN_OUTPUT_IU4=0
-export PROMPTFORGE_ENABLE_IU4_DECODE_M2_M5=0
-export PROMPTFORGE_OUTPUT_K8_STRICT_GREEDY=0
-
-MMAP=""
-case "$CFG" in
-  runner)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512;  REA=off;  RFMT=none;     EXTRA="" ;;
-  nommap)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1; B=2048; UB=512;  REA=off;  RFMT=none;     EXTRA=""; MMAP="--no-mmap" ;;
-  nofast)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512;  REA=off;  RFMT=none;     EXTRA="" ;;
-  prod)    export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=4096; REA=on;   RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
-  prod2048)  export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=2048; UB=512;  REA=on;   RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
-  prodnommap) export LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=0; B=4096; UB=2048; REA=on; RFMT=deepseek; EXTRA="--chat-template-kwargs {\"preserve_thinking\":\"true\"}"; MMAP="--no-mmap" ;;
-esac
-
-"$BIN" -m "$MDIR/Qwen3.8-27B-IU4-Kairic-Edge.gguf" \
-  --alias bench --host 127.0.0.1 --port "$PORT" --jinja \
-  -dev ROCm0 -ngl 999 -c "$CTX" -b "$B" -ub "$UB" -fa on -ctk f16 -ctv f16 \
-  -t 16 -tb 32 -np 1 -ctxcp 32 --cache-ram 8192 --cache-prompt --cache-idle-slots $MMAP \
-  --metrics --kairic-edge \
-  --spec-type draft-mtp --spec-draft-device ROCm0 --spec-draft-ngl 999 \
-  --spec-draft-type-k f16 --spec-draft-type-v f16 \
-  --spec-draft-n-max 4 --spec-draft-n-min 0 --spec-draft-p-min 0.0 --spec-draft-p-split 0.10 \
-  --spec-draft-backend-sampling \
-  --temp 0 --top-p 1.0 --top-k 0 --min-p 0.0 \
-  --reasoning "$REA" --reasoning-format "$RFMT" --reasoning-budget -1 \
-  $EXTRA >> "$LOG" 2>&1 &
-SPID=$!
-trap 'kill $SPID 2>/dev/null; wait $SPID 2>/dev/null' EXIT
-
-for i in $(seq 1 300); do
-  kill -0 $SPID 2>/dev/null || { echo "SERVER DIED"; tail -20 "$LOG"; exit 2; }
-  curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1 && break
-  sleep 2
-done
-echo "=== CFG=$CFG ready ==="
-
-python3 - "$PORT" "$CFG" "$PCHARS" <<'PY'
-import sys, json, time, urllib.request
-port, cfg, pchars = sys.argv[1], sys.argv[2], int(sys.argv[3])
-VAR = [
-    "narrative nonfiction blends observation with reflection, ",
-    "and a careful writer lets each clause earn its place ",
-    "in the sentence's rising line, so meaning accumulates slowly. ",
-    "The reader is not hurried; the reader is led. Loose and periodic, ",
-    "the style turns on concrete nouns and verbs that carry weight. ",
-    "A paragraph is not a list but a single turn of thought that ",
-    "opens, develops, and lands, and clarity is a courtesy owed ",
-    "to the willing reader who gives the writer their attention. ",
-]
-parts = []
-t = 0
-while len("".join(parts)) < pchars:
-    for k in range(3):
-        parts.append(VAR[(t + k) % len(VAR)])
-    parts.append(f" Note {t}. ")
-    t += 1
-longtext = "".join(parts)
-prompt = longtext + "\n\nNow write a long, detailed continuation of this essay (at least five paragraphs) about the philosophy of language.\n"
-body = {
-    "model": "bench",
-    "prompt": prompt,
-    "max_tokens": 512,
-    "temperature": 0,
-    "top_p": 1.0,
-    "top_k": 0,
-    "min_p": 0.0,
-    "stream": False,
-    "cache_prompt": True,
-}
-# ignore_eos is rejected by the exact-Q8 greedy argmax fast path (runner cfg).
-if cfg not in ("runner",):
-    body["ignore_eos"] = True
-def run(tag):
-    req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
-                                 data=json.dumps(body).encode(),
-                                 headers={"Content-Type": "application/json"})
-    t0 = time.time()
-    try:
-        with urllib.request.urlopen(req, timeout=3600) as resp:
-            out = json.load(resp)
-    except Exception as e:
-        body_txt = getattr(e, "read", lambda: b"")()
-        print(f"[{tag}] REQUEST FAILED: {e}")
-        print(f"[{tag}] body: {body_txt.decode('utf-8', 'ignore')[:500]}")
-        raise SystemExit(3)
-    wall = time.time() - t0
-    u = out.get("usage", {})
-    tm = out.get("timings", {})
-    pt = u.get("prompt_tokens", 0); ct = u.get("completion_tokens", 0)
-    pps = tm.get("prompt_per_second", 0); tps = tm.get("predicted_per_second", 0)
-    print(f"[{tag}] cfg={cfg} prompt={pt} completion={ct} prefill={pps:.1f} t/s decode={tps:.2f} t/s wall={wall:.1f}s")
-run("cold")
-run("warm")
-PY
-
-grep -E "print_timing|statistics" "$LOG" | tail -6
-exit 0

scripts/bench-spec-stack.sh

@@ -1,191 +0,0 @@
-#!/usr/bin/env bash
-# bench-spec-stack.sh — compare MTP-only vs MTP+ngram-mod spec on the v3
-# CIRU prompt-forge stack (Dankclaw, gfx1151), at 8k / 16k / 64k context.
-#
-# Reads authoritative prefill/generation tok/s from llama-server's own
-# `print_timing` lines (prompt eval time / eval time). Runs one server
-# instance per (spec, ctx) on a scratch port.
-#
-# Usage:
-#   ./scripts/bench-spec-stack.sh <draft-mtp|draft-mtp,ngram-mod> <ctx> [out_tokens]
-# Example:
-#   ./scripts/bench-spec-stack.sh draft-mtp,ngram-mod 16384 256
-set -euo pipefail
-
-SPEC="${1:?usage: bench-spec-stack.sh <spec> <ctx> [out_tokens]}"
-CTX="${2:?usage: bench-spec-stack.sh <spec> <ctx> [out_tokens]}"
-OUT_TOKENS="${3:-256}"
-
-ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-large.env
-source "$ENV_FILE"
-
-# This harness benchmarks the retired v3 stack. Force the v3 values so the
-# harness works regardless of the live service env (now Kairic).
-MODEL_DIR=/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3
-MODEL_FILE=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-v3-Q8-FFNEQ-LateQ6.gguf
-FFN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-FFN.pfs
-GDN_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-GDN.pfs
-OUTPUT_K8_PFS=Qwen3.8-27B-CIRU-ActiveFPX-PromptForge-Output-K8.pfs
-MMPROJ_FILE=Qwen3.8-27B-mmproj-BF16.mmproj
-
-# Derive the PromptForge sidecar paths from the env base values.
-export PROMPTFORGE_SIDECAR="$MODEL_DIR/$FFN_PFS"
-export PROMPTFORGE_GDN_SIDECAR="$MODEL_DIR/$GDN_PFS"
-export PROMPTFORGE_MTP_OUTPUT_K8_PROXY="$MODEL_DIR/$OUTPUT_K8_PFS"
-
-MODEL="$MODEL_DIR/$MODEL_FILE"
-LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
-PORT=7893
-PARALLEL=1
-BATCH=4096
-UBATCH=4096
-LOG="/tmp/specstack-$(echo "$SPEC" | tr ',' '-')-ctx$CTX.log"
-: > "$LOG"
-
-for f in "$MODEL" "$PROMPTFORGE_SIDECAR" "$PROMPTFORGE_GDN_SIDECAR" "$PROMPTFORGE_MTP_OUTPUT_K8_PROXY" "$MODEL_DIR/$MMPROJ_FILE"; do
-    [[ -f "$f" ]] || { echo "missing: $f" >&2; exit 1; }
-done
-[[ -x "$LLAMA_BIN" ]] || { echo "missing: $LLAMA_BIN" >&2; exit 1; }
-
-export PROMPTFORGE_MODE="m2048_fused_tail1476"
-export GGML_CUDA_GRAPH_OPT=0
-export PROMPTFORGE_ENABLE_SMALLM_W8=1
-export PROMPTFORGE_MTP_OUTPUT_K8=1
-export PROMPTFORGE_MTP_OUTPUT_K8_VALIDATE=0
-export PROMPTFORGE_MTP_OUTPUT_K8_PROXY
-unset LLAMA_MTP_CPU_ARGMAX_FASTPATH LLAMA_MTP_BACKEND_GREEDY || true
-unset PROMPTFORGE_IU4_SIDECAR PROMPTFORGE_GDN_IU4_SIDECAR || true
-unset PROMPTFORGE_ENABLE_NGRAM_MOD PROMPTFORGE_ENABLE_NGRAM_M65_IU4 || true
-
-export ROCM_PATH="$ROCM_DEVEL"
-export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
-
-echo "=== spec-stack bench: spec=$SPEC ctx=$CTX model=$MODEL_FILE ==="
-echo "=== env: OUTPUT_K8=$PROMPTFORGE_MTP_OUTPUT_K8 smallm=$PROMPTFORGE_ENABLE_SMALLM_W8 ==="
-
-# ngram-mod tuning matches the retired fallback line: n_min 48 / n_max 64 / n_match 24
-"$LLAMA_BIN" \
-    --model "$MODEL" \
-    --mmproj "$MODEL_DIR/$MMPROJ_FILE" \
-    --alias bench \
-    --host 127.0.0.1 \
-    --port "$PORT" \
-    -dev ROCm0 \
-    -ngl 999 \
-    -c "$CTX" \
-    -b "$BATCH" \
-    -ub "$UBATCH" \
-    -fa on \
-    -ctk "$CACHE_TYPE_K" \
-    -ctv "$CACHE_TYPE_V" \
-    -np "$PARALLEL" \
-    -ctxcp "$CTX_CHECKPOINTS" \
-    --cache-ram "$CACHE_RAM" \
-    --cache-prompt \
-    --no-mmap \
-    --spec-type "$SPEC" \
-    --spec-draft-device ROCm0 \
-    --spec-draft-ngl 999 \
-    --spec-draft-type-k f16 \
-    --spec-draft-type-v f16 \
-    --spec-draft-n-max 4 \
-    --spec-draft-n-min 0 \
-    --spec-draft-p-min 0.0 \
-    --spec-draft-p-split 0.10 \
-    --spec-draft-backend-sampling \
-    --spec-ngram-mod-n-min 48 \
-    --spec-ngram-mod-n-max 64 \
-    --spec-ngram-mod-n-match 24 \
-    --temp 0.7 \
-    --top-p 0.95 \
-    -lv 4 \
-    >> "$LOG" 2>&1 &
-SERVER_PID=$!
-trap 'kill $SERVER_PID 2>/dev/null; wait $SERVER_PID 2>/dev/null' EXIT
-
-echo "server pid $SERVER_PID on 127.0.0.1:$PORT"
-echo -n "waiting for server"
-ready=0
-crashed=0
-for i in $(seq 1 900); do
-    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
-        crashed=1
-        break
-    fi
-    if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
-        echo " [ready]"
-        ready=1
-        break
-    fi
-    [ $((i % 10)) = 0 ] && echo -n .
-    sleep 2
-done
-if [ "$crashed" = "1" ]; then
-    echo " BACKEND_CRASHED (server died before ready)"
-    tail -30 "$LOG"
-    exit 2
-fi
-[ "$ready" = "1" ] || { echo " FAIL (timeout)"; tail -25 "$LOG"; exit 1; }
-
-# Print the implementations that initialized (feasibility evidence)
-grep -E "adding speculative implementation|ngram_mod|draft-mtp" "$LOG" | head -6
-
-python3 - "$SPEC" "$CTX" "$PORT" "$OUT_TOKENS" "$LOG" <<'PY'
-import sys, time, json, urllib.request
-spec, ctx, port, out_tokens, log = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5]
-
-# Flowing prose (varied) at ~75% of ctx so MTP acceptance is realistic.
-VAR = (
-    "narrative nonfiction blends observation with reflection, ",
-    "and a careful writer lets each clause earn its place ",
-    "in the sentence's rising line, so meaning accumulates slowly. ",
-    "The reader is not hurried; the reader is led. Loose and periodic, ",
-    "the style turns on concrete nouns and verbs that carry weight, ",
-    "and it resists the abstract noun that hides a vague claim. ",
-    "A paragraph is not a list. It is a single turn of thought that ",
-    "opens, develops, and lands. This is the craft of prose. Choose, ",
-    "arrange, and above all omit. Omission is the discipline that ",
-    "keeps the whole clear, and clarity is the courtesy we owe the ",
-    "willing reader."
-)
-n = len(VAR)
-parts = []
-t = 0
-while t * 76 < (ctx * 3) // 4:
-    for k in range(5):
-        parts.append(VAR[(t + k) % n])
-    parts.append(f" This is cued by note {t}. ")
-    t += 1
-prompt = "".join(parts)
-prompt += ("\n\nNow, in a detailed and flowing paragraph, continue this essay about "
-           "the philosophy of language and its influence on modern computation. ")
-
-body = json.dumps({
-    "prompt": prompt,
-    "max_tokens": out_tokens,
-    "temperature": 0.0,
-    "stream": False,
-}).encode()
-req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/completions",
-                             data=body, headers={"Content-Type": "application/json"})
-t0 = time.time()
-with urllib.request.urlopen(req, timeout=7200) as resp:
-    out = json.load(resp)
-dt = time.time() - t0
-pt = out.get("usage", {}).get("prompt_tokens", 0)
-ct = out.get("usage", {}).get("completion_tokens", 0)
-print(f"request done: prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s", flush=True)
-
-lines = open(log, encoding="utf-8", errors="ignore").read().splitlines()
-prompt_evals = [l for l in lines if "prompt eval time" in l]
-evals       = [l for l in lines if "eval time" in l and "prompt eval" not in l]
-
-def grab(lines, label):
-    if lines:
-        print(label + ": " + lines[-1].strip())
-
-print(f"== RESULT spec={spec} ctx={ctx} ==")
-grab(prompt_evals, "PREFILL")
-grab(evals, "GENERATION")
-print(f"prompt_tokens={pt} completion_tokens={ct} wall={dt:.1f}s")
-PY

scripts/bench-ubatch.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Benchmark ubatch with llama-bench for frank-default-large.
+#
+# This is intentionally a ubatch-only benchmark. It uses one llama-bench
+# process, an 8K prompt workload, and equal batch/ubatch values. No server
+# process, HTTP requests, or log scraping are used.
+#
+# Run on Dankclaw after stopping frank-default-large:
+#   sudo systemctl stop frank-default-large
+#   ./scripts/bench-ubatch.sh
+#   sudo systemctl start frank-default-large
+set -euo pipefail
+
+: "${LLAMA_BENCH:=/home/dank/GitPackages/ROCmFPX/build-strix-rocmfp4/bin/llama-bench}"
+: "${MODEL:=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
+: "${UBATCH_VALUES:=256,512,768,1024,1280,1536,1792,2048,3072,4096}"
+: "${CONTEXT:=8192}"
+: "${GEN_TOKENS:=128}"
+: "${REPETITIONS:=3}"
+: "${DEVICE:=ROCm0}"
+: "${OUTPUT:=jsonl}"
+
+if [[ ! -x "$LLAMA_BENCH" ]]; then
+    printf 'missing executable llama-bench: %s\n' "$LLAMA_BENCH" >&2
+    exit 1
+fi
+if [[ ! -r "$MODEL" ]]; then
+    printf 'missing or unreadable model: %s\n' "$MODEL" >&2
+    exit 1
+fi
+
+exec "$LLAMA_BENCH" \
+    --model "$MODEL" \
+    --n-prompt "$CONTEXT" \
+    --n-gen "$GEN_TOKENS" \
+    --batch-size "$UBATCH_VALUES" \
+    --ubatch-size "$UBATCH_VALUES" \
+    --device "$DEVICE" \
+    --n-gpu-layers 999 \
+    --cache-type-k f16 \
+    --cache-type-v f16 \
+    --flash-attn on \
+    --mmap 0 \
+    --repetitions "$REPETITIONS" \
+    --output "$OUTPUT"

scripts/deploy-open-webui-quadlet.sh

@@ -1,104 +1,14 @@
 #!/usr/bin/env bash
-# deploy-open-webui-quadlet.sh — Install and (re)start open-webui on dankclaw.
+#!/usr/bin/env bash
-#
+# Compatibility wrapper for the service-scoped deployment dispatcher.
-# open-webui is boot-persistent via Quadlet (see AGENTS.md). This script:
-#   1. Pushes main to origin
-#   2. Pulls the repo on the target
-#   3. Creates the data dir and copies the unit + network into ~/.config/containers/systemd/
-#   4. daemon-reloads and starts open-webui (first run pulls the image)
-#
-# It is idempotent: it stops any running unit and removes any container with the
-# name (a prior run or a bare-container leftover) before reinstalling and
-# starting. The data dir is preserved across reruns, so existing chats survive.
-#
-# Usage:
-#   ./scripts/deploy-open-webui-quadlet.sh <target> [--dry-run]
-#   ./scripts/deploy-open-webui-quadlet.sh dankclaw
 set -euo pipefail
 
-REMOTE_USER="dank"
-REMOTE_HOST="dankclaw"
-REMOTE_DIR="/home/dank/serverAppConfigs"
-QDIR="dankclaw/open-webui/quadlet"
-UNIT="open-webui.service"
-DATA_DIR="/home/dank/Containers/open-webui/data"
-
 TARGET="${1:-}"
-DRY_RUN=0
+shift || true
-if [ "${2:-}" = "--dry-run" ]; then
+case "$TARGET" in
-    DRY_RUN=1
+    dankclaw) exec "$(dirname "$0")/deploy.sh" dankclaw --service open-webui "$@" ;;
-fi
+    *)
-
+        echo "This stack runs only on dankclaw (got: ${TARGET:-<missing>})" >&2
-if [ -z "${TARGET:-}" ]; then
+        exit 2
-    echo "Usage: $0 <target> [--dry-run]"
+        ;;
-    exit 1
+ esac
-fi
-if [ "${TARGET}" != "dankclaw" ]; then
-    echo "This stack runs only on dankclaw (got: ${TARGET})"
-    exit 1
-fi
-
-# ---------------------------------------------------------------------------
-# Local pre-flight
-# ---------------------------------------------------------------------------
-cd "$(dirname "$0")/.."
-if ! git diff --quiet HEAD; then
-    echo "FAIL: Local working tree has uncommitted changes. Commit first."
-    exit 1
-fi
-
-if [ "$DRY_RUN" = "1" ]; then
-    echo "[dry-run] Would push main, pull on ${REMOTE_HOST}, install ${UNIT}, and start it (data at ${DATA_DIR})."
-    exit 0
-fi
-
-echo "=== Deploying open-webui quadlet to ${REMOTE_HOST} ==="
-echo "[1/3] Pushing local main to origin..."
-git push origin main
-
-REMOTE="${REMOTE_USER}@${REMOTE_HOST}"
-
-echo "[2/3] Installing and starting ${UNIT} on ${REMOTE_HOST}..."
-ssh "$REMOTE" bash -s <<REMOTE_EOF
-set -euo pipefail
-cd '${REMOTE_DIR}'
-
-echo '  Pulling repo...'
-git pull origin main
-
-echo '  Stopping any running unit (idempotent)...'
-systemctl --user stop '${UNIT}' 2>/dev/null || true
-
-echo '  Removing any container with the name (prior run or bare-container leftover)...'
-podman rm -f open-webui 2>/dev/null || true
-
-echo '  Creating the data dir (preserved across reruns)...'
-mkdir -p '${DATA_DIR}'
-
-echo '  Copying the quadlet unit + network into ~/.config/containers/systemd/ ...'
-mkdir -p ~/.config/containers/systemd
-cp ${QDIR}/open-webui.container ~/.config/containers/systemd/
-cp ${QDIR}/open-webui.network ~/.config/containers/systemd/
-
-echo '  daemon-reload (generates the unit)...'
-# Quadlet units are generator-managed: systemctl enable fails on them. The
-# [Install] WantedBy=default.target in the .container file is itself the
-# enablement (is-enabled reports "generated"); with user linger on it starts at
-# boot. So: daemon-reload, then start.
-systemctl --user daemon-reload
-
-echo '  Starting (first run pulls the image; this can take a while)...'
-systemctl --user start '${UNIT}'
-
-echo '  Unit state:'
-systemctl --user --no-pager status '${UNIT}' 2>&1 | grep -E "●|Active:" || true
-REMOTE_EOF
-
-echo "[3/3] Verifying open-webui is up..."
-ssh "$REMOTE" '
-  echo "  is-enabled: $(systemctl --user is-enabled open-webui.service 2>&1) (boot-persistent)"
-  echo "  port 3005:"; (ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null) | grep -E ":3005\b" || echo "  (not listening yet)"
-  echo "  HTTP probe:"; curl -s -o /dev/null -w "  http://127.0.0.1:3005 -> %{http_code}\n" -m 10 http://127.0.0.1:3005/ || echo "  (no HTTP yet)"
-' || true
-
-echo "=== open-webui quadlet deployed to ${REMOTE_HOST} ==="

scripts/deploy-pihole-quadlet.sh

@@ -1,118 +1,13 @@
 #!/usr/bin/env bash
-# deploy-pihole-quadlet.sh — Install and (re)start the Pi-hole LAN DNS on
+# Compatibility wrapper for the service-scoped deployment dispatcher.
-# dankclaw. Pi-hole retired the dnsmasq quadlet (2026-08-24).
-#
-# Pi-hole is boot-persistent via Quadlet (see AGENTS.md). This script:
-#   1. Pushes main to origin
-#   2. Pulls the repo on the target
-#   3. Creates the data dirs and copies the quadlet unit + env + addn-hosts
-#   4. daemon-reloads and starts pihole (first run pulls the image)
-#
-# It is idempotent: it stops any running unit and removes any container with
-# the name (a prior run or a bare-container leftover) before reinstalling and
-# starting. The data dirs (etc-pihole, etc-dockers) are preserved across
-# reruns, so the blocklist and web settings survive. The env file is created
-# from env.example only if absent; fill the real WEBPASSWORD before the web
-# UI is usable.
-#
-# Usage:
-#   ./scripts/deploy-pihole-quadlet.sh <target> [--dry-run]
-#   ./scripts/deploy-pihole-quadlet.sh dankclaw
 set -euo pipefail
 
-REMOTE_USER="dank"
-REMOTE_HOST="dankclaw"
-REMOTE_DIR="/home/dank/serverAppConfigs"
-QDIR="dankclaw/pihole"
-UNIT="pihole.service"
-DATA_DIR="/home/dank/Containers/pihole"
-
 TARGET="${1:-}"
-DRY_RUN=0
+shift || true
-if [ "${2:-}" = "--dry-run" ]; then
+case "$TARGET" in
-    DRY_RUN=1
+    dankclaw) exec "$(dirname "$0")/deploy.sh" dankclaw --service pihole "$@" ;;
-fi
+    *)
-
+        echo "This stack runs only on dankclaw (got: ${TARGET:-<missing>})" >&2
-if [ -z "${TARGET:-}" ]; then
+        exit 2
-    echo "Usage: $0 <target> [--dry-run]"
+        ;;
-    exit 1
+esac
-fi
-if [ "${TARGET}" != "dankclaw" ]; then
-    echo "This stack runs only on dankclaw (got: ${TARGET})"
-    exit 1
-fi
-
-# ---------------------------------------------------------------------------
-# Local pre-flight
-# ---------------------------------------------------------------------------
-cd "$(dirname "$0")/.."
-if ! git diff --quiet HEAD; then
-    echo "FAIL: Local working tree has uncommitted changes. Commit first."
-    exit 1
-fi
-
-if [ "$DRY_RUN" = "1" ]; then
-    echo "[dry-run] Would push main, pull on ${REMOTE_HOST}, install ${UNIT}, and start it (data at ${DATA_DIR})."
-    exit 0
-fi
-
-echo "=== Deploying pihole quadlet to ${REMOTE_HOST} ==="
-echo "[1/3] Pushing local main to origin..."
-git push origin main
-
-REMOTE="${REMOTE_USER}@${REMOTE_HOST}"
-
-echo "[2/3] Installing and starting ${UNIT} on ${REMOTE_HOST}..."
-ssh "$REMOTE" bash -s <<REMOTE_EOF
-set -euo pipefail
-cd '${REMOTE_DIR}'
-
-echo '  Pulling repo...'
-git pull origin main
-
-echo '  Stopping any running unit (idempotent)...'
-systemctl --user stop '${UNIT}' 2>/dev/null || true
-
-echo '  Removing any container with the name (prior run or bare-container leftover)...'
-podman rm -f pihole 2>/dev/null || true
-
-echo '  Creating the data dirs (preserved across reruns)...'
-mkdir -p '${DATA_DIR}/etc-pihole' '${DATA_DIR}/etc-dockers'
-
-echo '  Creating the env file from env.example if absent...'
-if [ ! -f '${DATA_DIR}/env' ]; then
-    cp ${QDIR}/env.example '${DATA_DIR}/env'
-    echo '  env created: set the real WEBPASSWORD before the web UI is usable'
-else
-    echo '  env present, keeping it'
-fi
-
-echo '  Copying the record file (addn-hosts)...'
-cp ${QDIR}/addn-hosts '${DATA_DIR}/addn-hosts'
-
-echo '  Copying the quadlet unit into ~/.config/containers/systemd/ ...'
-mkdir -p ~/.config/containers/systemd
-cp ${QDIR}/quadlet/pihole.container ~/.config/containers/systemd/
-
-echo '  daemon-reload (generates the unit)...'
-# Quadlet units are generator-managed: systemctl enable fails on them. The
-# [Install] WantedBy=default.target in the .container file is itself the
-# enablement (is-enabled reports "generated"); with user linger on it starts
-# at boot. So: daemon-reload, then start.
-systemctl --user daemon-reload
-
-echo '  Starting (first run pulls the image; this can take a while)...'
-systemctl --user start '${UNIT}'
-
-echo '  Unit state:'
-systemctl --user --no-pager status '${UNIT}' 2>&1 | grep -E "●|Active:" || true
-REMOTE_EOF
-
-echo "[3/3] Verifying pihole is up..."
-ssh "$REMOTE" '
-  echo "  is-enabled: $(systemctl --user is-enabled pihole.service 2>&1) (boot-persistent)"
-  echo "  port 53:"; (ss -ulpn 2>/dev/null) | grep -E ":53\b" || echo "  (not listening yet)"
-  echo "  resolution:"; dig @127.0.0.1 romm.franksdank.cloud +short 2>/dev/null || echo "  (no answer yet)"
-' || true
-
-echo "=== pihole quadlet deployed to ${REMOTE_HOST} ==="

scripts/deploy-plan.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""Plan runtime services affected by repository changes.
+
+The planner is intentionally side-effect free.  It accepts either an explicit
+path list or two Git revisions.  The latter lets compose-file changes resolve
+to only the service definitions that actually changed instead of restarting an
+entire compose project.
+"""
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import PurePosixPath
+from typing import Any
+
+import yaml
+
+# Dependency-sensitive services come before their consumers.  Independent
+# compose services follow.  LLM services are deliberately last and are never
+# implied by a generic host, stack, documentation, or config change.
+SERVICE_ORDER = (
+    "gluetun",
+    "searxng-valkey",
+    "searxng",
+    "pihole",
+    "open-webui",
+    "gitea-postgres",
+    "gitea-server",
+    "comfyui",
+    "hermes",
+    "hermes-dashboard",
+    "hermes-webui",
+    "prometheus",
+    "grafana",
+    "loki",
+    "alloy",
+    "node-exporter",
+    "authentik-postgres",
+    "authentik-server",
+    "authentik-worker",
+    "caddy",
+    "homepage",
+    "mariadb",
+    "redis",
+    "romm",
+    "copyparty",
+    "jellyfin",
+    "calibre-web",
+    "prowlarr",
+    "qbittorrent",
+    "radarr",
+    "sonarr",
+    "lidarr",
+    "readarr",
+    "frank-default-large",
+    "frank-default-npu",
+    "frank-default-small",
+)
+
+DANKCLAW_SERVICES = {
+    "hermes-webui",
+    "gluetun",
+    "searxng-valkey",
+    "searxng",
+    "pihole",
+    "open-webui",
+    "gitea-postgres",
+    "gitea-server",
+    "comfyui",
+    "hermes",
+    "hermes-dashboard",
+    "prometheus",
+    "grafana",
+    "loki",
+    "alloy",
+    "node-exporter",
+    "authentik-postgres",
+    "authentik-server",
+    "authentik-worker",
+    "caddy",
+    "homepage",
+    "frank-default-large",
+    "frank-default-npu",
+}
+
+DANKSERVER_SERVICES = {
+    "mariadb",
+    "redis",
+    "romm",
+    "copyparty",
+    "jellyfin",
+    "calibre-web",
+    "prowlarr",
+    "qbittorrent",
+    "radarr",
+    "sonarr",
+    "lidarr",
+    "readarr",
+    "frank-default-small",
+}
+
+DANKCLAW_COMPOSE_SERVICES = {
+    "comfyui",
+    "hermes",
+    "hermes-dashboard",
+    "hermes-webui",
+    "prometheus",
+    "grafana",
+    "loki",
+    "alloy",
+    "node-exporter",
+    "authentik-postgres",
+    "authentik-server",
+    "authentik-worker",
+    "caddy",
+}
+
+DANKSERVER_COMPOSE_SERVICES = {
+    "mariadb",
+    "redis",
+    "romm",
+    "copyparty",
+    "jellyfin",
+    "calibre-web",
+    "prowlarr",
+    "qbittorrent",
+    "radarr",
+    "sonarr",
+    "lidarr",
+    "readarr",
+}
+
+
+def _add(services: set[str], *names: str) -> None:
+    services.update(names)
+
+
+def target_services(target: str) -> set[str]:
+    if target == "dankclaw":
+        return DANKCLAW_SERVICES
+    if target == "dankserver":
+        return DANKSERVER_SERVICES
+    raise ValueError(f"unknown target: {target}")
+
+
+def _compose_services(target: str) -> set[str]:
+    return DANKCLAW_COMPOSE_SERVICES if target == "dankclaw" else DANKSERVER_COMPOSE_SERVICES
+
+
+def _git_show(repo: str, revision: str, path: str) -> str | None:
+    result = subprocess.run(
+        ["git", "-C", repo, "show", f"{revision}:{path}"],
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        check=False,
+    )
+    return result.stdout if result.returncode == 0 else None
+
+
+def _yaml_document(contents: str | None) -> dict[str, Any]:
+    if contents is None:
+        return {}
+    try:
+        data = yaml.safe_load(contents) or {}
+    except yaml.YAMLError:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def _yaml_services(contents: str | None) -> dict[str, Any]:
+    services = _yaml_document(contents).get("services", {})
+    return services if isinstance(services, dict) else {}
+
+
+def compose_changes(repo: str, target: str, base: str, head: str) -> set[str]:
+    """Return only compose services whose YAML definitions or dependencies changed."""
+    path = f"{target}/compose.yaml"
+    old_doc = _yaml_document(_git_show(repo, base, path))
+    new_doc = _yaml_document(_git_show(repo, head, path))
+    old = old_doc.get("services", {}) if isinstance(old_doc.get("services", {}), dict) else {}
+    new = new_doc.get("services", {}) if isinstance(new_doc.get("services", {}), dict) else {}
+    names = set(old) | set(new)
+    allowed = _compose_services(target)
+
+    # An unavailable or malformed revision is safer as a broad compose plan
+    # than as a silent no-op. It still cannot include LLM units.
+    if (old_doc and not new_doc) or (new_doc and not old_doc and names):
+        return allowed
+
+    # Top-level networks/configuration can affect every compose service even
+    # when no individual service mapping changed. Ignore the services key while
+    # comparing the rest of the Compose document.
+    old_globals = {key: value for key, value in old_doc.items() if key != "services"}
+    new_globals = {key: value for key, value in new_doc.items() if key != "services"}
+    if old_globals != new_globals:
+        return allowed
+
+    return {name for name in names if name in allowed and old.get(name) != new.get(name)}
+
+
+def services_for_path(path: str, target: str) -> set[str]:
+    """Return services affected by one repository-relative path."""
+    path = path.replace("\\", "/")
+    p = PurePosixPath(path)
+    parts = p.parts
+    services: set[str] = set()
+    if not parts:
+        return services
+
+    # Non-runtime material is deliberately a no-op.
+    if parts[0] in {"docs", "tests", "old", ".githooks"}:
+        return services
+    if path in {"README.md", "AGENTS.md"} or parts[0] == ".obsidian":
+        return services
+
+    if parts[0] == "shared":
+        if len(parts) < 2:
+            return services
+        group = parts[1]
+        if group == "searxng":
+            _add(services, "searxng")
+        elif group == "prometheus":
+            _add(services, "prometheus")
+        elif group == "loki":
+            _add(services, "loki")
+        elif group == "alloy":
+            _add(services, "alloy")
+        elif group == "homepage":
+            # Homepage is a Dankclaw-owned deployment. Shared homepage files
+            # are intentionally not copied during a Dankserver-only run.
+            if target == "dankclaw":
+                _add(services, "homepage")
+        return services
+
+    if parts[0] == "dankclaw":
+        if len(parts) >= 2 and parts[1] == "compose.yaml":
+            # Revision-aware callers replace this broad fallback with the
+            # service-level YAML comparison in compose_changes().
+            _add(services, *DANKCLAW_COMPOSE_SERVICES)
+            return services
+        if len(parts) >= 2 and parts[1] == "Caddyfile":
+            _add(services, "caddy")
+            return services
+        if len(parts) >= 2 and parts[1] == "Containerfile.comfyui":
+            _add(services, "comfyui")
+            return services
+        if len(parts) >= 2 and parts[1] in {"caddy", "comfyui", "loki", "alloy", "hermes"}:
+            _add(services, parts[1])
+            return services
+        if len(parts) >= 2 and parts[1] == "searxng":
+            if len(parts) >= 4 and parts[2] == "quadlet":
+                name = parts[3]
+                if name == "gluetun.container":
+                    # SearXNG shares Gluetun's network namespace.  A Gluetun
+                    # recreation requires both units to be restarted.
+                    _add(services, "gluetun", "searxng")
+                elif name == "searxng.container":
+                    _add(services, "searxng")
+                elif name == "searxng-valkey.container":
+                    _add(services, "searxng-valkey", "searxng")
+                elif name == "searxng.network":
+                    _add(services, "gluetun", "searxng-valkey", "searxng")
+            return services
+        if len(parts) >= 2 and parts[1] == "pihole":
+            _add(services, "pihole")
+            return services
+        if len(parts) >= 2 and parts[1] == "open-webui":
+            _add(services, "open-webui")
+            return services
+        if len(parts) >= 2 and parts[1] == "gitea":
+            if len(parts) >= 4 and parts[2] == "quadlet":
+                name = parts[3]
+                if name == "gitea-postgres.container":
+                    _add(services, "gitea-postgres")
+                elif name == "gitea-server.container":
+                    _add(services, "gitea-server")
+                elif name == "gitea-caddy.container":
+                    # Removing the old split proxy requires the canonical
+                    # Caddy handler to retire its installed unit/container
+                    # before it claims ports 80/443.
+                    _add(services, "caddy")
+                elif name == "gitea.network":
+                    _add(services, "gitea-postgres", "gitea-server")
+            return services
+        return services
+
+    if parts[0] == "dankserver":
+        if len(parts) >= 2 and parts[1] == "compose.yaml":
+            _add(services, *DANKSERVER_COMPOSE_SERVICES)
+        return services
+
+    if parts[0] == "scripts":
+        name = p.name
+        if name in {"frank-default-large.env", "frank-default-large.service", "serve-frank-large.sh"}:
+            _add(services, "frank-default-large")
+        elif name in {"frank-default-npu.env", "frank-default-npu.service", "serve-frank-default-npu.sh"}:
+            _add(services, "frank-default-npu")
+        elif name in {"frank-default-small.env", "frank-default-small.service", "serve-frank-small.sh"}:
+            _add(services, "frank-default-small")
+        elif name == "chat-template.jinja":
+            # Only the selected target's LLM consumes this shared template.
+            _add(services, "frank-default-large" if target == "dankclaw" else "frank-default-small")
+        elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
+            _add(services, "homepage")
+        # Deployment/test/benchmark scripts and other repo tooling do not
+        # restart a runtime service.
+        return services
+
+    # Examples and env templates are not deployed runtime state.
+    if p.name in {".env.example", "env.example"}:
+        return services
+    return services
+
+
+def changed_paths(repo: str, base: str, head: str) -> list[str]:
+    result = subprocess.run(
+        ["git", "-C", repo, "diff", "--name-only", "--diff-filter=ACDMRTUXB", base, head],
+        text=True,
+        stdout=subprocess.PIPE,
+        check=True,
+    )
+    return [line for line in result.stdout.splitlines() if line]
+
+
+def plan(paths: list[str], target: str, repo: str | None = None,
+         base: str | None = None, head: str | None = None) -> list[str]:
+    services: set[str] = set()
+    for path in paths:
+        if path == f"{target}/compose.yaml" and repo and base and head:
+            services.update(compose_changes(repo, target, base, head))
+        else:
+            services.update(services_for_path(path, target))
+    allowed = target_services(target)
+    return [name for name in SERVICE_ORDER if name in services and name in allowed]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--target", choices=("dankclaw", "dankserver"), required=True)
+    parser.add_argument("--paths", nargs="+")
+    parser.add_argument("--repo", help="repository path for revision-aware planning")
+    parser.add_argument("--base", help="base Git revision")
+    parser.add_argument("--head", help="head Git revision")
+    args = parser.parse_args()
+
+    if not args.paths and not (args.repo and args.base and args.head):
+        parser.error("provide --paths or all of --repo, --base, and --head")
+    paths = args.paths or changed_paths(args.repo, args.base, args.head)
+    print("\n".join(plan(paths, args.target, args.repo, args.base, args.head)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/deploy-remote.sh

@@ -0,0 +1,584 @@
+#!/usr/bin/env bash
+# deploy-remote.sh — apply an explicit service plan on one target host.
+#
+# This script runs on the target checkout. It deliberately does not infer a
+# plan, pull Git, or restart an unrelated service. deploy.sh computes the plan
+# from the pushed revision and passes the selected service names here.
+# Per-service revisions live in /var/lib/serverAppConfigs/deploy-state (or
+# DEPLOY_STATE_DIR), outside the checkout, so an unselected change remains
+# pending until its service is explicitly deployed.
+set -euo pipefail
+
+TARGET="${1:-}"
+REVISION="${2:-}"
+shift 2 || true
+FROM_REVISION=""
+PENDING=0
+SERVICES=()
+
+usage() {
+    cat <<'EOF'
+Usage: deploy-remote.sh <dankclaw|dankserver> <git-revision> [--pending] [service]...
+
+The service list is explicit. An empty list is a successful no-op.
+EOF
+}
+
+while [[ "$#" -gt 0 ]]; do
+    case "$1" in
+        --from-revision)
+            [[ "$#" -ge 2 ]] || { echo "--from-revision requires a revision" >&2; exit 2; }
+            FROM_REVISION="$2"
+            shift 2
+            ;;
+        --pending)
+            PENDING=1
+            shift
+            ;;
+        --help|-h)
+            usage
+            exit 0
+            ;;
+        *)
+            SERVICES+=("$1")
+            shift
+            ;;
+    esac
+done
+
+REMOTE_DIR="${REMOTE_DIR:-/home/dank/serverAppConfigs}"
+STATE_DIR="${DEPLOY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/serverAppConfigs/deploy-state}"
+
+run_privileged() {
+    if [[ -n "${FRANK_SUDO_PW:-}" ]]; then
+        printf '%s\n' "$FRANK_SUDO_PW" | sudo -S -- "$@"
+    else
+        sudo -- "$@"
+    fi
+}
+
+if [[ "$TARGET" != dankclaw && "$TARGET" != dankserver ]]; then
+    usage >&2
+    exit 2
+fi
+if [[ -z "$REVISION" ]]; then
+    usage >&2
+    exit 2
+fi
+if [[ "$REVISION" == *[!0-9a-fA-F]* ]]; then
+    echo "invalid Git revision: $REVISION" >&2
+    exit 2
+fi
+if ! git -C "$REMOTE_DIR" cat-file -e "$REVISION^{commit}" 2>/dev/null; then
+    echo "revision is not a commit in the target checkout: $REVISION" >&2
+    exit 2
+fi
+cd "$REMOTE_DIR"
+
+# Keep the state directory outside Git. The file is written only after a
+# handler completes successfully; an interrupted handler remains pending.
+mkdir -p "$STATE_DIR"
+
+service_is_allowed() {
+    local service="$1"
+    case "$TARGET:$service" in
+        dankclaw:gluetun|dankclaw:searxng-valkey|dankclaw:searxng|dankclaw:pihole|\
+        dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
+        dankclaw:comfyui|dankclaw:hermes|\
+        dankclaw:hermes-dashboard|dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|\
+        dankclaw:loki|dankclaw:alloy|dankclaw:node-exporter|\
+        dankclaw:authentik-postgres|dankclaw:authentik-server|\
+        dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\
+        dankclaw:frank-default-large|dankclaw:frank-default-npu|\
+        dankserver:mariadb|dankserver:redis|dankserver:romm|\
+        dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|\
+        dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
+        dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
+        dankserver:frank-default-small)
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+mark_status() {
+    local service="$1" status="$2" tmp
+    tmp="$(mktemp "$STATE_DIR/.${service}.XXXXXX")"
+    printf 'revision=%s\nstatus=%s\n' "$REVISION" "$status" > "$tmp"
+    mv -f -- "$tmp" "$STATE_DIR/$service"
+}
+
+mark_deployed() {
+    mark_status "$1" deployed
+}
+
+state_revision() {
+    local service="$1" line value
+    if [[ -r "$STATE_DIR/$service" ]]; then
+        while IFS= read -r line; do
+            case "$line" in
+                revision=*) value="${line#revision=}" ;;
+            esac
+        done < "$STATE_DIR/$service"
+        printf '%s' "${value:-}"
+    fi
+}
+
+state_status() {
+    local service="$1" line value=pending
+    if [[ -r "$STATE_DIR/$service" ]]; then
+        while IFS= read -r line; do
+            case "$line" in
+                status=*) value="${line#status=}" ;;
+            esac
+        done < "$STATE_DIR/$service"
+    fi
+    printf '%s' "$value"
+}
+
+register_changed_services() {
+    local base="$1" service recorded status
+    [[ -n "$base" ]] || return 0
+    while IFS= read -r service; do
+        [[ -n "$service" ]] || continue
+        recorded="$(state_revision "$service")"
+        status="$(state_status "$service")"
+        if [[ "$status" != deployed || "$recorded" != "$REVISION" ]]; then
+            mark_status "$service" pending
+        fi
+    done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
+        --base "$base" --head "$REVISION")
+}
+
+add_changed_services() {
+    local base="$1"
+    local service
+    [[ -n "$base" ]] || return 0
+    while IFS= read -r service; do
+        [[ -n "$service" ]] || continue
+        SERVICES+=("$service")
+    done < <(python3 scripts/deploy-plan.py --target "$TARGET" --repo "$REMOTE_DIR" \
+        --base "$base" --head "$REVISION")
+}
+
+add_pending_services() {
+    local service recorded status changed
+    for state_file in "$STATE_DIR"/*; do
+        [[ -f "$state_file" ]] || continue
+        service="${state_file##*/}"
+        recorded="$(state_revision "$service")"
+        status="$(state_status "$service")"
+        if [[ "$status" != deployed ]]; then
+            # A pending service was intentionally skipped or its handler
+            # failed. Retry it even when the checkout has not advanced.
+            SERVICES+=("$service")
+            continue
+        fi
+        [[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
+        if [[ "$TARGET" == dankclaw && "$service" == frank-default-small ]]; then continue; fi
+        if [[ "$TARGET" == dankserver && "$service" == frank-default-large ]]; then continue; fi
+        if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
+            --repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
+            echo "WARN: unable to compare state for $service; retaining it as pending" >&2
+            SERVICES+=("$service")
+            continue
+        fi
+        if grep -Fxq "$service" <<< "$changed"; then
+            SERVICES+=("$service")
+        fi
+    done
+}
+
+copy_searxng_config() {
+    run_privileged mkdir -p /opt/searxng/config
+    run_privileged cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
+    run_privileged cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
+}
+
+copy_prometheus_config() {
+    run_privileged mkdir -p /opt/prometheus/config
+    run_privileged cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
+}
+
+copy_loki_config() {
+    run_privileged mkdir -p /opt/loki/config
+    run_privileged cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
+}
+
+copy_alloy_config() {
+    run_privileged mkdir -p /opt/alloy/config
+    run_privileged cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
+}
+
+render_caddyfile() {
+    local hash
+    hash="$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
+    if [[ -z "$hash" ]]; then
+        echo "WARN: CADDY_BASICAUTH_HASH is not set; refusing to overwrite rendered Caddyfile" >&2
+        return 1
+    fi
+    run_privileged mkdir -p /home/dank/Containers/caddy
+    sed -e "s|REPLACE_WITH_HASH|$hash|g" dankclaw/Caddyfile > /tmp/serverAppConfigs.Caddyfile
+    run_privileged cp /tmp/serverAppConfigs.Caddyfile /home/dank/Containers/caddy/Caddyfile
+}
+
+deploy_homepage() {
+    if [[ "$TARGET" != dankclaw ]]; then
+        echo "homepage is owned by dankclaw; refusing to deploy it on $TARGET" >&2
+        return 2
+    fi
+    run_privileged mkdir -p /srv/homepage
+    run_privileged cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \
+        shared/homepage/style.css shared/homepage/services.json /srv/homepage/
+    if [[ -d shared/homepage/fonts ]]; then
+        run_privileged cp -r shared/homepage/fonts /srv/homepage/
+    fi
+    local deployed_at
+    deployed_at="$(git log -1 --format=%cI)"
+    printf 'window.DEPLOYED_AT = "%s";\n' "$deployed_at" > /tmp/deployed.js
+    run_privileged cp /tmp/deployed.js /srv/homepage/deployed.js
+    run_privileged chmod -R a+rX /srv/homepage
+    run_privileged cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
+    run_privileged cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
+    run_privileged systemctl daemon-reload
+    run_privileged systemctl enable homepage-status.timer
+    run_privileged systemctl restart homepage-status.timer
+    run_privileged python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
+}
+
+stage_legacy_gitea_caddy_state() {
+    local legacy_state="$STATE_DIR/gitea-caddy"
+    [[ -e "$legacy_state" ]] || return 0
+    local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
+    mkdir -p "$legacy_trash"
+    mv -- "$legacy_state" "$legacy_trash/gitea-caddy.state"
+}
+
+retire_legacy_gitea_caddy() {
+    # The old split proxy owns ports 80/443 on hosts upgraded in place. Stop
+    # and disable its generated unit before the canonical Caddy is started.
+    if systemctl --user is-active --quiet gitea-caddy.service 2>/dev/null; then
+        systemctl --user stop gitea-caddy.service
+    fi
+    if systemctl --user is-enabled --quiet gitea-caddy.service 2>/dev/null; then
+        # A Quadlet-generated unit may report "generated" without having an
+        # enablement symlink. Disable is therefore best-effort; moving the
+        # source out of the generator directory is the durable action.
+        systemctl --user disable gitea-caddy.service 2>/dev/null || true
+    fi
+    local unit_dir="$HOME/.config/containers/systemd"
+    local legacy_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-gitea-caddy"
+    # Podman generators scan the unit directory recursively on this host, so
+    # leaving the retired file in systemd/.AGENTTRASH would regenerate it.
+    # Stage it outside the unit tree in the dated home trash instead.
+    for unit_file in "$unit_dir/gitea-caddy.container" \
+        "$unit_dir/.AGENTTRASH/gitea-caddy.container"; do
+        if [[ -e "$unit_file" ]]; then
+            mkdir -p "$legacy_trash"
+            mv -- "$unit_file" "$legacy_trash/"
+        fi
+    done
+    stage_legacy_gitea_caddy_state
+    systemctl --user daemon-reload
+    if podman container exists gitea-caddy; then
+        podman rm --force gitea-caddy
+    fi
+}
+
+deploy_caddy() {
+    if [[ "$TARGET" != dankclaw ]]; then
+        echo "caddy is owned by dankclaw; refusing to deploy it on $TARGET" >&2
+        return 2
+    fi
+    render_caddyfile
+    deploy_homepage
+    local cfhash oldhash image
+    cfhash="$(sha256sum dankclaw/caddy/Containerfile | cut -d' ' -f1)"
+    image=franksdank/caddy-porkbun
+    oldhash="$(podman image inspect "$image:latest" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
+    if [[ "$oldhash" != "$cfhash" ]]; then
+        podman build --no-cache --label "cfhash=$cfhash" -t "$image:latest" \
+            -f dankclaw/caddy/Containerfile .
+    fi
+    # Retire the old split proxy before claiming ports 80/443. This is
+    # idempotent and leaves the old unit recoverable in the target trash.
+    retire_legacy_gitea_caddy
+    # Caddy's env: Porkbun DNS-01 credentials from the checkout's .env. The
+    # live file holds secrets, so it exists only on the host; the quadlet
+    # reads it via EnvironmentFile.
+    mkdir -p /home/dank/Containers/caddy
+    if [[ ! -f /home/dank/Containers/caddy/env ]]; then
+        awk -F= '/^PORKBUN_API_KEY=|^PORKBUN_API_SECRET_KEY=/ { print }' \
+            dankclaw/.env > /home/dank/Containers/caddy/env
+    fi
+    local var
+    for var in PORKBUN_API_KEY PORKBUN_API_SECRET_KEY; do
+        grep -q "^${var}=." /home/dank/Containers/caddy/env \
+            || { echo "caddy env missing ${var}; set it in dankclaw/.env" >&2; return 1; }
+    done
+    # Retire the bare compose container: all state (Caddyfile, certs, config)
+    # lives in host volumes, so removing the container loses nothing.
+    podman stop caddy >/dev/null 2>&1 || true
+    podman rm caddy >/dev/null 2>&1 || true
+    install_quadlet caddy dankclaw/caddy/quadlet caddy
+    systemctl --user enable caddy.service
+    systemctl --user restart caddy.service
+    if podman container exists caddy; then
+        local reloaded=0
+        for _ in 1 2 3 4 5; do
+            if podman exec caddy caddy reload --config /etc/caddy/Caddyfile \
+                --adapter caddyfile >/dev/null 2>&1; then
+                reloaded=1
+                echo "Caddy reloaded."
+                break
+            fi
+            sleep 3
+        done
+        if [[ "$reloaded" -ne 1 ]]; then
+            echo "Caddy reload failed; restarting only caddy."
+            podman restart caddy
+        fi
+    fi
+}
+
+deploy_compose_service() {
+    local service="$1"
+    if [[ "$TARGET" == dankserver ]]; then
+        # Dankserver's compose file is the source-of-truth inventory, but its
+        # production containers are individually systemd-managed/bare. Do not
+        # use compose up here: it can create a second container with a wrong
+        # name and can interfere with a running workload.
+        echo "WARN: $service is recorded in dankserver/compose.yaml; production container management is host-specific and no restart was attempted."
+        return 0
+    fi
+    case "$service" in
+        comfyui|hermes|hermes-dashboard|hermes-webui|prometheus|grafana|loki|alloy|node-exporter|authentik-postgres|authentik-server|authentik-worker)
+            case "$service" in
+                comfyui|loki|alloy)
+                    build_compose_image "$service"
+                    ;;
+                prometheus)
+                    copy_prometheus_config
+                    ;;
+                loki)
+                    copy_loki_config
+                    ;;
+                alloy)
+                    copy_alloy_config
+                    ;;
+            esac
+            podman-compose -f dankclaw/compose.yaml up -d --no-deps "$service"
+            ;;
+        *)
+            echo "unknown compose service: $service" >&2
+            return 2
+            ;;
+    esac
+}
+
+build_compose_image() {
+    local service="$1" image cfhash oldhash containerfile
+    case "$service" in
+        comfyui)
+            image=comfyui-rocm:gfx1151
+            containerfile=dankclaw/Containerfile.comfyui
+            ;;
+        loki|alloy)
+            image="franksdank/$service:latest"
+            containerfile="dankclaw/$service/Containerfile"
+            ;;
+        *)
+            return 0
+            ;;
+    esac
+    cfhash="$(sha256sum "$containerfile" | cut -d' ' -f1)"
+    oldhash="$(podman image inspect "$image" --format '{{index .Config.Labels "cfhash"}}' 2>/dev/null || echo none)"
+    if [[ "$oldhash" != "$cfhash" ]]; then
+        podman build --no-cache --label "cfhash=$cfhash" -t "$image" -f "$containerfile" .
+    fi
+}
+
+quadlet_unit_dir="$HOME/.config/containers/systemd"
+install_quadlet() {
+    local service="$1"
+    local source_dir="$2"
+    local unit="$3"
+    mkdir -p "$quadlet_unit_dir"
+    cp "$source_dir/$unit.container" "$quadlet_unit_dir/"
+    if [[ -f "$source_dir/$unit.network" ]]; then
+        cp "$source_dir/$unit.network" "$quadlet_unit_dir/"
+    fi
+    systemctl --user daemon-reload
+}
+
+deploy_searxng() {
+    copy_searxng_config
+    install_quadlet searxng dankclaw/searxng/quadlet searxng
+    install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
+    install_quadlet gluetun dankclaw/searxng/quadlet gluetun
+    # The network is shared by all three units; install it even when only the
+    # SearXNG service was selected. These copies are user-owned Quadlet state.
+    cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
+    systemctl --user daemon-reload
+    systemctl --user start searxng-network.service
+    systemctl --user restart gluetun.service searxng-valkey.service searxng.service
+}
+
+deploy_gluetun() {
+    install_quadlet gluetun dankclaw/searxng/quadlet gluetun
+    cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
+    systemctl --user daemon-reload
+    systemctl --user restart gluetun.service searxng.service
+}
+
+deploy_searxng_valkey() {
+    install_quadlet searxng-valkey dankclaw/searxng/quadlet searxng-valkey
+    cp dankclaw/searxng/quadlet/searxng.network "$quadlet_unit_dir/"
+    systemctl --user daemon-reload
+    systemctl --user restart searxng-valkey.service searxng.service
+}
+
+deploy_open_webui() {
+    install_quadlet open-webui dankclaw/open-webui/quadlet open-webui
+    mkdir -p /home/dank/Containers/open-webui/data
+    systemctl --user daemon-reload
+    systemctl --user restart open-webui.service
+}
+
+deploy_pihole() {
+    # Rootless port 53 needs net.ipv4.ip_unprivileged_port_start <= 53, and it
+    # must hold at boot (systemd-sysctl applies /etc/sysctl.d). The repo
+    # drop-in is the single source of truth; a later-sorted file that resets
+    # the value to 80 took LAN DNS down after the 2026-08-26 reboot, so stage
+    # any such conflict in the home trash instead of deleting it.
+    run_privileged install -m 0644 dankclaw/pihole/sysctl/99-dnsmasq.conf \
+        /etc/sysctl.d/99-dnsmasq.conf
+    if grep -qs 'ip_unprivileged_port_start' /etc/sysctl.d/99-rootless-ports.conf; then
+        local sysctl_trash="$HOME/.AGENTTRASH/$(date +%Y%m%d)-sysctl-rootless-ports"
+        mkdir -p "$sysctl_trash"
+        run_privileged mv /etc/sysctl.d/99-rootless-ports.conf "$sysctl_trash/"
+    fi
+    run_privileged sysctl --system >/dev/null
+    mkdir -p /home/dank/Containers/pihole/etc-pihole /home/dank/Containers/pihole/etc-dockers
+    if [[ ! -f /home/dank/Containers/pihole/env ]]; then
+        cp dankclaw/pihole/env.example /home/dank/Containers/pihole/env
+        echo "WARN: created Pi-hole env from the example; set WEBPASSWORD before using its UI" >&2
+    fi
+    cp dankclaw/pihole/addn-hosts /home/dank/Containers/pihole/addn-hosts
+    mkdir -p "$quadlet_unit_dir"
+    cp dankclaw/pihole/quadlet/pihole.container "$quadlet_unit_dir/"
+    systemctl --user daemon-reload
+    systemctl --user restart pihole.service
+}
+
+deploy_gitea_unit() {
+    local service="$1"
+    local unit="$2"
+    mkdir -p "$quadlet_unit_dir"
+    cp "dankclaw/gitea/quadlet/$unit.container" "$quadlet_unit_dir/"
+    [[ -f dankclaw/gitea/quadlet/gitea.network ]] && cp dankclaw/gitea/quadlet/gitea.network "$quadlet_unit_dir/"
+    systemctl --user daemon-reload
+    systemctl --user restart "$service.service"
+}
+
+relink_unit() {
+    local service="$1"
+    run_privileged ln -sfn "$REMOTE_DIR/scripts/$service.service" "/etc/systemd/system/$service.service"
+}
+
+deploy_llm_large() {
+    relink_unit frank-default-large
+    # NPU is intentionally not restarted here. It is a separate, retired
+    # endpoint and its unit is only relinked when explicitly selected.
+    run_privileged systemctl daemon-reload
+    run_privileged systemctl enable frank-default-large
+    run_privileged systemctl restart frank-default-large
+    wait_for_health frank-default-large http://127.0.0.1:7770/health 1200
+}
+
+deploy_llm_small() {
+    relink_unit frank-default-small
+    run_privileged systemctl daemon-reload
+    run_privileged systemctl enable frank-default-small
+    run_privileged systemctl restart frank-default-small
+    wait_for_health frank-default-small http://127.0.0.1:7770/health 1200
+}
+
+deploy_llm_npu() {
+    relink_unit frank-default-npu
+    run_privileged systemctl daemon-reload
+    echo "Relinked frank-default-npu; it was not restarted."
+}
+
+wait_for_health() {
+    local service="$1" url="$2" limit="$3"
+    local i
+    for ((i=1; i<=limit; i++)); do
+        if curl -fsS "$url" >/dev/null 2>&1; then
+            echo "$service healthy after approximately $((i * 2))s"
+            return 0
+        fi
+        sleep 2
+    done
+    echo "FAIL: $service did not become healthy in $((limit * 2))s" >&2
+    return 1
+}
+
+deploy_service() {
+    local service="$1"
+    case "$service" in
+        caddy) deploy_caddy ;;
+        homepage) deploy_homepage ;;
+        searxng) deploy_searxng ;;
+        gluetun) deploy_gluetun ;;
+        searxng-valkey) deploy_searxng_valkey ;;
+        open-webui) deploy_open_webui ;;
+        pihole) deploy_pihole ;;
+        gitea-postgres) deploy_gitea_unit gitea-postgres gitea-postgres ;;
+        gitea-server) deploy_gitea_unit gitea-server gitea-server ;;
+        frank-default-large) deploy_llm_large ;;
+        frank-default-small) deploy_llm_small ;;
+        frank-default-npu) deploy_llm_npu ;;
+        *) deploy_compose_service "$service" ;;
+    esac
+}
+
+if [[ -n "$FROM_REVISION" ]]; then
+    # Register every changed service before applying the requested scope. This
+    # preserves an explicit --service selection as pending work for the next
+    # ordinary deployment.
+    register_changed_services "$FROM_REVISION"
+fi
+if [[ "$PENDING" -eq 1 ]]; then
+    add_changed_services "$FROM_REVISION"
+    add_pending_services
+fi
+
+# Remove duplicate service names while preserving caller/planner order.
+UNIQUE_SERVICES=()
+declare -A SEEN_SERVICES=()
+for service in "${SERVICES[@]}"; do
+    [[ -n "$service" ]] || continue
+    if [[ -z "${SEEN_SERVICES[$service]:-}" ]]; then
+        UNIQUE_SERVICES+=("$service")
+        SEEN_SERVICES[$service]=1
+    fi
+done
+SERVICES=("${UNIQUE_SERVICES[@]}")
+
+if [[ "${#SERVICES[@]}" -eq 0 ]]; then
+    echo "No services selected; nothing to deploy."
+    exit 0
+fi
+
+for service in "${SERVICES[@]}"; do
+    [[ "$service" =~ ^[a-z0-9][a-z0-9-]*$ ]] || { echo "invalid service: $service" >&2; exit 2; }
+    service_is_allowed "$service" || { echo "service $service is not deployed on $TARGET" >&2; exit 2; }
+    echo "=== Deploying $service at $REVISION ==="
+    mark_status "$service" pending
+    deploy_service "$service"
+    mark_deployed "$service"
+    echo "  STATE[$service] = $REVISION (status=deployed)"
+done

scripts/deploy-searxng-quadlet.sh

@@ -1,97 +1,16 @@
 #!/usr/bin/env bash
-# deploy-searxng-quadlet.sh — Install and (re)start the SearXNG quadlet stack.
+#!/usr/bin/env bash
-#
+# Compatibility wrapper for the service-scoped deployment dispatcher.
-# The gluetun + searxng + valkey trio is boot-persistent via Quadlet, not
+# Use deploy.sh for Git push/pull and planning; this name remains for operators
-# podman-compose (see AGENTS.md). This script:
+# and runbooks that historically deployed the Quadlet trio directly.
-#   1. Pushes main to origin
-#   2. Pulls the repo on the target
-#   3. Copies the quadlet units into ~/.config/containers/systemd/
-#   4. daemon-reloads, enables the units for boot, and starts them in order
-#
-# It is idempotent: it stops any running units and removes any containers with
-# the trio's names (compose leftovers or the previous quadlet run) before
-# reinstalling and starting, so it is safe to run on first migration and on
-# every later config change.
-#
-# Usage:
-#   ./scripts/deploy-searxng-quadlet.sh <target> [--dry-run]
-#   ./scripts/deploy-searxng-quadlet.sh dankclaw
 set -euo pipefail
 
-REMOTE_USER="dank"
-REMOTE_HOST="dankclaw"
-REMOTE_DIR="/home/dank/serverAppConfigs"
-QDIR="dankclaw/searxng/quadlet"
-UNITS="searxng-network.service gluetun.service searxng-valkey.service searxng.service"
-
 TARGET="${1:-}"
-DRY_RUN=0
+shift || true
-if [ "${2:-}" = "--dry-run" ]; then
+case "$TARGET" in
-    DRY_RUN=1
+    dankclaw) exec "$(dirname "$0")/deploy.sh" dankclaw --service searxng "$@" ;;
-fi
+    *)
-
+        echo "This stack runs only on dankclaw (got: ${TARGET:-<missing>})" >&2
-if [ -z "${TARGET:-}" ]; then
+        exit 2
-    echo "Usage: $0 <target> [--dry-run]"
+        ;;
-    exit 1
+ esac
-fi
-if [ "${TARGET}" != "dankclaw" ]; then
-    echo "This stack runs only on dankclaw (got: ${TARGET})"
-    exit 1
-fi
-
-# ---------------------------------------------------------------------------
-# Local pre-flight
-# ---------------------------------------------------------------------------
-cd "$(dirname "$0")/.."
-if ! git diff --quiet HEAD; then
-    echo "FAIL: Local working tree has uncommitted changes. Commit first."
-    exit 1
-fi
-
-if [ "$DRY_RUN" = "1" ]; then
-    echo "[dry-run] Would push main, pull on ${REMOTE_HOST}, install quadlet units, and start: ${UNITS}"
-    exit 0
-fi
-
-echo "=== Deploying SearXNG quadlet stack to ${REMOTE_HOST} ==="
-echo "[1/3] Pushing local main to origin..."
-git push origin main
-
-REMOTE="${REMOTE_USER}@${REMOTE_HOST}"
-
-echo "[2/3] Installing and starting the quadlet units on ${REMOTE_HOST}..."
-ssh "$REMOTE" bash -s <<REMOTE_EOF
-set -euo pipefail
-cd '${REMOTE_DIR}'
-
-echo '  Pulling repo...'
-git pull origin main
-
-echo '  Stopping any running units (idempotent)...'
-systemctl --user stop searxng.service gluetun.service searxng-valkey.service searxng-network.service 2>/dev/null || true
-
-echo '  Removing any containers with the trio names (compose leftovers or prior run)...'
-podman rm -f searxng searxng-valkey gluetun 2>/dev/null || true
-
-echo '  Copying quadlet units into ~/.config/containers/systemd/ ...'
-mkdir -p ~/.config/containers/systemd
-cp ${QDIR}/*.container ${QDIR}/*.network ~/.config/containers/systemd/
-
-echo '  daemon-reload (generates the units)...'
-# Quadlet units are generator-managed: systemctl enable fails on them ("transient
-# or generated"). The [Install] WantedBy=default.target in each .container file is
-# itself the enablement (is-enabled reports "generated"), and with user linger on
-# they start at boot. The .network unit is pulled in by the containers that set
-# Wants=/After=searxng-network.service. So: daemon-reload, then start.
-systemctl --user daemon-reload
-
-echo '  Starting in dependency order...'
-systemctl --user start searxng-network.service
-systemctl --user start gluetun.service searxng-valkey.service
-systemctl --user start searxng.service
-REMOTE_EOF
-
-echo "[3/3] Verifying unit state..."
-ssh "$REMOTE" 'systemctl --user --no-pager status searxng-network.service gluetun.service searxng-valkey.service searxng.service 2>&1 | grep -E "●|Active:"' || true
-
-echo "=== SearXNG quadlet stack deployed to ${REMOTE_HOST} ==="

scripts/deploy.sh

@@ -1,302 +1,215 @@
 #!/usr/bin/env bash
-# deploy.sh — Deploy the serverAppConfigs repo to a target and restart its
+# deploy.sh — push main, pull one target, and deploy only selected services.
-# podman stack and LLM fleet services.
 #
-# Implements the git deployment workflow from AGENTS.md. Assumes local
+# Planning is path-aware and happens before the push. The target-side runner
-# changes are committed and pushed, then:
+# then applies only those services, with per-service deployment revisions kept
-#   1. Pushes main to origin
+# outside the Git checkout. A docs-only commit is a no-op. An LLM service is
-#   2. Pulls the repo on the target
+# restarted only when its own files (or the shared chat template) are selected.
-#   3. Copies shared configs into place and restarts the podman stack
-#      (dankclaw only; the dankserver media stack runs as bare containers)
-#   4. Relinks the LLM systemd units and restarts the active LLM service
-#   5. Waits for the Caddy homepage and the LLM health endpoint
-#
-# Sudo: the fleet hosts allow passwordless sudo for the deploy user. For
-# headless runs on a host without that, set FRANK_SUDO_PW so remote sudo
-# reads the password from stdin (`sudo -S`). The password is never stored
-# in this repo.
 #
 # Usage:
-#   ./scripts/deploy.sh <target> [--dry-run]
+#   ./scripts/deploy.sh <dankclaw|dankserver> [--service NAME]... [--dry-run]
 #
-# Targets:
+# Without --service, the planner selects services changed since origin/main and
-#   dankclaw   - AI/search/dev/observability stack + frank-default-large
+# the remote runner also applies pending service changes from its state store.
-#                (CIRU ROCmFPX llama.cpp, Qwen3.8-27B, port 7770)
+# --service intentionally restricts the run to the named services; it is useful
-#   dankserver - media/library stack (bare containers) + frank-default-small
+# for a safe, explicit retry after another service failed.
-#                (upstream llama.cpp, Qwen3.6-35B-A3B, port 7770)
-#
-# Example:
-#   ./scripts/deploy.sh dankclaw
-#   ./scripts/deploy.sh dankserver --dry-run
 set -euo pipefail
 
-# ---------------------------------------------------------------------------
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# Target configuration
+REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." && pwd)"
-# ---------------------------------------------------------------------------
+PLANNER="$SCRIPT_DIR/deploy-plan.py"
-declare -A REMOTE_USER REMOTE_HOST REMOTE_DIR COMPOSE HEALTH_URL COMPOSE_UP COPY_SHARED
+REMOTE_USER="dank"
-declare -A LLM_SERVICES LLM_UNITS LLM_HEALTH LLM_WAIT
+REMOTE_DIR="/home/dank/serverAppConfigs"
 
-# Caddy is consolidated on dankclaw: it fronts both hosts' services. The
+usage() {
-# dankserver media stack runs as bare containers (not this repo's compose),
+    cat <<'EOF'
-# so its stack deploy is pull-only; health is checked against the shared Caddy.
+Usage: deploy.sh <dankclaw|dankserver> [options]
-REMOTE_USER[dankclaw]=dank
-REMOTE_HOST[dankclaw]=dankclaw
-REMOTE_DIR[dankclaw]="/home/dank/serverAppConfigs"
-COMPOSE[dankclaw]="dankclaw/compose.yaml"
-HEALTH_URL[dankclaw]="http://127.0.0.1/"
-COMPOSE_UP[dankclaw]=1
-COPY_SHARED[dankclaw]=1
-# LLM: frank-default-large is the active service (restart + health wait).
-# frank-default-npu is relinked only (not enabled on the host); its unit
-# picks up the new paths at its next manual start.
-LLM_SERVICES[dankclaw]="frank-default-large"
-LLM_UNITS[dankclaw]="frank-default-large frank-default-npu"
-LLM_HEALTH[dankclaw]="http://127.0.0.1:7770/health"
-# CIRU llama.cpp (ROCm 7.15) loads the v3 five-file payload (GGUF + FFN +
-# GDN + Output-K8 .pfs + mmproj, ~36.7 GiB) + MTP draft/Output-K8 proxy and
-# initializes PromptForge views; allow 20 minutes.
-LLM_WAIT[dankclaw]=600
 
-REMOTE_USER[dankserver]=dank
+Options:
-REMOTE_HOST[dankserver]=dankserver
+  --service NAME  deploy only NAME (repeatable; bypasses automatic planning)
-REMOTE_DIR[dankserver]="/home/dank/serverAppConfigs"
+  --dry-run       show the plan without pushing or connecting to a target
-COMPOSE[dankserver]="dankserver/compose.yaml"
+  -h, --help      show this help
-HEALTH_URL[dankserver]="http://192.168.0.143/"
+
-COMPOSE_UP[dankserver]=0
+Examples:
-COPY_SHARED[dankserver]=0
+  ./scripts/deploy.sh dankclaw
-# LLM: frank-default-small is the active service (restart + health wait).
+  ./scripts/deploy.sh dankclaw --service searxng
-LLM_SERVICES[dankserver]="frank-default-small"
+  ./scripts/deploy.sh dankserver --service frank-default-small
-LLM_UNITS[dankserver]="frank-default-small"
+  ./scripts/deploy.sh dankclaw --dry-run
-LLM_HEALTH[dankserver]="http://127.0.0.1:7770/health"
+EOF
-# Cold load: --no-mmap reads the 30.6 GB Q6_K_P through RAM (MoE experts on
+}
-# CPU); allow 20 minutes.
-LLM_WAIT[dankserver]=600
 
 TARGET="${1:-}"
+if [[ "$TARGET" == "-h" || "$TARGET" == "--help" ]]; then
+    usage
+    exit 0
+fi
+if [[ -z "$TARGET" ]]; then
+    usage >&2
+    exit 2
+fi
+shift
+
+case "$TARGET" in
+    dankclaw) REMOTE_HOST="dankclaw" ;;
+    dankserver) REMOTE_HOST="dankserver" ;;
+    *)
+        echo "Unknown target: $TARGET" >&2
+        usage >&2
+        exit 2
+        ;;
+esac
+
 DRY_RUN=0
-if [ "${2:-}" = "--dry-run" ]; then
+EXPLICIT_SERVICES=()
-    DRY_RUN=1
+while [[ "$#" -gt 0 ]]; do
-fi
+    case "$1" in
-
+        --dry-run)
-# ---------------------------------------------------------------------------
+            DRY_RUN=1
-# Validation
+            shift
-# ---------------------------------------------------------------------------
+            ;;
-if [ -z "${TARGET:-}" ]; then
+        --service)
-    echo "Usage: $0 <dankclaw|dankserver> [--dry-run]"
+            [[ "$#" -ge 2 ]] || { echo "--service requires a name" >&2; exit 2; }
-    exit 1
+            EXPLICIT_SERVICES+=("$2")
-fi
+            shift 2
-if [ -z "${REMOTE_HOST[$TARGET]:-}" ]; then
+            ;;
-    echo "Unknown target: $TARGET"
+        -h|--help)
-    exit 1
+            usage
-fi
-
-# ---------------------------------------------------------------------------
-# Sudo wrapper (headless-friendly)
-# ---------------------------------------------------------------------------
-SUDO="sudo"
-if [ -n "${FRANK_SUDO_PW:-}" ]; then
-    SUDO="printf '%s\n' '$FRANK_SUDO_PW' | sudo -S"
-fi
-
-# ---------------------------------------------------------------------------
-# Local checks (pre-flight)
-# ---------------------------------------------------------------------------
-cd "$(dirname "$0")/.."
-if ! git diff --quiet HEAD; then
-    echo "FAIL: Local working tree has uncommitted changes."
-    echo "Commit or stash them before deploying."
-    exit 1
-fi
-if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then
-    echo "WARN: Not on 'main'; deploying a non-main branch."
-fi
-
-echo "=== Deploying to ${TARGET} ==="
-echo "  host:       ${REMOTE_HOST[$TARGET]}"
-echo "  dir:        ${REMOTE_DIR[$TARGET]}"
-echo "  compose:    ${COMPOSE[$TARGET]}"
-echo "  llm svc:    ${LLM_SERVICES[$TARGET]}"
-echo "  llm health: ${LLM_HEALTH[$TARGET]}"
-echo "  homepage:   ${HEALTH_URL[$TARGET]}"
-echo ""
-
-# ---------------------------------------------------------------------------
-# Push (unless dry-run)
-# ---------------------------------------------------------------------------
-if [ "$DRY_RUN" = "1" ]; then
-    echo "[dry-run] Would push local main to origin"
-    echo "[dry-run] Would SSH to ${REMOTE_HOST[$TARGET]} and git pull"
-else
-    echo "[1/5] Pushing local main to origin..."
-    git push origin main
-fi
-
-REMOTE="$(
-    printf '%s@%s' "${REMOTE_USER[$TARGET]}" "${REMOTE_HOST[$TARGET]}"
-)"
-
-# ---------------------------------------------------------------------------
-# Remote: pull + copy shared configs + restart stack + restart LLM service
-# ---------------------------------------------------------------------------
-REMOTE_SCRIPT="
-set -euo pipefail
-cd '${REMOTE_DIR[$TARGET]}'
-
-echo '[2/5] Pulling repo on ${TARGET}...'
-git pull origin main
-
-if [ -f .env ]; then
-  echo '  .env present, using it'
-else
-  echo '  WARN: no .env on target; copy the .env.example and fill it in'
-fi
-
-if [ '${COPY_SHARED[$TARGET]}' = '1' ]; then
-  echo '[3/5] Copying shared configs into place...'
-  mkdir -p /opt/searxng/config /opt/prometheus/config /opt/loki/config /opt/alloy/config
-  cp shared/searxng/settings.yml /opt/searxng/config/settings.yml
-  cp shared/searxng/limiter.toml /opt/searxng/config/limiter.toml
-  cp shared/prometheus/prometheus.yml /opt/prometheus/config/prometheus.yml
-  cp shared/loki/loki-config.yaml /opt/loki/config/loki-config.yaml
-  cp shared/alloy/config.alloy /opt/alloy/config/config.alloy
-
-  echo '  Rendering Caddyfile (basicauth hash from .env)...'
-  # Extract the hash with grep/cut rather than sourcing .env: sourcing a
-  # bcrypt value in a set -u shell dies on its dollar-sign expansion.
-  # '|| true' so a missing hash warns below instead of aborting under set -e.
-  # Only the shared family hash remains: the homepage's per-user hashes were
-  # retired in favor of Authentik forward_auth (the /admin/ page and the user
-  # page are now behind the same SSO session).
-  CADDY_BASICAUTH_HASH=\"\$(grep -E '^CADDY_BASICAUTH_HASH=' .env 2>/dev/null | tail -1 | cut -d= -f2- || true)\"
-  if [ -n \"\${CADDY_BASICAUTH_HASH}\" ]; then
-    # '|' delimiter: the bcrypt hash contains '/' characters.
-    mkdir -p /home/dank/Containers/caddy
-    sed -e \"s|REPLACE_WITH_HASH|\${CADDY_BASICAUTH_HASH}|g\" \\
-      dankclaw/Caddyfile > /home/dank/Containers/caddy/Caddyfile
-  else
-    echo '  WARN: CADDY_BASICAUTH_HASH is not set in .env; keeping the placeholder(s)'
-  fi
-
-  echo '  Deploying the homepage...'
-  ${SUDO} mkdir -p /srv/homepage
-  ${SUDO} cp shared/homepage/index.html shared/homepage/admin.html shared/homepage/app.js \\
-     shared/homepage/style.css shared/homepage/services.json /srv/homepage/
-  if [ -d shared/homepage/fonts ]; then ${SUDO} cp -r shared/homepage/fonts /srv/homepage/; fi
-  # The last master deployment datetime (Q43): the deployed commit's datetime.
-  DEPLOYED_AT=\"\$(git log -1 --format=%cI)\"
-  printf 'window.DEPLOYED_AT = \"%s\";\\n' \"\$DEPLOYED_AT\" > /tmp/deployed.js
-  ${SUDO} cp /tmp/deployed.js /srv/homepage/deployed.js
-  ${SUDO} chmod -R a+rX /srv/homepage
-  # The status checker: install the timer + service, run it once, enable it.
-  ${SUDO} cp scripts/homepage-status.py /usr/local/bin/homepage-status.py
-  ${SUDO} cp scripts/homepage-status.service scripts/homepage-status.timer /etc/systemd/system/
-  ${SUDO} systemctl daemon-reload
-  ${SUDO} systemctl enable homepage-status.timer
-  ${SUDO} systemctl restart homepage-status.timer
-  ${SUDO} python3 /usr/local/bin/homepage-status.py /srv/homepage/services.json /srv/homepage/status.json
-fi
-
-if [ '${COMPOSE_UP[$TARGET]}' = '1' ]; then
-  echo '[4/5] Restarting stack...'
-  # Locally built images: caddy + porkbun module, and loki/alloy + a static
-  # busybox (their upstream images lack the wget the healthchecks need).
-  for dir in caddy loki alloy; do
-    img=franksdank/\$dir
-    [ \"\$dir\" = caddy ] && img=franksdank/caddy-porkbun
-    # Rebuild when the Containerfile changed. buildah's layer cache can
-    # serve a stale COPY --from layer (keyed on stage name), so a rebuild
-    # always runs with --no-cache.
-    cfhash=\$(sha256sum dankclaw/\$dir/Containerfile | cut -d' ' -f1)
-    oldhash=\$(podman image inspect \$img:latest --format '{{index .Config.Labels \"cfhash\"}}' 2>/dev/null || echo none)
-    if [ \"\$oldhash\" != \"\$cfhash\" ]; then
-      echo \"  \$img image missing or stale; building it...\"
-      podman build --no-cache --label cfhash=\$cfhash -t \$img:latest -f dankclaw/\$dir/Containerfile .
-    fi
-  done
-  podman-compose -f '${COMPOSE[$TARGET]}' up -d
-
-  # A Caddyfile render is a bind-mount content change, not a container
-  # config change: the caddy container often survives 'up -d' and keeps
-  # serving its in-memory config. Reload explicitly; the file watcher
-  # (admin on) is the backstop if the API is not up yet. If the reload
-  # cannot be applied at all (e.g. the running config has the admin API
-  # disabled -- the fix cannot ride the API it is meant to enable),
-  # restart the container so it reads the file on start.
-  caddy_reloaded=0
-  for i in 1 2 3 4 5; do
-    if podman exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null 2>&1; then
-      caddy_reloaded=1
-      echo \"  Caddy reloaded.\"
-      break
-    fi
-    sleep 3
-  done
-  if [ \"\$caddy_reloaded\" -ne 1 ]; then
-    echo \"  Caddy reload failed; restarting the container to load the file.\"
-    podman restart caddy
-  fi
-else
-  echo '[4/5] Skipping compose restart (stack runs as bare containers on this host)'
-fi
-
-echo '[4/5] Relinking LLM units and restarting LLM services...'
-for unit in ${LLM_UNITS[$TARGET]}; do
-  # The loop variable must expand on the remote, so it is escaped here and
-  # the path is unquoted (it has no spaces). Quoting the source path would
-  # ship a literal loop-variable token and create a dangling symlink
-  # (the 2026-08-22 relink bug).
-  ${SUDO} ln -sf ${REMOTE_DIR[$TARGET]}/scripts/\${unit}.service /etc/systemd/system/\${unit}.service
-done
-${SUDO} systemctl daemon-reload
-for svc in ${LLM_SERVICES[$TARGET]}; do
-  ${SUDO} systemctl enable \"\${svc}\"
-  ${SUDO} systemctl restart \"\${svc}\"
-  echo \"  restarted \${svc}\"
-done
-"
-
-if [ "$DRY_RUN" = "1" ]; then
-    echo "[dry-run] Remote commands would run:"
-    echo "$REMOTE_SCRIPT"
-else
-    # shellcheck disable=SC2087
-    ssh "$REMOTE" bash -s <<REMOTE_EOF
-${REMOTE_SCRIPT}
-REMOTE_EOF
-fi
-
-# ---------------------------------------------------------------------------
-# Health checks: homepage, then the LLM endpoint
-# ---------------------------------------------------------------------------
-if [ "$DRY_RUN" = "1" ]; then
-    echo "[dry-run] Would wait for health at ${HEALTH_URL[$TARGET]} and ${LLM_HEALTH[$TARGET]}"
-else
-    echo "[5/5] Waiting for homepage ${HEALTH_URL[$TARGET]} (via ssh)..."
-    HEALTH="${HEALTH_URL[$TARGET]}"
-    for i in $(seq 1 60); do
-        if ssh "$REMOTE" "curl -sf '$HEALTH'" > /dev/null 2>&1; then
-            echo "  Homepage healthy after approx $((i * 2))s"
-            break
-        fi
-        sleep 2
-    done
-    if ! ssh "$REMOTE" "curl -sf '$HEALTH'" > /dev/null 2>&1; then
-        echo "FAIL: Homepage did not become healthy in 120s"
-        exit 1
-    fi
-
-    echo "[5/5] Waiting for LLM health ${LLM_HEALTH[$TARGET]} (via ssh)..."
-    LLM_URL="${LLM_HEALTH[$TARGET]}"
-    LLM_ITERS="${LLM_WAIT[$TARGET]}"
-    for i in $(seq 1 "$LLM_ITERS"); do
-        if ssh "$REMOTE" "curl -sf '$LLM_URL'" > /dev/null 2>&1; then
-            echo "  LLM healthy after approx $((i * 2))s"
-            echo ""
-            echo "=== Deploy complete: ${TARGET} ==="
             exit 0
-        fi
+            ;;
-        sleep 2
+        *)
-    done
+            echo "Unknown option: $1" >&2
-    echo "FAIL: LLM service did not become healthy in $((LLM_ITERS * 2))s"
+            usage >&2
+            exit 2
+            ;;
+    esac
+done
+
+cd "$REPO_ROOT"
+if [[ ! -x "$PLANNER" ]]; then
+    echo "FAIL: missing executable planner: $PLANNER" >&2
     exit 1
 fi
+
+# A deployment is a release operation, not a way to publish an arbitrary
+# worktree. Include untracked files in this guard; the old diff-only check let
+# an untracked secret or half-written config pass the pre-flight.
+if [[ -n "$(git status --porcelain)" ]]; then
+    echo "FAIL: Local working tree has uncommitted changes." >&2
+    echo "Commit or remove them before deploying." >&2
+    exit 1
+fi
+if [[ "$(git branch --show-current)" != "main" ]]; then
+    echo "FAIL: deploy.sh must run from the main branch." >&2
+    exit 1
+fi
+
+HEAD_REVISION="$(git rev-parse HEAD)"
+if BASE_REVISION="$(git rev-parse origin/main 2>/dev/null)"; then
+    :
+else
+    BASE_REVISION="$(git rev-parse HEAD^ 2>/dev/null || git rev-parse HEAD)"
+fi
+
+valid_service() {
+    local service="$1"
+    case "$TARGET:$service" in
+        dankclaw:gluetun|dankclaw:searxng-network|dankclaw:searxng-valkey|dankclaw:searxng|\
+        dankclaw:pihole|dankclaw:open-webui|dankclaw:gitea-postgres|dankclaw:gitea-server|\
+        dankclaw:comfyui|dankclaw:hermes|dankclaw:hermes-dashboard|\
+        dankclaw:hermes-webui|dankclaw:prometheus|dankclaw:grafana|dankclaw:loki|dankclaw:alloy|\
+        dankclaw:node-exporter|dankclaw:authentik-postgres|dankclaw:authentik-server|\
+        dankclaw:authentik-worker|dankclaw:caddy|dankclaw:homepage|\
+        dankclaw:frank-default-large|dankclaw:frank-default-npu|\
+        dankserver:mariadb|dankserver:redis|dankserver:romm|dankserver:copyparty|\
+        dankserver:jellyfin|dankserver:calibre-web|dankserver:prowlarr|\
+        dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
+        dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
+        dankserver:frank-default-small)
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+for service in "${EXPLICIT_SERVICES[@]}"; do
+    if ! valid_service "$service"; then
+        echo "Service '$service' is not deployed on $TARGET." >&2
+        exit 2
+    fi
+done
+
+PLAN=()
+if [[ "${#EXPLICIT_SERVICES[@]}" -gt 0 ]]; then
+    PLAN=("${EXPLICIT_SERVICES[@]}")
+else
+    mapfile -t PLAN < <(
+        python3 "$PLANNER" --target "$TARGET" --repo "$REPO_ROOT" \
+            --base "$BASE_REVISION" --head "$HEAD_REVISION"
+    )
+fi
+
+# Preserve planner order while removing duplicate explicit service names.
+UNIQUE_PLAN=()
+declare -A SEEN=()
+for service in "${PLAN[@]}"; do
+    [[ -n "$service" ]] || continue
+    if [[ -z "${SEEN[$service]:-}" ]]; then
+        UNIQUE_PLAN+=("$service")
+        SEEN[$service]=1
+    fi
+done
+PLAN=("${UNIQUE_PLAN[@]}")
+
+printf '=== Deployment plan: %s ===\n' "$TARGET"
+printf '  checkout: %s\n' "$HEAD_REVISION"
+if [[ "${#PLAN[@]}" -eq 0 ]]; then
+    echo "  changed services: (none; the remote will check pending service state)"
+else
+    printf '  changed services:\n'
+    printf '    - %s\n' "${PLAN[@]}"
+fi
+
+REMOTE="${REMOTE_USER}@${REMOTE_HOST}"
+REMOTE_ARGS=("$TARGET" "$HEAD_REVISION")
+if [[ "${#EXPLICIT_SERVICES[@]}" -eq 0 ]]; then
+    REMOTE_ARGS+=(--pending)
+fi
+REMOTE_ARGS+=("${PLAN[@]}")
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+    echo "[dry-run] Would push main to origin."
+    echo "[dry-run] Would SSH to $REMOTE, pull with --ff-only, and run the service-scoped runner."
+    printf '[dry-run] Runner arguments:'
+    printf ' %q' "${REMOTE_ARGS[@]}"
+    printf '\n'
+    exit 0
+fi
+
+echo "[1/2] Pushing main to origin..."
+git push origin main
+
+echo "[2/2] Pulling and deploying on $REMOTE_HOST..."
+# The target runner computes any pending revisions from its own state after
+# the pull. This keeps a target that missed an earlier service deployment
+# recoverable without restarting services that are already current.
+ssh "$REMOTE" bash -s -- "${REMOTE_ARGS[@]}" <<'REMOTE_EOF'
+set -euo pipefail
+TARGET="$1"
+EXPECTED_REVISION="$2"
+shift 2
+cd "/home/dank/serverAppConfigs"
+OLD_REVISION="$(git rev-parse HEAD)"
+echo "  pulling origin/main..."
+git pull --ff-only origin main
+NEW_REVISION="$(git rev-parse HEAD)"
+if [[ "$NEW_REVISION" != "$EXPECTED_REVISION" ]]; then
+    echo "FAIL: target checkout is $NEW_REVISION, expected $EXPECTED_REVISION" >&2
+    exit 1
+fi
+exec ./scripts/deploy-remote.sh "$TARGET" "$NEW_REVISION" "$@" \
+    --from-revision "$OLD_REVISION"
+REMOTE_EOF
+
+echo "=== Deploy complete: $TARGET ==="

scripts/download-q8agent.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Download the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT payload for
+# frank-default-large from HuggingFace into /home/dank/models/.../Q8_0-AGENT.
+#
+# Uses the token in ~/.cache/huggingface/token so the ~28 GB main file does not
+# hit the anonymous 10 GB/30 min rate limit. Run on Dankclaw.
+#
+#   bash scripts/download-q8agent.sh
+#   bash scripts/download-q8agent.sh --verify-only
+set -euo pipefail
+
+REPO="kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF"
+DIR="/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT"
+EXPECTED=(
+  "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf:28214425952"
+  "mmproj-Qwen3.8-27B-BF16.gguf:931145888"
+  "mtp-Qwen3.8-27B-Q4_0.gguf:1680271648"
+)
+
+cd -- "$(dirname -- "$0")/.."
+
+main() {
+  local mode="download"
+  [[ "${1:-}" == "--verify-only" ]] && mode="verify"
+  mkdir -p "$DIR"
+  cd "$DIR"
+
+  local token
+  token="$(cat "$HOME/.cache/huggingface/token")"
+  [[ -n "$token" ]] || { echo "no HF token in $HOME/.cache/huggingface/token" >&2; exit 1; }
+
+  dl() {
+    local f="$1"
+    curl -fsSL -C - -o "$f" --retry 5 --retry-delay 3 --retry-max-time 3600 \
+      -H "Authorization: Bearer $token" \
+      "https://huggingface.co/$REPO/resolve/main/$f"
+    echo "  done: $f ($(du -h "$f" | cut -f1))"
+  }
+
+  if [[ "$mode" == "download" ]]; then
+    echo "downloading $REPO into $DIR"
+    dl "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf"
+    dl "mmproj-Qwen3.8-27B-BF16.gguf"
+    dl "mtp-Qwen3.8-27B-Q4_0.gguf"
+  fi
+
+  echo "verifying sizes"
+  local ok=1 f want act
+  for entry in "${EXPECTED[@]}"; do
+    f="${entry%%:*}"; want="${entry##*:}"
+    act="$(stat -c%s "$f")"
+    if [[ "$act" -eq "$want" ]]; then
+      echo "  OK   $f ($act bytes)"
+    else
+      echo "  BAD  $f ($act bytes, want $want)" >&2
+      ok=0
+    fi
+  done
+  [[ "$ok" -eq 1 ]] || exit 1
+  echo "all files present and sized correctly"
+}
+
+main "$@"

scripts/frank-default-large.env

@@ -1,73 +1,80 @@
-# Tunables for frank-default-large (CIRU llama.cpp, dankclaw, Strix Halo
+# Tunables for frank-default-large (kingjones777 Qwen3.8-Flash-Next
-# gfx1151).
+# ROCmFP4 STRIX_LEAN, qwen4exp hybrid MoE, Dankclaw, Strix Halo gfx1151).
 #
-# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag
+# Runtime: ROCmFPX llama.cpp fork with PR #27742 (qwen4exp architecture)
-# qwen3.8-activefpx-promptforge-v2.3 plus the v3 output-K8 runtime patch
+# merged and built at /home/dank/GitPackages/ROCmFPX/build-qwen38.
-# serving kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4 (dense, Q6 AGENT
+# Build flags: -DGGML_HIP=ON -DGGML_HIP_ROCMI4_W4A4=ON
-# GGUF) with the standalone MTP Q4_0 draft head. Alias frank-default-large,
+# -DCMAKE_HIP_ARCHITECTURES=gfx1151 -DGGML_CUDA_FA_ALL_QUANTS=ON
-# port 7770.
+# -DLLAMA_BUILD_WEBUI=OFF -DCMAKE_EXE_LINKER_FLAGS=-no-pie.
+# Replaced the kingjones Qwen3.8-27B Q6_0 AGENT deployment on 2026-09-01.
 #
-# This model replaced jcbtc/Qwen3.8-27B-CIRU-ActiveFPX-PromptForge (v3) on
+# Model: kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF.
-# 2026-08-24. The new model is a plain ROCmFP4 GGUF (no PromptForge .pfs
+# 56B MoE (512 experts, 10 used), 48 layers (12 attention + 36 SSM,
-# sidecars, no Output-K8 proxy); the requested file is Q6_.
+# full_attention_interval=4), 2 KV heads × 256 dim, embedding 2560,
+# QSA indexer (4 heads, key_length 128, top_k 2048), hyper-connections
+# (count=4, low_rank=320), PLE (layer 1, ngram_size=3, heads_per_ngram=8).
+# Q4_0_ROCMFP4_STRIX_LEAN quant, 98.5 GiB, 3-part GGUF split.
+# Text-only (no mmproj, no MTP draft head).
+#
+# Context: 131072 (128k) × 1 slot.  f16 KV cache.  --fit off because the
+# auto-fit code segfaults on this model's hyper-connection/indexer config;
+# -ngl 999 places all layers on the SVM-backed GPU pool (116 GiB unified).
+#
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
+# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
+# incident). llama.cpp reads --host/--port/--temp as CLI flags, never as
+# env vars.
 
-# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The ROCm
+# ROCmFPX llama-server (PR #27742 merged, build-qwen38).
-# HIP runtime reads those bare env vars (see the 2026-08-13 TEMP incident).
+LLAMA_BIN=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server
-# llama.cpp reads --host/--port/--temp as CLI flags, never as env vars.
 
-# CIRU llama-server binary (ROCmFPX v2.3 + v3 output-K8 patch build, built
+# Model files from
-# against the private TheRock ROCm 7.15 SDK kept in the retired ling3-ciru
+# https://huggingface.co/kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF
-# venv). ROCm 7.15 is required for the advertised Strix Halo performance.
+MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN
-LLAMA_BIN=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server
+MODEL_FILE=Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-00001-of-00003.gguf
 
-# Model files. Plain dense Qwen3.8-27B ROCmFP4 Q6 AGENT GGUF plus the standalone
+# Qwen-Sharp chat template (tracked in the repo, staged on the host). The
-# MTP draft head and the vision projector. Kingjones 2026-08-24.
+# Sharp template drives thinking on by default (medium effort); the server
-MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP
+# emits reasoning in DeepSeek format so clients receive reasoning_content
-MODEL_FILE=Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf
+# separately instead of literal thinking tags in content.
-MTP_DRAFT_FILE=mtp-Qwen3.8-27B-Q4_0.gguf
-MMPROJ_FILE=mmproj-Qwen3.8-27B-Q8_0.gguf
-
-# Qwen-Sharp chat template (tracked in the repo, staged on the host).
 CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
-# enable_thinking on by default in the Sharp template; medium effort default.
+CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true,"preserve_reasoning":true}'
-CHAT_TEMPLATE_KWARGS='{"enable_thinking":true}'
-
-# ROCm 7.15 SDK paths (inside the retired ling3-ciru venv).
-ROCM_DEVEL=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel
-ROCM_LIB=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries
-ROCM_CORE=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core
 
+# Alias + network.
 ALIAS=frank-default-large
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
-# Serving: full native 262144 context, 1 slot. f16 KV, context checkpoints,
+# Serving: 131072 context (128k) × 1 slot. f16 KV, context checkpoints,
-# prompt cache, draft-mtp with standalone Q4_0 head, n_max 4 (the knee for
+# prompt cache. The model is 98.5 GiB loaded via mmap into the Strix Halo
-# the dense model). Unified memory; rocm-smi reports the 64 GiB budget.
+# unified memory pool (116 GiB SVM-backed). --fit off avoids the auto-fit
-CTX_SIZE=262144
+# segfault on this model's qwen4exp hyper-connection/indexer configuration.
+CTX_SIZE=131072
 PARALLEL=1
 BATCH_SIZE=4096
 UBATCH_SIZE=4096
+NUM_THREADS=16
+THREAD_BATCH=32
 CACHE_TYPE_K=f16
 CACHE_TYPE_V=f16
 CTX_CHECKPOINTS=16
 CACHE_RAM=8192
+FIT=off
 
-# Dense-model MTP draft (standalone Q4_0 head, n_max 4). No PromptForge .pfs
+# Sampling (Sharp default).
-# sidecars are loaded; the LLAMA_CPU_ARGMAX fast path stays off.
+TEMPERATURE=0.7
-SPEC_DRAFT_N_MAX=4
-SPEC_DRAFT_N_MIN=0
-SPEC_DRAFT_P_MIN=0.0
-SPEC_DRAFT_P_SPLIT=0.10
-
-# Sampling (kingjones model card / Sharp default).
-TEMPERATURE=1.0
 TOP_K=20
-TOP_P=0.95
+TOP_P=0.8
+MIN_P=0.0
+PRESENCE_PENALTY=1.5
 
-# Thinking on (Qwen-Sharp template drives thinking on; reasoning format
+# Reasoning on (Sharp template drives it; preserve_thinking + preserve_reasoning
-# projects the tar into reasoning_content).
+# keep the reasoning trail in the conversation). DeepSeek format projects the
+# reasoning into the API's reasoning_content field instead of content tags.
 REASONING=on
 REASONING_FORMAT=deepseek
 
+# Strix Halo unified memory + native gfx1151.
+HSA_OVERRIDE_GFX_VERSION=11.5.1
+
 VERBOSITY=4

scripts/frank-default-large.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=CIRU llama.cpp Qwen3.8-27B ROCmFP4 STRIX MTP + Sharp template (dankclaw, frank-default-large)
+Description=ROCmFPX llama.cpp Qwen3.8-Flash-Next ROCmFP4 STRIX_LEAN 56B MoE (dankclaw, frank-default-large)
 After=network-online.target
 Wants=network-online.target
 

scripts/frank-default-small.env

@@ -1,77 +1,71 @@
-# Tunables for frank-default-small (Ornith-1.5-35B-A3B CRACK Q6_K, MTP MoE,
+# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL,
-# Dankserver, RTX 3090).
+# qwen35moe hybrid MoE, Dankserver, RTX 3090).
 #
-# Runtime: upstream llama.cpp llama-server (b10121). RTX 3090 24 GB; Ryzen
+# Runtime: upstream llama.cpp (latest master, CUDA build with
-# 7 7700X; 62 GB RAM. MoE expert weights pinned to CPU with an explicit
+# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090
-# --override-tensor regex (not --n-cpu-moe). Replaced the HauhauCS
+# 24 GB; Ryzen 7 7700X; 62 GB RAM.  Replaced the ik_llama.cpp Ornith
-# Qwen3.6-35B-A3B model on 2026-08-24. Switched from CRACK Q8_0 to Q6_K on
+# CRACK Q8_0 deployment on 2026-09-01.
-# 2026-08-25.
+#
+# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant
+# (30 GB).  Architecture: qwen35moe — 40 layers, full_attention_interval=4
+# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256
+# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000.
+# Text-only (no mmproj).  Built-in chat template in the GGUF.
+#
+# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
+# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card.
+#
+# KV cache: Q8_0 for both K and V.  With CTX_SIZE=200700 (66 900 per slot
+# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the
+# requested 2 GB VRAM buffer.
 #
 # WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
 # ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
 # incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
 
-# Model files (Q6_K CRACK GGUF + f16 mmproj vision encoder + MTP head bundled
+# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid).
-# in the GGUF). Dealign2026-08-24.
+MODEL_DIR=/home/dank/models/tiel-coder-35b
-MODEL_DIR=/home/dank/models/Ornith-1.5-35B-A3B-CRACK
+MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf
-MODEL_FILE=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf
-MMPROJ_FILE=mmproj-Ornith-1.5-35B-A3B-f16.gguf
-# Upstream llama.cpp binary (b10121).
-LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
 
-# Qwen-Sharp chat template from peculiar-ragdoll (tracked in the repo,
+# Upstream llama.cpp llama-server (CUDA build, latest master).
-# staged on the host).
+# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86
-# Source: https://huggingface.co/peculiar-ragdoll/Qwen-Sharp-Chat-Templates
+# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage.
-CHAT_TEMPLATE_FILE=/home/dank/serverAppConfigs/scripts/chat-template.jinja
+LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
 
 ALIAS=frank-default-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7770
 
-# 3 parallel slots x 163840 ctx each (262144 lowered by 96k per stream for
+# Total context budget across all parallel slots.  200700 / 3 = 66 900
-# cache and checkpoint headroom) = 491520 total context pool. The per-slot
+# per slot.  The model's context_length is 262 144, so 66 900 is well
-# 163840 is advertised in models.json.
+# within range.
-CTX_SIZE=491520
+CTX_SIZE=200700
 PARALLEL=3
 
-# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
+# Expert offload: all MoE expert weights pinned to CPU RAM.
-SPEC_TYPE=ngram-mod
+CPU_MOE=1
-SPEC_NGRAM_MOD_N_MIN=48
-SPEC_NGRAM_MOD_N_MAX=64
-SPEC_NGRAM_MOD_N_MATCH=24
 
-# Expert offload via explicit tensor override (replaces --n-cpu-moe). The Q6_K
+# All non-expert layers on GPU (auto-fit will determine the exact count;
-# layer audit found 630 MiB of expert tensors per block. Keep every
+# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room).
-# full-attention block plus every block 35-40 on CUDA. The unique GPU set is
+N_GPU_LAYERS=999
-# 3,7,11,15,19,23,27,31,35,36,37,38,39,40. Move all other expert tensors to
-# CPU. All non-expert tensors stay on the 24 GB card. Buffer type "CPU" is the
-# CPU buft name on this host.
-OVERRIDE_EXPERT='blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps'
-OVERRIDE_BUFT=CPU
-N_GPU_LAYERS=all
 
-# KV cache on GPU, both K and V at f16.
+# KV cache: Q8_0 for both K and V.  ~2 GB total for 200700 ctx × 3 slots
-CACHE_TYPE_K=f16
+# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V).
-CACHE_TYPE_V=f16
+CACHE_TYPE_K=q8_0
+CACHE_TYPE_V=q8_0
 
-VERBOSITY=4
+VERBOSITY=2
-# Store context checkpoints in RAM to hold peak GPU memory down.
-CTX_CHECKPOINTS=16
 
-# Prefill batch sizing. ubatch 1024 is selected for reduced compute-memory
+# Prefill batch sizing.
-# pressure and predictable latency under the ngram-mod deployment.
 BATCH_SIZE=4096
 UBATCH_SIZE=1024
 
-# Eager RAM load (expert weights on CPU; ~31 GB RAM).
+# Sampling.
-NO_MMAP=1
-
-# Sampling (Dealign Ornith model card, thinking-general).
 TEMPERATURE=1.0
 TOP_K=20
 TOP_P=0.95
 MIN_P=0.0
 
-# Compute threads lowered to 8 so other processes keep a core.
+# Compute threads lowered so other processes keep a core.
 THREADS=7
 THREADS_HTTP=8

scripts/frank-default-small.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama.cpp Ornith-1.5-35B-A3B CRACK Q6_K ngram-mod + Qwen-Sharp template (dankserver, frank-default-small)
+Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small)
 After=network-online.target
 Wants=network-online.target
 

scripts/mount-entropy.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Unlock and mount Dankserver's encrypted entropy volume.
+#
+# The desktop flow uses the system's crypttab and fstab entries.  Reuse those
+# entries here instead of embedding a passphrase, mount options, or ownership
+# changes in this helper.  udisksctl reads the passphrase through its secure
+# desktop/terminal authentication path.
+set -euo pipefail
+
+MOUNTPOINT='/mnt/entropy'
+MAPPER='/dev/mapper/luks-cd86c728-459e-4f3a-bcc2-41ecf84685ec'
+
+if mountpoint -q "$MOUNTPOINT"; then
+    printf '%s is already mounted.\n' "$MOUNTPOINT"
+    exit 0
+fi
+
+if [ ! -e /dev/md0 ]; then
+    printf 'ERROR: RAID device /dev/md0 is not available.\n' >&2
+    exit 1
+fi
+
+if [ ! -e "$MAPPER" ]; then
+    udisksctl unlock --block-device /dev/md0
+fi
+
+# udisks2 may mount automatically as part of unlocking.  Do not attempt a
+# second mount in that case.
+if ! mountpoint -q "$MOUNTPOINT"; then
+    sudo mount "$MOUNTPOINT"
+fi
+printf 'Mounted %s.\n' "$MOUNTPOINT"

scripts/serve-frank-large.sh

@@ -1,28 +1,32 @@
 #!/usr/bin/env bash
-# Serve kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP (dense) as
+# Serve kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN as
-# frank-default-large via the CIRU ROCmFPX llama.cpp runtime on Dankclaw
+# frank-default-large via the charlie12345/ROCmFPX llama.cpp fork with
-# (Strix Halo gfx1151).
+# PR #27742 (qwen4exp architecture) merged, on Dankclaw (Strix Halo
+# gfx1151, unified memory).
 #
-# Runtime: the PromptForge llama-server built from ciru-ai/ROCmFPX tag
+# Runtime: ROCmFPX llama.cpp built at
-# qwen3.8-activefpx-promptforge-v2.3 (commit 103af8b41) plus the v3 output-K8
+# /home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server
-# runtime patch (commit af0eac6), against the private TheRock ROCm 7.15 SDK
+# with -DGGML_HIP=ON -DGGML_HIP_ROCMI4_W4A4=ON
-# (kept in the retired ling3-ciru venv). The model is the plain ROCmFP4
+# -DCMAKE_HIP_ARCHITECTURES=gfx1151 -DLLAMA_BUILD_WEBUI=OFF.
-# Q6 AGENT GGUF (no PromptForge .pfs sidecars) with the standalone MTP draft
-# head mtp-Qwen3.8-27B-Q4_0.gguf. The Qwen-Sharp chat template is applied at
-# runtime.
 #
-# Spec: dense model -> native MTP draft at n_max 4 (the knee, per the model
+# Model: 56B MoE (512 experts, 10 used), 48 layers (12 attention + 36
-# card). The draft head is a separate Q4_0 GGUF loaded via --model-draft.
+# SSM/Mamba, full_attention_interval=4), 2 KV heads × 256 dim, QSA
+# indexer (4 heads, top_k=2048), hyper-connections (count=4, low_rank=320),
+# PLE (layer 1).  Q4_0_ROCMFP4_STRIX_LEAN quant, 98.5 GiB, 3-part split.
 #
-# The model card requires ROCm 7.15 (not the system ROCm 7.2.4) for the
+# Context: 131072 (128k) × 1 slot.  f16 KV cache.  Model is loaded with
-# advertised Strix Halo performance. LD_LIBRARY_PATH therefore points at the
+# mmap (default) — the 98.5 GiB file is memory-mapped; the Strix Halo
-# 7.15 SDK libraries first.
+# unified memory (116 GiB SVM-backed) provides the GPU address space.
+# --fit off is set because the auto-fit code segfaults on this model's
+# hyper-connection/indexer configuration; -ngl 999 places all layers on
+# the GPU (SVM-backed), which the unified memory pool handles.
 #
-# Tunables come from frank-default-large.env in this directory unless
+# The Qwen-Sharp chat template (scripts/chat-template.jinja) is applied at
-# FRANK_LARGE_ENV_FILE overrides that path.
+# runtime so the model emits reasoning in DeepSeek format; the reasoning
+# trail is preserved across turns via CHAT_TEMPLATE_KWARGS.
 set -euo pipefail
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-default-large.env}"
 if [[ -f "$ENV_FILE" ]]; then
     set -a
@@ -32,101 +36,85 @@ if [[ -f "$ENV_FILE" ]]; then
 fi
 
 # --- defaults (used when the env file does not set them) -------------------
-: "${LLAMA_BIN:=/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server}"
+: "${LLAMA_BIN:=/home/dank/GitPackages/ROCmFPX/build-qwen38/bin/llama-server}"
-: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-STRIX-MTP}"
+: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN}"
-: "${MODEL_FILE:=Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
+: "${MODEL_FILE:=Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-00001-of-00003.gguf}"
-: "${MTP_DRAFT_FILE:=mtp-Qwen3.8-27B-Q4_0.gguf}"
-: "${MMPROJ_FILE:=mmproj-Qwen3.8-27B-Q8_0.gguf}"
 : "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
-: "${ROCM_DEVEL:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_devel}"
+: "${CHAT_TEMPLATE_KWARGS:={}}"
-: "${ROCM_LIB:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_libraries}"
-: "${ROCM_CORE:=/home/dank/ling3-ciru/Ling-3.0-Flash-CIRU-int4-Strix-native/.venv/lib/python3.12/site-packages/_rocm_sdk_core}"
 : "${ALIAS:=frank-default-large}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=262144}"
+: "${CTX_SIZE:=131072}"
 : "${PARALLEL:=1}"
 : "${BATCH_SIZE:=4096}"
 : "${UBATCH_SIZE:=4096}"
+: "${NUM_THREADS:=16}"
+: "${THREAD_BATCH:=32}"
 : "${CACHE_TYPE_K:=f16}"
 : "${CACHE_TYPE_V:=f16}"
 : "${CTX_CHECKPOINTS:=16}"
 : "${CACHE_RAM:=8192}"
-: "${SPEC_DRAFT_N_MAX:=4}"
+: "${FIT:=off}"
-: "${SPEC_DRAFT_N_MIN:=0}"
+: "${TEMPERATURE:=0.7}"
-: "${SPEC_DRAFT_P_MIN:=0.0}"
-: "${SPEC_DRAFT_P_SPLIT:=0.10}"
-: "${TEMPERATURE:=1.0}"
 : "${TOP_K:=20}"
-: "${TOP_P:=0.95}"
+: "${TOP_P:=0.8}"
+: "${MIN_P:=0.0}"
+: "${PRESENCE_PENALTY:=1.5}"
 : "${REASONING:=on}"
 : "${REASONING_FORMAT:=deepseek}"
-: "${CHAT_TEMPLATE_KWARGS:={}}"
+: "${HSA_OVERRIDE_GFX_VERSION:=11.5.1}"
-: "${VERBOSITY:=4}"
 
 MODEL="$MODEL_DIR/$MODEL_FILE"
-MTP_DRAFT="$MODEL_DIR/$MTP_DRAFT_FILE"
+for f in "$LLAMA_BIN" "$MODEL" "$CHAT_TEMPLATE_FILE"; do
-
+    if [[ ! -r "$f" ]]; then
-# Plain dense model: no PromptForge .pfs sidecars, no Output-K8 proxy, no IU4
+        echo "missing or unreadable required artifact: $f" >&2
-# lane. The v3 MTP is driven by the standalone Q4_0 draft head through the
-# standard draft-mtp path. Unset any leftover PromptForge env lanes.
-unset PROMPTFORGE_SIDECAR PROMPTFORGE_GDN_SIDECAR PROMPTFORGE_MTP_OUTPUT_K8_PROXY || true
-unset PROMPTFORGE_MODE PROMPTFORGE_ENABLE_SMALLM_W8 || true
-unset PROMPTFORGE_MTP_OUTPUT_K8 PROMPTFORGE_MTP_OUTPUT_K8_VALIDATE || true
-unset PROMPTFORGE_IU4_SIDECAR PROMPTFORGE_GDN_IU4_SIDECAR || true
-unset PROMPTFORGE_ENABLE_NGRAM_MOD PROMPTFORGE_ENABLE_NGRAM_M65_IU4 || true
-
-for f in "$MODEL" "$MTP_DRAFT" "$MODEL_DIR/$MMPROJ_FILE" "$CHAT_TEMPLATE_FILE"; do
-    if [[ ! -f "$f" ]]; then
-        echo "missing required file: $f" >&2
         exit 1
     fi
 done
 if [[ ! -x "$LLAMA_BIN" ]]; then
-    echo "missing llama-server binary: $LLAMA_BIN" >&2
+    echo "missing executable runtime: $LLAMA_BIN" >&2
     exit 1
 fi
 
-# ROCm 7.15 SDK libraries must precede any system ROCm in LD_LIBRARY_PATH.
+SERVER_DIR="$(cd -- "$(dirname -- "$LLAMA_BIN")" && pwd)"
-export ROCM_PATH="$ROCM_DEVEL"
+ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
-export LD_LIBRARY_PATH="${ROCM_LIB}/lib:${ROCM_CORE}/lib:${ROCM_DEVEL}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+LIBRARY_PATH="$SERVER_DIR:$ROCM_PATH/lib:$ROCM_PATH/lib/rocm_sysdeps/lib:$ROCM_PATH/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
 
-exec "$LLAMA_BIN" \
+exec /usr/bin/env \
+    ROCM_PATH="$ROCM_PATH" \
+    HIP_VISIBLE_DEVICES=0 \
+    LD_LIBRARY_PATH="$LIBRARY_PATH" \
+    HSA_OVERRIDE_GFX_VERSION="$HSA_OVERRIDE_GFX_VERSION" \
+    "$LLAMA_BIN" \
     --model "$MODEL" \
-    --mmproj "$MODEL_DIR/$MMPROJ_FILE" \
+    --alias "$ALIAS" \
     --chat-template-file "$CHAT_TEMPLATE_FILE" \
     --chat-template-kwargs "$CHAT_TEMPLATE_KWARGS" \
-    --alias "$ALIAS" \
     --host "$BIND_ADDR" \
     --port "$LISTEN_PORT" \
-    -dev ROCm0 \
+    --jinja \
-    -ngl 999 \
+    --device ROCm0 \
-    -c "$CTX_SIZE" \
+    --gpu-layers 999 \
-    -b "$BATCH_SIZE" \
+    --parallel "$PARALLEL" \
-    -ub "$UBATCH_SIZE" \
+    --ctx-size "$CTX_SIZE" \
-    -fa on \
+    --batch-size "$BATCH_SIZE" \
-    -ctk "$CACHE_TYPE_K" \
+    --ubatch-size "$UBATCH_SIZE" \
-    -ctv "$CACHE_TYPE_V" \
+    --fit "$FIT" \
-    -np "$PARALLEL" \
+    --flash-attn on \
-    -ctxcp "$CTX_CHECKPOINTS" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --threads "$NUM_THREADS" \
+    --threads-batch "$THREAD_BATCH" \
+    --ctx-checkpoints "$CTX_CHECKPOINTS" \
     --cache-ram "$CACHE_RAM" \
     --cache-prompt \
+    --cache-idle-slots \
     --metrics \
-    --no-mmap \
-    --spec-type draft-mtp \
-    --model-draft "$MTP_DRAFT" \
-    --spec-draft-device ROCm0 \
-    --spec-draft-ngl 999 \
-    --spec-draft-type-k f16 \
-    --spec-draft-type-v f16 \
-    --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
-    --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \
-    --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
-    --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT" \
     --temp "$TEMPERATURE" \
-    --top-k "$TOP_K" \
     --top-p "$TOP_P" \
+    --top-k "$TOP_K" \
+    --min-p "$MIN_P" \
+    --presence-penalty "$PRESENCE_PENALTY" \
     --reasoning "$REASONING" \
     --reasoning-format "$REASONING_FORMAT" \
-    --reasoning-budget -1 \
+    --reasoning-budget -1
-    -lv "$VERBOSITY"

scripts/serve-frank-small.sh

@@ -1,18 +1,23 @@
 #!/usr/bin/env bash
-# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q6_K) as
+# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as
-# frank-default-small via upstream llama.cpp on Dankserver (RTX 3090, MoE
+# frank-default-small via stock llama.cpp on Dankserver (RTX 3090).
-# experts on CPU).
 #
-# Runtime: upstream llama.cpp llama-server (b10121). The model is a 35B-total /
+# Runtime: upstream llama.cpp (latest master, CUDA build with
-# 3B-active MoE-dense hybrid with an MTP head bundled in the GGUF; this
+# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86).
-# service uses ngram-mod instead of MTP speculation.
+# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with
+# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM
+# layers out of 40).  256 experts, 8 used per token.
 #
-# Expert offload: the expert (FFN exps) tensors selected by the layer-bounded
+# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
-# --override-tensor pattern are pinned to CPU. Selected full-attention and
+# Non-expert weights (attention, SSM, shared expert, embeddings, output)
-# final output-adjacent blocks stay on CUDA with all non-expert tensors,
+# are ~1.6 GB and fit entirely on the 24 GB RTX 3090.
-# without relying on MTP speculation.
 #
-# Speculative decoding: ngram-mod only. MTP is intentionally disabled.
+# KV cache: Q8_0 for both K and V.  With CTX_SIZE=200700 (66 900 per
+# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim),
+# the KV cache is ~2 GB — the requested 2 GB VRAM buffer.
+#
+# Model is loaded with mmap (default).  The 30 GB Q6_K_XL file is
+# memory-mapped; only accessed pages enter RAM.
 #
 # Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
 # overrides that path.
@@ -28,33 +33,21 @@ if [[ -f "$ENV_FILE" ]]; then
 fi
 
 # --- defaults (used when the env file does not set them) -----------------
-: "${MODEL_DIR:=/home/dank/models}"
+: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}"
-: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf}"
+: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}"
-: "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}"
 : "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
-: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
 : "${ALIAS:=frank-default-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=491520}"
+: "${CTX_SIZE:=200700}"
 : "${PARALLEL:=3}"
-: "${SPEC_TYPE:=ngram-mod}"
+: "${CACHE_TYPE_K:=q8_0}"
-: "${SPEC_NGRAM_MOD_N_MIN:=48}"
+: "${CACHE_TYPE_V:=q8_0}"
-: "${SPEC_NGRAM_MOD_N_MAX:=64}"
+: "${N_GPU_LAYERS:=999}"
-: "${SPEC_NGRAM_MOD_N_MATCH:=24}"
-: "${CACHE_TYPE_K:=f16}"
-: "${CACHE_TYPE_V:=f16}"
-: "${VERBOSITY:=4}"
-# Expert offload: --override-tensor regex pinned to the CPU buft. Same pattern
-# the internal --cpu-moe / --n-cpu-moe use; explicit here so we only move the
-# MoE expert weights and leave every other tensor on the GPU.
-: "${OVERRIDE_EXPERT:=blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps}"
-: "${OVERRIDE_BUFT:=CPU}"
-: "${N_GPU_LAYERS:=all}"
 : "${BATCH_SIZE:=4096}"
 : "${UBATCH_SIZE:=1024}"
-: "${CTX_CHECKPOINTS:=16}"
+: "${VERBOSITY:=2}"
-: "${NO_MMAP:=1}"
+: "${CPU_MOE:=1}"
 : "${TEMPERATURE:=1.0}"
 : "${TOP_K:=20}"
 : "${TOP_P:=0.95}"
@@ -63,14 +56,11 @@ fi
 : "${THREADS_HTTP:=8}"
 
 MODEL="$MODEL_DIR/$MODEL_FILE"
-MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
 
-for f in "$MODEL" "$MMPROJ" "$CHAT_TEMPLATE_FILE"; do
+if [[ ! -f "$MODEL" ]]; then
-    if [[ ! -f "$f" ]]; then
+    echo "missing required file: $MODEL" >&2
-        echo "missing required file: $f" >&2
+    exit 1
-        exit 1
+fi
-    fi
-done
 if [[ ! -x "$LLAMA_SERVER" ]]; then
     echo "missing llama-server binary: $LLAMA_SERVER" >&2
     exit 1
@@ -78,23 +68,14 @@ fi
 
 exec "$LLAMA_SERVER" \
     --model "$MODEL" \
-    --mmproj "$MMPROJ" \
+    $([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \
-    --chat-template-file "$CHAT_TEMPLATE_FILE" \
-    --spec-type "$SPEC_TYPE" \
-    --spec-ngram-mod-n-min "$SPEC_NGRAM_MOD_N_MIN" \
-    --spec-ngram-mod-n-max "$SPEC_NGRAM_MOD_N_MAX" \
-    --spec-ngram-mod-n-match "$SPEC_NGRAM_MOD_N_MATCH" \
-    --override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT" \
     --cache-type-k "$CACHE_TYPE_K" \
     --cache-type-v "$CACHE_TYPE_V" \
-    --verbosity "$VERBOSITY" \
     --ctx-size "$CTX_SIZE" \
     --parallel "$PARALLEL" \
     --gpu-layers "$N_GPU_LAYERS" \
     --batch-size "$BATCH_SIZE" \
     --ubatch-size "$UBATCH_SIZE" \
-    --ctx-checkpoints "$CTX_CHECKPOINTS" \
-    $([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
     --temp "$TEMPERATURE" \
     --top-k "$TOP_K" \
     --top-p "$TOP_P" \

scripts/test-changed.sh

@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+# Run only the Bats tests affected by the staged paths.
+#
+# Usage:
+#   scripts/test-changed.sh              # inspect the staged commit
+#   scripts/test-changed.sh --list       # print selected test files
+#   scripts/test-changed.sh --paths FILE ...
+#
+# --paths is intended for tests and local experimentation. The normal mode is
+# deliberately staged-only so a pre-commit hook cannot accidentally validate
+# an unrelated dirty working-tree file.
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+LIST_ONLY=0
+USE_PATHS=0
+declare -a CHANGED=()
+
+die() {
+    printf 'ERROR: %s\n' "$*" >&2
+    exit 2
+}
+
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        --list)
+            LIST_ONLY=1
+            shift
+            ;;
+        --paths)
+            USE_PATHS=1
+            shift
+            while [ "$#" -gt 0 ] && [[ "$1" != --* ]]; do
+                CHANGED+=("$1")
+                shift
+            done
+            ;;
+        --)
+            shift
+            CHANGED+=("$@")
+            break
+            ;;
+        *)
+            die "unknown option '$1' (use --list or --paths FILE ...)"
+            ;;
+    esac
+done
+
+cd "$REPO_ROOT"
+if [ "$USE_PATHS" -eq 0 ]; then
+    while IFS= read -r -d '' path; do
+        CHANGED+=("$path")
+    done < <(git diff --cached --name-only -z --diff-filter=ACMRTUXB)
+fi
+
+[ "${#CHANGED[@]}" -gt 0 ] || {
+    printf 'No staged paths; no focused tests to run.\n'
+    exit 0
+}
+
+declare -a TESTS=()
+add_test() {
+    local test="$1"
+    local existing
+    for existing in "${TESTS[@]:-}"; do
+        [ "$existing" != "$test" ] || return 0
+    done
+    TESTS+=("$test")
+}
+
+for path in "${CHANGED[@]}"; do
+    case "$path" in
+        tests/*.bats)
+            add_test "$path"
+            ;;
+        tests/helpers.bash|tests/run.sh)
+            die "$path is shared test infrastructure; run ./tests/run.sh explicitly"
+            ;;
+        .githooks/*)
+            add_test tests/changed_tests.bats
+            ;;
+        dankserver/compose.yaml|dankclaw/compose.yaml)
+            add_test tests/compose_validity.bats
+            add_test tests/env_consistency.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        dankserver/Caddyfile|dankclaw/Caddyfile)
+            add_test tests/caddyfile.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        dankserver/.env.example|dankclaw/.env.example)
+            add_test tests/env_consistency.bats
+            ;;
+        shared/homepage/*)
+            add_test tests/homepage_caddy.bats
+            add_test tests/homepage_config.bats
+            add_test tests/homepage_deploy.bats
+            add_test tests/homepage_pages.bats
+            add_test tests/homepage_checker.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        shared/searxng/*)
+            add_test tests/searxng_limiter.bats
+            add_test tests/searxng_quadlet.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        # Caddy left compose.yaml on 2026-08-26 for a quadlet; caddyfile.bats
+        # now asserts against the quadlet itself. Route every change under
+        # dankclaw/caddy/ to both seams (this block must precede the
+        # dankclaw/*.container glob, which would otherwise swallow the unit).
+        dankclaw/caddy/*)
+            add_test tests/caddy_quadlet.bats
+            add_test tests/caddyfile.bats
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        dankclaw/*.container|dankclaw/*.network)
+            add_test tests/open_webui_quadlet.bats
+            add_test tests/pihole.bats
+            add_test tests/searxng_quadlet.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        # Quadlets live under dankclaw/<svc>/quadlet/ (reorganized out of the
+        # top-level dankclaw/*.container globs above). Map the whole subtree so
+        # changes to the pihole, open-webui, searxng, and gitea quadlets hit
+        # their seam tests. The pihole-specific block must come first: shell
+        # case matches the first pattern, and dankclaw/pihole/quadlet/*.container
+        # also satisfies the dankclaw/*/quadlet/*.container fallback.
+        dankclaw/pihole/*)
+            add_test tests/pihole.bats
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        dankclaw/*/quadlet/*.container|dankclaw/*/quadlet/*.network)
+            add_test tests/open_webui_quadlet.bats
+            add_test tests/pihole.bats
+            add_test tests/searxng_quadlet.bats
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        scripts/deploy.sh|scripts/deploy-remote.sh|scripts/deploy-plan.py|scripts/deploy-*-quadlet.sh)
+            add_test tests/deployment_flow.bats
+            add_test tests/deployment_scoping.bats
+            add_test tests/scripts_syntax.bats
+            ;;
+        scripts/mount-entropy.sh)
+            add_test tests/mount_entropy.bats
+            ;;
+        scripts/frank-default-large.env|scripts/serve-frank-large.sh)
+            add_test tests/llm_units.bats
+            add_test tests/llm_secrets.bats
+            add_test tests/scripts_syntax.bats
+            ;;
+        scripts/frank-default-small.env|scripts/serve-frank-small.sh)
+            add_test tests/llm_units.bats
+            add_test tests/llm_secrets.bats
+            add_test tests/scripts_syntax.bats
+            ;;
+        scripts/*.service)
+            add_test tests/llm_units.bats
+            add_test tests/scripts_syntax.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        scripts/*)
+            add_test tests/scripts_syntax.bats
+            ;;
+        docs/*)
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        dankclaw/hermes/*)
+            add_test tests/docs_completeness.bats
+            ;;
+        README.md|AGENTS.md)
+            # Project guidance changes can affect the documented deployment
+            # contract; validate the docs vault and scoped deployment seams.
+            add_test tests/docs_completeness.bats
+            add_test tests/deployment_scoping.bats
+            ;;
+        .AGENTTRASH/*)
+            # Trash moves are not testable; skip.
+            ;;
+        *)
+            die "no focused-test mapping for '$path'; add one before committing"
+            ;;
+    esac
+done
+
+if [ "$LIST_ONLY" -eq 1 ]; then
+    printf '%s\n' "${TESTS[@]}"
+    exit 0
+fi
+
+command -v bats >/dev/null 2>&1 || die 'bats is not installed'
+printf 'Running focused tests for %d changed path(s):\n' "${#CHANGED[@]}"
+printf '  %s\n' "${TESTS[@]}"
+bats --print-output-on-failure "${TESTS[@]}"

scripts/tmux-resurrect.sh

@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+#
+# tmux-resurrect.sh — snapshot and restore the exact layout of a tmux session
+# on Dankclaw, including each pane's start directory and foreground command.
+#
+# Use it after a reboot (or when the display session is gone) to reopen the
+# same windows/panes on the Dankclaw console:
+#
+#   scripts/tmux-resurrect.sh snapshot      # record the live session to JSON
+#   scripts/tmux-resurrect.sh restore       # rebuild it, then attach
+#
+# Data file: ~/.tmux-resurrect/<session>.json (set TMUX_RES_DIR to relocate).
+# Re-run snapshot as the session changes so the restore stays faithful.
+#
+# How each pane's command is captured: a tmux pane runs one main shell
+# (#{pane_pid}). The foreground command is that shell's direct child — chosen
+# as the child that launched its own child (ssh, sudo ...) or the first child
+# when none did (btop sitting at the top of a pane). An idle shell yields no
+# command and is left at its prompt on restore.
+set -euo pipefail
+
+TMUX_RES_DIR="${TMUX_RES_DIR:-$HOME/.tmux-resurrect}"
+DEFAULT_SESSION="0"
+
+die() { printf 'tmux-resurrect: %s\n' "$*" >&2; exit 1; }
+b64() { base64 -w0; }
+b64d() { base64 -d; }
+
+# fg_command <shell_pid>: print the foreground command of a tmux pane, or the
+# empty string when the shell is idle.
+fg_command() {
+    local pid="$1" kids=() kid cmd_pid=""
+    mapfile -t kids < <(pgrep -P "$pid" 2>/dev/null || true)
+    if [ "${#kids[@]}" -eq 0 ]; then return 0; fi
+    # Prefer a child that launched its own child (a launcher such as ssh or
+    # sudo); fall back to the first direct child (e.g. btop at the top).
+    for kid in "${kids[@]}"; do
+        if pgrep -P "$kid" >/dev/null 2>&1; then
+            cmd_pid="$kid"; break
+        fi
+    done
+    if [ -z "$cmd_pid" ]; then cmd_pid="${kids[0]}"; fi
+    tr '\0' ' ' < "/proc/$cmd_pid/cmdline" 2>/dev/null | sed -E 's/[[:space:]]+$//'
+}
+
+# snapshot <session> [dest.json]
+snapshot() {
+    local session="${1:-$DEFAULT_SESSION}" dest="${2:-}"
+    local exists
+    exists=$(tmux list-sessions -F '#{session_name}' 2>/dev/null | grep -Fxq "$session" && echo 1 || echo 0)
+    [ "$exists" = 1 ] || die "no tmux session named '$session'"
+    mkdir -p "$TMUX_RES_DIR"
+    [ -z "$dest" ] && dest="$TMUX_RES_DIR/$session.json"
+    python3 - "$dest" "$session" 3< <(
+    {
+        while IFS=$'\t' read -r widx wname wlayout; do
+            printf 'W\t%s\t%s\t%s\n' "$widx" "$(printf '%s' "$wname" | b64)" "$(printf '%s' "$wlayout" | b64)"
+            while IFS=$'\t' read -r pidx ppid pdir; do
+                pcmd=$(fg_command "$ppid")
+                printf 'P\t%s\t%s\t%s\t%s\n' "$widx" "$pidx" \
+                    "$(printf '%s' "$pdir" | b64)" "$(printf '%s' "$pcmd" | b64)"
+            done < <(tmux list-panes -t "$session:$widx" \
+                        -F $'#{pane_index}\t#{pane_pid}\t#{pane_current_path}' | sort -n -k1,1)
+        done < <(tmux list-windows -t "$session" \
+                    -F $'#{window_index}\t#{window_name}\t#{window_layout}' | sort -n -k1,1)
+    }
+    ) > /dev/null <<'PY'
+import sys, json, base64, os
+dest, session = sys.argv[1], sys.argv[2]
+wmap, panes = {}, {}
+for line in os.fdopen(3).read().splitlines():
+    if not line:
+        continue
+    p = line.split('\t')
+    if p[0] == 'W':
+        wmap[int(p[1])] = {"name": base64.b64decode(p[2]).decode(),
+                           "layout": base64.b64decode(p[3]).decode()}
+    elif p[0] == 'P':
+        widx = int(p[1])
+        panes.setdefault(widx, []).append({
+            "index": int(p[2]),
+            "dir": base64.b64decode(p[3]).decode(),
+            "cmd": base64.b64decode(p[4]).decode(),
+        })
+windows = []
+for widx in sorted(wmap):
+    windows.append({
+        "index": widx,
+        "name": wmap[widx]["name"],
+        "layout": wmap[widx]["layout"],
+        "panes": sorted(panes.get(widx, []), key=lambda x: x["index"]),
+    })
+with open(dest, "w") as f:
+    json.dump({"session": session, "windows": windows}, f, indent=2, ensure_ascii=False)
+    f.write("\n")
+PY
+    printf 'snapshot: wrote %s (session %s)\n' "$dest" "$session"
+}
+
+# emit_plan <src.json> [session]: python reads the JSON and prints a base64
+# TSV plan (W/P records) for the bash orchestrator to consume.
+emit_plan() {
+    python3 - "$1" "$2" <<'PY'
+import sys, json, base64
+src, session = sys.argv[1], sys.argv[2]
+with open(src) as f:
+    data = json.load(f)
+wins = data.get("windows", [])
+if session:
+    wins = [w for w in wins if str(w.get("index")) == "0"]
+def b64(s): return base64.b64encode(s.encode()).decode()
+for w in wins:
+    print('W\t%s\t%s\t%s' % (w["index"], b64(w["name"]), b64(w["layout"])))
+    for pan in w["panes"]:
+        print('P\t%s\t%s\t%s\t%s' % (w["index"], pan["index"],
+                                      b64(pan["dir"]), b64(pan["cmd"])))
+PY
+}
+
+# restore [session] [--no-attach] [--dry-run]
+restore() {
+    local session="${1:-$DEFAULT_SESSION}"
+    shift || true
+    local keep=0 noattach=0 dryrun=0 a
+    for a in "$@"; do
+        case "$a" in
+            --keep) keep=1 ;;
+            --no-attach) noattach=1 ;;
+            --dry-run) dryrun=1 ;;
+            *) die "unknown restore option: $a" ;;
+        esac
+    done
+    local dest="$TMUX_RES_DIR/$session.json"
+    [ -f "$dest" ] || die "no snapshot at $dest (run snapshot first)"
+
+    # Restore wrapper: run tmux, or print the command under --dry-run.
+    tmuxr() {
+        if [ "$dryrun" = 1 ]; then printf '  tmux %s\n' "$*"; else tmux "$@"; fi
+    }
+
+    local target_exists
+    target_exists=$(tmux list-sessions -F '#{session_name}' 2>/dev/null | grep -Fxq "$session" && echo 1 || echo 0)
+    if [ "$target_exists" = 1 ]; then
+        if [ "$keep" = 1 ]; then
+            die "session '$session' already exists; rebuild without --keep or save a copy"
+        fi
+        if [ "$dryrun" = 1 ]; then printf '  # kill existing session %s\n' "$session"
+        else tmux kill-session --force "$session"; fi
+    fi
+
+    # Parse the plan into flat, window-major lists.
+    local -a WNAME=() WLAY=() WSTART=() PD=() PC=()
+    local tag idx nameb layoutb dirb cmdb
+    while IFS=$'\t' read -r tag idx nameb layoutb dirb cmdb; do
+        case "$tag" in
+            W) WNAME+=("$(printf '%s' "$nameb" | b64d)"); WLAY+=("$(printf '%s' "$layoutb" | b64d)"); WSTART+=("${#PD[@]}") ;;
+            P) PD+=("$(printf '%s' "$dirb" | b64d)"); PC+=("$(printf '%s' "$cmdb" | b64d)") ;;
+        esac
+    done < <(emit_plan "$dest" "")
+
+    local n=${#WNAME[@]} w first last npanes i curwin wname wlayout dir cmd
+    for ((w = 0; w < n; w++)); do
+        first=${WSTART[w]}
+        if (( w + 1 < n )); then last=${WSTART[w + 1]}; else last=${#PD[@]}; fi
+        npanes=$(( last - first ))
+        wname="${WNAME[w]}"; wlayout="${WLAY[w]}"
+        if [ "$dryrun" = 1 ]; then
+            if (( w == 0 )); then
+                tmuxr new-session -d -s "$session" -n "$wname" -c "${PD[first]}"
+            else
+                tmuxr new-window -d -t "$session" -n "$wname" -c "${PD[first]}"
+            fi
+            curwin="$session:$wname"
+        elif (( w == 0 )); then
+            curwin=$(tmux new-session -d -P -F '#{window_id}' \
+                -s "$session" -n "$wname" -c "${PD[first]}")
+        else
+            curwin=$(tmux new-window -d -P -F '#{window_id}' \
+                -t "$session" -n "$wname" -c "${PD[first]}")
+        fi
+        # Create the remaining panes in recorded order (index == creation order).
+        for ((i = first + 1; i < last; i++)); do
+            tmuxr split-window -t "$curwin" -c "${PD[i]}"
+        done
+        # Restore the exact geometry for this window.
+        tmuxr select-layout -t "$curwin" "$wlayout"
+        # Launch each pane's recorded command, matched to panes by ascending
+        # pane index (so this is correct regardless of base-index).
+        local -a ordered=()
+        mapfile -t ordered < <(tmux list-panes -t "$curwin" \
+            -F $'#{pane_id}\t#{pane_target}' | sort -n -k1,1)
+        local k=0 tgt
+        for tgt in "${ordered[@]}"; do
+            cmd="${PC[first + k]}"
+            if [ -n "$cmd" ]; then tmuxr send-keys -t "$tgt" "$cmd" Enter; fi
+            k=$((k + 1))
+        done
+    done
+
+    if [ "$noattach" = 1 ]; then
+        return 0
+    elif [ -t 0 ]; then
+        tmuxr attach -t "$session"
+    elif [ "$dryrun" = 0 ]; then
+        echo "session '$session' rebuilt; attach with: tmux attach -t $session"
+    fi
+}
+
+usage() {
+    cat <<'USAGE'
+tmux-resurrect.sh — snapshot & restore a tmux session on Dankclaw.
+
+Usage:
+  tmux-resurrect.sh snapshot [session] [dest.json]   record the live session
+  tmux-resurrect.sh restore  [session] [options]      rebuild the session
+    options:
+      --keep        skip if the session already exists (default: rebuild)
+      --no-attach   rebuild without attaching to this terminal
+      --dry-run     print the commands instead of running them
+USAGE
+}
+
+main() {
+    local cmd="${1:-snapshot}"
+    shift || true
+    case "$cmd" in
+        snapshot) snapshot "$@" ;;
+        restore)  restore "$@" ;;
+        -h|--help|help) usage ;;
+        *) die "unknown command: $cmd (use snapshot|restore)" ;;
+    esac
+}
+
+main "$@"

shared/searxng/settings.yml

@@ -21,13 +21,93 @@ valkey:
   url: "valkey://searxng-valkey:6379/0"
 
 outgoing:
-  request_timeout: 3.0
+  request_timeout: 5.0
-  max_request_timeout: 10.0
+  max_request_timeout: 15.0
   pool_connections: 100
   pool_maxsize: 20
   enable_http2: true
   keepalive_expiry: 5.0
-  retries: 3
+  retries: 1
+
+# Keep default aggregation anonymous and bounded. Disabled engines remain
+# available for explicit selection/on-demand use; inactive engines are removed.
+# Provider-specific overrides below also reduce pagination fan-out.
+engines:
+  - name: bing
+    disabled: false
+    weight: 2
+    paging: false
+    timeout: 5.0
+  - name: bing images
+    paging: false
+    timeout: 5.0
+  - name: bing news
+    paging: false
+    timeout: 5.0
+  - name: bing videos
+    paging: false
+    timeout: 5.0
+  - name: duckduckgo
+    disabled: false
+    weight: 2
+    paging: false
+    timeout: 5.0
+  - name: duckduckgo images
+    paging: false
+    timeout: 5.0
+  - name: duckduckgo news
+    paging: false
+    timeout: 5.0
+  - name: duckduckgo videos
+    paging: false
+    timeout: 5.0
+  - name: wikipedia
+    weight: 1
+    timeout: 5.0
+  - name: google cse
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: google cse images
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: google news
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: google scholar
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: brave
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: brave.images
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: brave.news
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: brave.videos
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: startpage
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: startpage images
+    disabled: true
+    paging: false
+    timeout: 5.0
+  - name: startpage news
+    disabled: true
+    paging: false
+    timeout: 5.0
 
 plugins:
   searx.plugins.unit_converter.SXNGPlugin:

tests/caddy_quadlet.bats

@@ -0,0 +1,63 @@
+#!/usr/bin/env bats
+# Seam: canonical Caddy as a boot-persistent, self-healing quadlet on
+# dankclaw. It left compose.yaml on 2026-08-26 after a reboot exposed that a
+# bare compose container never comes back (rootless Podman has no boot-time
+# auto-start for bare containers; `restart: unless-stopped` does not survive
+# the explicit stop that shutdown performs). See AGENTS.md "Boot Persistence
+# & Self-Healing" and docs/Infrastructure/Caddy.md.
+
+load helpers
+
+CADDY_DIR="$REPO_ROOT/dankclaw/caddy"
+QUADLET="$CADDY_DIR/quadlet/caddy.container"
+ENV_EXAMPLE="$CADDY_DIR/env.example"
+
+@test "the caddy quadlet container file exists" {
+    [ -f "$QUADLET" ]
+}
+
+@test "the caddy quadlet is installed for boot" {
+    grep -q 'WantedBy=default.target' "$QUADLET"
+}
+
+@test "the caddy quadlet restarts on failure (self-healing)" {
+    grep -q '^Restart=always' "$QUADLET"
+    grep -qE '^RestartSec=[1-9][0-9]*$' "$QUADLET"
+}
+
+@test "the caddy quadlet publishes 80 and 443 to the LAN" {
+    grep -qE '^PublishPort=0\.0\.0\.0:80:80/tcp$' "$QUADLET"
+    grep -qE '^PublishPort=0\.0\.0\.0:443:443/tcp$' "$QUADLET"
+}
+
+@test "the caddy quadlet joins the compose network for container-name upstreams" {
+    # The Caddyfile reverse-proxies to container names (authentik-server,
+    # grafana, ...); those resolve only on the shared bridge.
+    grep -q '^Network=dankclaw_default$' "$QUADLET"
+}
+
+@test "the caddy quadlet never pulls the local-only image at boot" {
+    grep -q '^Pull=never$' "$QUADLET"
+}
+
+@test "the caddy quadlet reads its env file and mounts its state" {
+    grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' "$QUADLET"
+    grep -qE '^Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro$' "$QUADLET"
+    grep -qE '^Volume=/home/dank/Containers/caddy/data:/data$' "$QUADLET"
+    grep -qE '^Volume=/home/dank/Containers/caddy/config:/config$' "$QUADLET"
+    grep -qE '^Volume=/srv/homepage:/srv/homepage:ro$' "$QUADLET"
+}
+
+@test "the caddy env example has placeholder values only" {
+    [ -f "$ENV_EXAMPLE" ]
+    grep -q '^PORKBUN_API_KEY=changeme$' "$ENV_EXAMPLE"
+    grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$ENV_EXAMPLE"
+}
+
+@test "no secret material in the caddy files" {
+    local f
+    while IFS= read -r f; do
+        has_no_leaked_secret "$f"
+        has_no_leaked_sudo_pw "$f"
+    done < <(find "$CADDY_DIR" -type f | sort)
+}

tests/caddyfile.bats

@@ -76,13 +76,47 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
 }
 
 @test "compose no longer defines the quadlet-replaced services" {
-    # gitea, its postgres, and open-webui run as boot-persistent quadlets on
+    # gitea, its postgres, open-webui, and caddy run as boot-persistent
-    # dankclaw (see dankclaw/gitea/quadlet, dankclaw/open-webui/quadlet).
+    # quadlets on dankclaw (see dankclaw/gitea/quadlet,
-    ! grep -qE '^\s+(gitea|postgres|openwebui):\s*$' "$REPO_ROOT/dankclaw/compose.yaml"
+    # dankclaw/open-webui/quadlet, dankclaw/caddy/quadlet).
+    ! grep -qE '^\s+(gitea|postgres|openwebui|caddy):\s*$' "$REPO_ROOT/dankclaw/compose.yaml"
 }
 
-@test "the caddy service healthcheck probes the bare-IP catch-all" {
+@test "the caddy quadlet healthcheck probes the bare-IP catch-all" {
-    grep -q 'curl.*-sf.*http://localhost:80/' "$REPO_ROOT/dankclaw/compose.yaml"
+    grep -q 'HealthCmd=curl -sf -o /dev/null http://localhost:80/' \
+        "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
+}
+
+@test "plex vhosts proxy to PMS over a TLS upstream (PMS drops plain HTTP off-loopback)" {
+    # 2026-08-26 diagnosis: PMS 1.43.x on :32400 accepts, reads, and then
+    # silently closes every non-TLS connection from a non-loopback source
+    # (loopback is exempt). A plain-HTTP reverse_proxy therefore gets EOF
+    # and Caddy answers 502 with an empty body (blank page in browsers).
+    # Both plex vhosts must dial the upstream over TLS. PMS serves a
+    # self-signed cert, so verification is skipped on this pinned-LAN-IP
+    # hop only; client-facing Let's Encrypt TLS is untouched.
+    local block name
+    for name in 'plex.franksdank.cloud {' 'http://plex.franksdank.casa {'; do
+        block=$(awk -v s="$name" '
+            index($0, s) == 1 { inblk = 1 }
+            inblk { print }
+            inblk && /^}/ { exit }' "$CADDYFILE")
+        echo "$block" | grep -q 'reverse_proxy 192.168.0.235:32400' \
+            || { echo "$name: missing PMS upstream"; return 1; }
+        echo "$block" | grep -q 'transport http' \
+            || { echo "$name: upstream must use the http transport with TLS"; return 1; }
+        echo "$block" | grep -q 'tls_insecure_skip_verify' \
+            || { echo "$name: TLS upstream must set tls_insecure_skip_verify (PMS self-signed cert)"; return 1; }
+    done
+}
+
+@test "Gitea has no standalone reverse-proxy Quadlet" {
+    [ ! -e "$REPO_ROOT/dankclaw/gitea/quadlet/gitea-caddy.container" ]
+}
+
+@test "the canonical Caddy owns the Gitea route" {
+    grep -qF 'gitea.franksdank.cloud {' "$CADDYFILE"
+    grep -qF 'reverse_proxy 192.168.0.143:3000' "$CADDYFILE"
 }
 
 # --- Phase 2: real TLS via DNS-01 (Porkbun wildcard cert) ---
@@ -106,13 +140,19 @@ CADDYFILE="$REPO_ROOT/dankclaw/Caddyfile"
     grep -q 'redir https://franksdank.cloud/' "$CADDYFILE"
 }
 
-@test "the caddy service uses the locally built porkbun image" {
+@test "the caddy quadlet uses the locally built porkbun image without pulling" {
-    grep -q 'franksdank/caddy-porkbun' "$REPO_ROOT/dankclaw/compose.yaml"
+    # The image is built on-host from dankclaw/caddy/Containerfile; a boot
+    # must not stall on a registry that does not have it.
+    grep -q '^Image=franksdank/caddy-porkbun:latest$' \
+        "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
+    grep -q '^Pull=never$' "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
 }
 
-@test "the caddy service passes the Porkbun credentials from .env" {
+@test "the caddy quadlet reads the Porkbun credentials from its env file" {
-    grep -q 'PORKBUN_API_KEY: \${PORKBUN_API_KEY}' "$REPO_ROOT/dankclaw/compose.yaml"
+    grep -q '^EnvironmentFile=/home/dank/Containers/caddy/env$' \
-    grep -q 'PORKBUN_API_SECRET_KEY: \${PORKBUN_API_SECRET_KEY}' "$REPO_ROOT/dankclaw/compose.yaml"
+        "$REPO_ROOT/dankclaw/caddy/quadlet/caddy.container"
+    grep -q '^PORKBUN_API_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example"
+    grep -q '^PORKBUN_API_SECRET_KEY=changeme$' "$REPO_ROOT/dankclaw/caddy/env.example"
 }
 
 @test ".env.example carries the Porkbun secret under the module's name" {

tests/changed_tests.bats

@@ -0,0 +1,30 @@
+#!/usr/bin/env bats
+# Seam: the changed-test wrapper selects only the tests associated with
+# staged paths and refuses to silently run the entire suite.
+
+load helpers
+
+@test "changed-test wrapper is executable and uses staged diff by default" {
+    [ -x "$REPO_ROOT/scripts/test-changed.sh" ]
+    [ -x "$REPO_ROOT/.githooks/pre-commit" ]
+    run grep -F 'scripts/test-changed.sh' "$REPO_ROOT/.githooks/pre-commit"
+    [ "$status" -eq 0 ]
+    run grep -F 'git diff --cached' "$REPO_ROOT/scripts/test-changed.sh"
+    [ "$status" -eq 0 ]
+}
+
+@test "changed-test wrapper maps a compose change to compose and environment tests" {
+    run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths dankserver/compose.yaml
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"tests/compose_validity.bats"* ]]
+    [[ "$output" == *"tests/env_consistency.bats"* ]]
+    [[ "$output" != *"tests/homepage_pages.bats"* ]]
+}
+
+@test "changed-test wrapper maps a service script to its focused tests" {
+    run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/serve-frank-small.sh
+    [ "$status" -eq 0 ]
+    [[ "$output" == *"tests/scripts_syntax.bats"* ]]
+    [[ "$output" == *"tests/llm_units.bats"* ]]
+    [[ "$output" == *"tests/llm_secrets.bats"* ]]
+}

tests/deployment_flow.bats

@@ -1,8 +1,9 @@
 #!/usr/bin/env bats
 # Seam: target reachability and deployment flow integrity.
-# These tests assert the deployment contract of the unified repo:
+# These tests assert the service-scoped deployment contract:
-#   - a deploy.sh exists and accepts <dankclaw|dankserver> and --dry-run
+#   - deploy.sh accepts a target, optional service scope, and --dry-run
-#   - deploy.sh covers both the podman stack and the LLM fleet services
+#   - deploy-remote.sh owns per-service handlers and deployment state
+#   - no host-wide inference restart is hidden in the dispatcher
 #   - every host referenced in the repo is reachable over SSH (dank@host)
 #   - the local git remote is configured before deploy is attempted
 # Reachability tests are SKIPPED when the target is unreachable so the suite
@@ -38,44 +39,47 @@ load helpers
 # Regression: the Caddyfile render step in deploy.sh must survive a bcrypt
 # hash (which contains '/' characters). Reproduce the sed expression with a
 # slash-bearing hash and confirm the placeholder is replaced.
-@test "deploy.sh reloads Caddy after rendering (hot reload, not just container start)" {
+@test "scoped remote runner owns Caddy reload and safe fallback" {
-    # The caddy container often survives 'up -d' (a Caddyfile change is a
+    # Caddy is handled only when selected by the planner. A failed reload may
-    # bind-mount content change, not a container config change). Without an
+    # restart Caddy, but must not restart the whole compose project.
-    # explicit reload, a rendered fix can sit on disk while Caddy serves the
+    grep -q 'caddy reload' "$REPO_ROOT/scripts/deploy-remote.sh"
-    # stale in-memory config.
+    grep -q 'podman restart caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
-    grep -q 'caddy reload' "$REPO_ROOT/scripts/deploy.sh"
+    ! grep -qE 'podman-compose.*(up|restart).*\$\(' "$REPO_ROOT/scripts/deploy-remote.sh"
 }
 
-@test "deploy.sh restarts caddy when the reload cannot be applied" {
+@test "Caddy migration retires the standalone Gitea proxy" {
-    # If the running config has the admin API disabled (or the API is down),
+    grep -q 'retire_legacy_gitea_caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
-    # 'caddy reload' can never apply the new file -- the fix cannot be
+    grep -q 'systemctl --user stop gitea-caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh"
-    # delivered over the API it is meant to enable. The fallback is a
+    grep -q 'systemctl --user disable gitea-caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh"
-    # container restart, which always reads the file.
+    ! grep -q 'deploy_gitea_unit gitea-caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
-    grep -q 'podman restart caddy' "$REPO_ROOT/scripts/deploy.sh"
 }
 
-# The unified deploy.sh must manage the LLM fleet services on both hosts:
+@test "canonical Caddy deployment installs its boot-persistent quadlet" {
-# frank-default-small (dankserver) and frank-default-large (dankclaw). A
+    # 2026-08-26: a bare compose container never came back after reboot.
-# deploy that only restarts the podman stack leaves the LLM units pointing
+    # deploy_caddy must install, enable, and start the quadlet unit instead
-# at the retired vllmScripts checkout.
+    # of `podman-compose up`.
-@test "deploy.sh restarts the LLM services on both targets" {
+    grep -q 'install_quadlet caddy dankclaw/caddy/quadlet caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
-    grep -q 'frank-default-small' "$REPO_ROOT/scripts/deploy.sh"
+    grep -q 'systemctl --user enable caddy.service' "$REPO_ROOT/scripts/deploy-remote.sh"
-    grep -q 'frank-default-large' "$REPO_ROOT/scripts/deploy.sh"
+    ! grep -q 'podman-compose -f dankclaw/compose.yaml up -d --no-deps caddy' "$REPO_ROOT/scripts/deploy-remote.sh"
+    ! grep -q 'no proxy was stopped or replaced' "$REPO_ROOT/scripts/deploy-remote.sh"
 }
 
-@test "deploy.sh health-checks both LLM endpoints (both on :7770, by host)" {
+@test "remote runner has separate handlers for both inference services" {
-    # 2026-08-22: frank-default-large moved from :52625 to :7770 on
+    grep -q 'deploy_llm_large' "$REPO_ROOT/scripts/deploy-remote.sh"
-    # dankclaw. Both LLM services now listen on :7770, distinguished by
+    grep -q 'deploy_llm_small' "$REPO_ROOT/scripts/deploy-remote.sh"
-    # host (dankserver vs dankclaw).
-    grep -qE '^LLM_HEALTH\[dankserver\]="http://127\.0\.0\.1:7770/health"' "$REPO_ROOT/scripts/deploy.sh"
-    grep -qE '^LLM_HEALTH\[dankclaw\]="http://127\.0\.0\.1:7770/health"' "$REPO_ROOT/scripts/deploy.sh"
 }
 
-@test "deploy.sh pulls on the merged checkout on both targets" {
+@test "inference restarts are absent from the top-level dispatcher" {
-    # Both targets run the whole stack (podman + LLM) from one checkout.
+    ! grep -qE 'systemctl .*restart (frank-default-small|frank-default-large)' "$REPO_ROOT/scripts/deploy.sh"
-    local count
+}
-    count=$(grep -c '/home/dank/serverAppConfigs' "$REPO_ROOT/scripts/deploy.sh")
+
-    [ "$count" -ge 2 ]
+@test "remote runner health-checks an inference endpoint" {
+    grep -q '127.0.0.1:7770/health' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
+@test "deploy.sh pulls the merged checkout on the target" {
+    grep -q 'git pull --ff-only origin main' "$REPO_ROOT/scripts/deploy.sh"
+    grep -q '/home/dank/serverAppConfigs' "$REPO_ROOT/scripts/deploy.sh"
 }
 
 @test "deploy.sh Caddyfile render survives a slash-bearing bcrypt hash" {

tests/deployment_scoping.bats

@@ -0,0 +1,179 @@
+#!/usr/bin/env bats
+# Seam: service-scoped deployment planning.
+# A repository commit can contain changes for several workloads while only a
+# subset is safe to restart. The planner must keep inference isolated from
+# unrelated service changes and treat docs-only changes as a no-op.
+
+load helpers
+
+PLANNER="$REPO_ROOT/scripts/deploy-plan.py"
+REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
+
+@test "deployment planner maps SearXNG changes to SearXNG only" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        shared/searxng/settings.yml docs/Dankclaw/SearXNG.md
+    [ "$status" -eq 0 ]
+    [ "$output" = "searxng" ]
+}
+
+@test "deployment planner maps large-model changes to the large model only" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        scripts/frank-default-large.env scripts/frank-default-large.service
+    [ "$status" -eq 0 ]
+    [ "$output" = "frank-default-large" ]
+}
+
+@test "shared chat-template changes map to the selected host's inference service" {
+    run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja
+    [ "$status" -eq 0 ]
+    [ "$output" = "frank-default-large" ]
+
+    run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja
+    [ "$status" -eq 0 ]
+    [ "$output" = "frank-default-small" ]
+}
+
+@test "deployment planner maps small-model changes to the small model only" {
+    run python3 "$PLANNER" --target dankserver --paths \
+        scripts/frank-default-small.service
+    [ "$status" -eq 0 ]
+    [ "$output" = "frank-default-small" ]
+}
+
+@test "documentation-only changes plan no runtime deployment" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        docs/Architecture/Deployment\ Flow.md docs/Dankclaw/SearXNG.md
+    [ "$status" -eq 0 ]
+    [ -z "$output" ]
+}
+
+@test "deployment planner maps shared config to one service" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        shared/prometheus/prometheus.yml
+    [ "$status" -eq 0 ]
+    [ "$output" = "prometheus" ]
+}
+
+@test "deployment planner maps shared observability configs independently" {
+    run python3 "$PLANNER" --target dankclaw --paths shared/loki/loki-config.yaml
+    [ "$status" -eq 0 ]
+    [ "$output" = "loki" ]
+
+    run python3 "$PLANNER" --target dankclaw --paths shared/alloy/config.alloy
+    [ "$status" -eq 0 ]
+    [ "$output" = "alloy" ]
+}
+
+@test "deployment planner maps service-owned non-compose files" {
+    run python3 "$PLANNER" --target dankclaw --paths dankclaw/hermes/config.yaml.example
+    [ "$status" -eq 0 ]
+    [ "$output" = "hermes" ]
+
+    run python3 "$PLANNER" --target dankclaw --paths scripts/serve-frank-default-npu.sh scripts/frank-default-npu.env
+    [ "$status" -eq 0 ]
+    [ "$output" = "frank-default-npu" ]
+}
+
+@test "homepage is owned by Dankclaw's consolidated proxy" {
+    run python3 "$PLANNER" --target dankserver --paths shared/homepage/index.html
+    [ "$status" -eq 0 ]
+    [ -z "$output" ]
+}
+
+@test "compose planner narrows a compose edit to the changed service" {
+    local base head
+    base=$(git -C "$REPO_ROOT" rev-parse HEAD)
+    head=$(git -C "$REPO_ROOT" rev-parse HEAD)
+    run python3 "$PLANNER" --target dankclaw --repo "$REPO_ROOT" \
+        --base "$base" --head "$head" --paths dankclaw/compose.yaml
+    [ "$status" -eq 0 ]
+    [ -z "$output" ]
+}
+
+@test "network-coupled SearXNG config changes include the required peers" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        dankclaw/searxng/quadlet/gluetun.container
+    [ "$status" -eq 0 ]
+    [ "$output" = $'gluetun\nsearxng' ]
+}
+
+@test "retiring the standalone Gitea proxy deploys the canonical Caddy" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        dankclaw/gitea/quadlet/gitea-caddy.container
+    [ "$status" -eq 0 ]
+    [ "$output" = "caddy" ]
+}
+
+@test "Gitea network changes exclude the retired standalone proxy" {
+    run python3 "$PLANNER" --target dankclaw --paths \
+        dankclaw/gitea/quadlet/gitea.network
+    [ "$status" -eq 0 ]
+    [ "$output" = $'gitea-postgres\ngitea-server' ]
+}
+
+@test "remote Caddy deployment no longer blocks on the retired Gitea proxy" {
+    ! grep -q 'refusing to replace it with caddy' "$REMOTE_DEPLOY"
+    ! grep -q 'podman container exists gitea-caddy &&' "$REMOTE_DEPLOY"
+}
+
+@test "remote Caddy deployment retires the installed Gitea proxy" {
+    grep -q 'retire_legacy_gitea_caddy' "$REMOTE_DEPLOY"
+    grep -q 'systemctl --user stop gitea-caddy.service' "$REMOTE_DEPLOY"
+    grep -q 'systemctl --user disable gitea-caddy.service' "$REMOTE_DEPLOY"
+    grep -q 'gitea-caddy.container' "$REMOTE_DEPLOY"
+    grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY"
+    grep -q 'stage_legacy_gitea_caddy_state' "$REMOTE_DEPLOY"
+}
+
+@test "remote deployment inventory no longer accepts gitea-caddy as a service" {
+    ! grep -qE 'dankclaw:gitea-caddy|gitea-caddy\) deploy' "$REPO_ROOT/scripts/deploy.sh" "$REPO_ROOT/scripts/deploy-remote.sh"
+    ! grep -q '"gitea-caddy"' "$REPO_ROOT/scripts/deploy-plan.py"
+}
+
+@test "remote deployer carries per-service deployment state" {
+    [ -x "$REMOTE_DEPLOY" ]
+    grep -q 'deploy-state' "$REMOTE_DEPLOY"
+    grep -q 'STATE\[' "$REMOTE_DEPLOY"
+    grep -q 'mark_status.*pending\|status=pending' "$REMOTE_DEPLOY"
+    grep -q 'mark_status.*deployed\|status=deployed' "$REMOTE_DEPLOY"
+}
+
+@test "remote deployer routes inference restarts through selected handlers" {
+    # The runner may restart inference only from the service-dispatch path. It
+    # must not contain a host-wide restart loop.
+    grep -q 'deploy_llm_large' "$REMOTE_DEPLOY"
+    grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
+    grep -q 'deploy_service' "$REMOTE_DEPLOY"
+    ! grep -qE 'for .*frank-default-(small|large)' "$REMOTE_DEPLOY"
+}
+
+@test "remote runner rejects services belonging to the other host" {
+    run env DEPLOY_STATE_DIR="$(mktemp -d)" REMOTE_DIR="$REPO_ROOT" \
+        bash "$REMOTE_DEPLOY" dankserver "$(git rev-parse HEAD)" frank-default-large
+    [ "$status" -eq 2 ]
+}
+
+@test "remote runner recognizes a pending service at the same revision" {
+    local state_dir="$(mktemp -d)"
+    printf 'revision=%s\nstatus=pending\n' "$(git rev-parse HEAD)" > "$state_dir/unknown-service"
+    run env DEPLOY_STATE_DIR="$state_dir" REMOTE_DIR="$REPO_ROOT" \
+        bash "$REMOTE_DEPLOY" dankclaw "$(git rev-parse HEAD)" --pending
+    [ "$status" -eq 2 ]
+    [[ "$output" == *"unknown-service"* || "$output" == *"not deployed"* ]]
+}
+
+@test "top-level deployer delegates to the service-scoped remote runner" {
+    grep -q 'deploy-remote.sh' "$REPO_ROOT/scripts/deploy.sh"
+    grep -qE 'SERVICE|service' "$REPO_ROOT/scripts/deploy.sh"
+    grep -q 'OLD_REVISION' "$REPO_ROOT/scripts/deploy.sh"
+    grep -q 'OLD_REVISION\|old checkout revision' "$REPO_ROOT/scripts/deploy.sh"
+}
+
+@test "legacy service deploy wrappers delegate to the scoped deployer" {
+    for script in deploy-searxng-quadlet.sh deploy-pihole-quadlet.sh deploy-open-webui-quadlet.sh; do
+        grep -q 'deploy.sh' "$REPO_ROOT/scripts/$script" || {
+            echo "$script does not delegate to deploy.sh"
+            return 1
+        }
+    done
+}

tests/frank_small_config.bats

@@ -1,64 +0,0 @@
-#!/usr/bin/env bats
-# Seam: frank-default-small model, context, tensor placement, and template.
-# This test covers only the Q6_K deployment contract.
-
-load helpers
-
-ENV_FILE="$REPO_ROOT/scripts/frank-default-small.env"
-SERVE_FILE="$REPO_ROOT/scripts/serve-frank-small.sh"
-UNIT_FILE="$REPO_ROOT/scripts/frank-default-small.service"
-TEMPLATE_FILE="$REPO_ROOT/scripts/chat-template.jinja"
-
-@test "frank-default-small selects the Ornith Q6_K model" {
-    grep -q '^MODEL_FILE=Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf$' "$ENV_FILE"
-    grep -q 'Ornith-1\.5-35B-A3B-CRACK-Q6_K\.gguf' "$SERVE_FILE"
-}
-
-@test "frank-default-small uses 163840 context per slot across three slots" {
-    grep -q '^CTX_SIZE=491520$' "$ENV_FILE"
-    grep -q '163840 ctx each' "$ENV_FILE"
-    grep -q '^PARALLEL=3$' "$ENV_FILE"
-}
-
-@test "frank-default-small keeps full-attention and final blocks on CUDA" {
-    source "$ENV_FILE"
-    [[ "$OVERRIDE_EXPERT" == 'blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps' ]]
-    [[ "$OVERRIDE_EXPERT" != *'|3|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|7|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|11|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|15|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|19|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|23|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|27|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|31|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|35|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|36|'* ]]
-    [[ "$OVERRIDE_EXPERT" != *'|40|'* ]]
-    [[ "$OVERRIDE_BUFT" == CPU ]]
-    [[ "$N_GPU_LAYERS" == all ]]
-}
-
-@test "frank-default-small uses ngram-mod without MTP" {
-    grep -q '^SPEC_TYPE=ngram-mod$' "$ENV_FILE"
-    grep -q '^SPEC_NGRAM_MOD_N_MIN=48$' "$ENV_FILE"
-    grep -q '^SPEC_NGRAM_MOD_N_MAX=64$' "$ENV_FILE"
-    grep -q '^SPEC_NGRAM_MOD_N_MATCH=24$' "$ENV_FILE"
-    ! grep -q -- 'SPEC_DRAFT_' "$ENV_FILE"
-    ! grep -q -- '--spec-type.*draft-mtp' "$SERVE_FILE"
-    grep -q -- '--spec-ngram-mod-n-max' "$SERVE_FILE"
-}
-
-@test "frank-default-small uses ubatch 1024" {
-    grep -q '^BATCH_SIZE=4096$' "$ENV_FILE"
-    grep -q '^UBATCH_SIZE=1024$' "$ENV_FILE"
-    grep -q ': "${UBATCH_SIZE:=1024}"' "$SERVE_FILE"
-}
-
-@test "frank-default-small uses the tracked Qwen-Sharp template" {
-    grep -q '^CHAT_TEMPLATE_FILE=' "$ENV_FILE"
-    grep -q 'peculiar-ragdoll/Qwen-Sharp-Chat-Templates' "$ENV_FILE"
-    grep -q 'chat-template\.jinja' "$SERVE_FILE"
-    [ -s "$TEMPLATE_FILE" ]
-    grep -q 'qwen3\.8-froggeric-v22\.4\.0' "$TEMPLATE_FILE"
-    grep -q 'Qwen-Sharp' "$UNIT_FILE"
-}

tests/hermes_webui.bats

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Seam: hermes-webui (the Hermex server) deployment integrity.
+# The webui is the phone-facing API for the Hermes agent. It must stay
+# loopback-bound so the SSH tunnel is its only ingress, keep its password in
+# the host .env, share the gateway data volume, and deploy as an isolated
+# compose service that cannot drag the gateway or inference along with it.
+
+load helpers
+
+COMPOSE="$REPO_ROOT/dankclaw/compose.yaml"
+ENV_EXAMPLE="$REPO_ROOT/dankclaw/.env.example"
+PLANNER="$REPO_ROOT/scripts/deploy-plan.py"
+DEPLOY="$REPO_ROOT/scripts/deploy.sh"
+REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
+
+webui_service_yaml() {
+    python3 - "$COMPOSE" <<'PYEOF'
+import sys, yaml
+with open(sys.argv[1]) as fh:
+    data = yaml.safe_load(fh)
+svc = (data.get("services") or {}).get("hermes-webui")
+assert svc, "dankclaw/compose.yaml: no hermes-webui service"
+print(yaml.safe_dump(svc))
+PYEOF
+}
+
+@test "hermes-webui runs the webui bootstrap in foreground mode" {
+    local y
+    y=$(webui_service_yaml)
+    grep -q 'nousresearch/hermes-agent' <<< "$y"
+    grep -q 'bootstrap.py' <<< "$y"
+    grep -q -- '--foreground' <<< "$y"
+}
+
+@test "hermes-webui is loopback-bound and password-gated" {
+    local y
+    y=$(webui_service_yaml)
+    # Host-side publish stays on loopback: the SSH tunnel is the only ingress.
+    grep -q '127.0.0.1:8787' <<< "$y"
+    grep -q 'HERMES_WEBUI_PASSWORD' <<< "$y"
+    # Container-internal bind must be 0.0.0.0 or Podman's port forwarder
+    # (which dials the bridge IP) resets every connection.
+    grep -q 'HERMES_WEBUI_HOST: 0.0.0.0' <<< "$y"
+}
+
+@test "hermes-webui shares the gateway data volume and mounts the webui checkout" {
+    local y
+    y=$(webui_service_yaml)
+    grep -q 'HERMES_HOME: /data' <<< "$y"
+    grep -q '/home/dank/hermes-data:/data' <<< "$y"
+    grep -q '/home/dank/hermes-webui:/webui' <<< "$y"
+}
+
+@test "hermes-webui has a healthcheck that probes /health" {
+    local y
+    y=$(webui_service_yaml)
+    grep -q 'healthcheck' <<< "$y"
+    grep -q '/health' <<< "$y"
+}
+
+@test "dankclaw .env.example defines the webui password placeholder" {
+    env_vars_defined "$ENV_EXAMPLE" | grep -qx 'HERMES_WEBUI_PASSWORD'
+}
+
+# Revision-aware planner: a change to only the hermes-webui service block must
+# plan hermes-webui and nothing else (the gateway is unchanged).
+webui_fixture_repo() {
+    local dir="$1"
+    git init -q "$dir"
+    git -C "$dir" config user.email t@t && git -C "$dir" config user.name t
+    mkdir -p "$dir/dankclaw"
+    cat > "$dir/dankclaw/compose.yaml" <<'EOF'
+services:
+  hermes:
+    image: docker.io/nousresearch/hermes-agent:latest
+  hermes-webui:
+    image: docker.io/nousresearch/hermes-agent:latest
+    ports:
+      - "127.0.0.1:8787:8787"
+EOF
+    git -C "$dir" add -A && git -C "$dir" commit -qm base
+    # Touch only the webui service block (its port line is unique in the file).
+    sed -i 's|127.0.0.1:8787:8787|127.0.0.1:8788:8787|' "$dir/dankclaw/compose.yaml"
+    git -C "$dir" commit -qam head
+}
+
+@test "planner narrows a hermes-webui compose edit to hermes-webui only" {
+    local dir base head
+    dir=$(mktemp -d)
+    webui_fixture_repo "$dir"
+    base=$(git -C "$dir" rev-parse HEAD~1)
+    head=$(git -C "$dir" rev-parse HEAD)
+    run python3 "$PLANNER" --target dankclaw --repo "$dir" \
+        --base "$base" --head "$head" --paths dankclaw/compose.yaml
+    [ "$status" -eq 0 ]
+    [ "$output" = "hermes-webui" ]
+}
+
+@test "remote deployer accepts and dispatches hermes-webui" {
+    grep -q 'dankclaw:hermes-webui' "$REMOTE_DEPLOY"
+    awk '/^deploy_compose_service\(\)/,/^}/' "$REMOTE_DEPLOY" | grep -q 'hermes-webui'
+}
+
+@test "top-level deployer accepts hermes-webui as a Dankclaw service" {
+    grep -q 'dankclaw:hermes-webui' "$DEPLOY"
+}

tests/homepage_deploy.bats

@@ -10,10 +10,12 @@ load helpers
 
 @test "deploy.sh ships the homepage (assets, timer, basicauth hash, deployed.js)" {
     local d="$REPO_ROOT/scripts/deploy.sh"
-    grep -q "/srv/homepage" "$d"
+    local r="$REPO_ROOT/scripts/deploy-remote.sh"
-    grep -q "homepage-status" "$d"
+    grep -q "deploy-remote.sh" "$d"
-    grep -q "CADDY_BASICAUTH_HASH" "$d"
+    grep -q "/srv/homepage" "$r"
-    grep -q "deployed.js" "$d"
+    grep -q "homepage-status" "$r"
+    grep -q "CADDY_BASICAUTH_HASH" "$r"
+    grep -q "deployed.js" "$r"
 }
 
 @test "the checker runs as a systemd timer pointing at the checker" {
@@ -27,5 +29,5 @@ load helpers
 }
 
 @test "deploy.sh is git-driven: it pulls main on the target" {
-    grep -q "git pull origin main" "$REPO_ROOT/scripts/deploy.sh"
+    grep -q "git pull --ff-only origin main" "$REPO_ROOT/scripts/deploy.sh"
 }

tests/mount_entropy.bats

@@ -0,0 +1,25 @@
+#!/usr/bin/env bats
+# Seam: the entropy helper unlocks the known LUKS-on-md device and mounts via
+# the host's fstab entry, preserving the existing mount options and directory
+# ownership rather than inventing a second mount configuration.
+
+load helpers
+
+@test "entropy helper is executable and has the expected safe shell shape" {
+    run bash -c "test -x '$REPO_ROOT/scripts/mount-entropy.sh'"
+    [ "$status" -eq 0 ]
+
+    run grep -E '^set -euo pipefail$' "$REPO_ROOT/scripts/mount-entropy.sh"
+    [ "$status" -eq 0 ]
+
+    run grep -F 'udisksctl unlock --block-device /dev/md0' "$REPO_ROOT/scripts/mount-entropy.sh"
+    [ "$status" -eq 0 ]
+
+    run grep -F 'sudo mount "$MOUNTPOINT"' "$REPO_ROOT/scripts/mount-entropy.sh"
+    [ "$status" -eq 0 ]
+}
+
+@test "entropy helper does not embed a secret or alter mount ownership" {
+    run grep -Eiq 'password|secret|chmod|chown' "$REPO_ROOT/scripts/mount-entropy.sh"
+    [ "$status" -eq 1 ]
+}

tests/pihole.bats

@@ -48,6 +48,20 @@ HOSTS="$PIHOLE_DIR/addn-hosts"
     grep -qE '^Volume=/home/dank/Containers/pihole/addn-hosts:/etc/pihole/hosts/addn-hosts:ro$' "$QUADLET"
 }
 
+@test "the pihole sysctl drop-in pins the unprivileged port threshold to 53" {
+    # Rootless port 53 needs ip_unprivileged_port_start <= 53 at boot. The
+    # repo drop-in is the single source of truth; a later-sorted file that
+    # reset it to 80 took LAN DNS down after the 2026-08-26 reboot.
+    local f="$PIHOLE_DIR/sysctl/99-dnsmasq.conf"
+    [ -f "$f" ]
+    grep -q '^net.ipv4.ip_unprivileged_port_start=53$' "$f"
+}
+
+@test "deploy_pihole installs the sysctl drop-in and applies it at boot-path" {
+    grep -q 'dankclaw/pihole/sysctl/99-dnsmasq.conf' "$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q 'sysctl --system' "$REPO_ROOT/scripts/deploy-remote.sh"
+}
+
 @test "the pihole env example has placeholder values only" {
     [ -f "$ENV_EXAMPLE" ]
     grep -q '^WEBPASSWORD=changeme$' "$ENV_EXAMPLE"

tests/searxng_limiter.bats

@@ -57,9 +57,42 @@ assert not missing, f"pass_ip missing {missing}; got {pass_ip}"
 PYEOF
 }
 
+@test "SearXNG outgoing policy allows modest VPN latency without retry amplification" {
+    python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
+import sys, yaml
+with open(sys.argv[1]) as fh:
+    data = yaml.safe_load(fh)
+outgoing = data.get("outgoing", {})
+assert outgoing.get("request_timeout") == 5.0, outgoing
+assert outgoing.get("max_request_timeout") == 15.0, outgoing
+assert outgoing.get("retries") == 1, outgoing
+assert outgoing["request_timeout"] < outgoing["max_request_timeout"], outgoing
+PYEOF
+}
+
+@test "SearXNG default provider policy favors anonymous healthy engines" {
+    python3 - "$REPO_ROOT/shared/searxng/settings.yml" <<'PYEOF'
+import sys, yaml
+with open(sys.argv[1]) as fh:
+    data = yaml.safe_load(fh)
+engines = {entry["name"]: entry for entry in data.get("engines", [])}
+assert engines["bing"]["disabled"] is False, engines.get("bing")
+assert engines["bing"]["weight"] > 1, engines["bing"]
+assert engines["bing"]["paging"] is False, engines["bing"]
+assert engines["duckduckgo"]["disabled"] is False, engines["duckduckgo"]
+assert engines["duckduckgo"]["paging"] is False, engines["duckduckgo"]
+for name in ("google cse", "startpage", "brave"):
+    assert engines[name]["disabled"] is True, engines.get(name)
+    assert engines[name].get("inactive", False) is False, engines[name]
+for name in ("bing", "duckduckgo", "wikipedia"):
+    assert engines[name]["timeout"] == 5.0, engines[name]
+PYEOF
+}
+
 @test "deploy.sh copies limiter.toml into /opt/searxng/config" {
-    grep -q "shared/searxng/limiter.toml" "$REPO_ROOT/scripts/deploy.sh" \
+    local d="$REPO_ROOT/scripts/deploy-remote.sh"
-        || { echo "deploy.sh does not copy shared/searxng/limiter.toml"; return 1; }
+    grep -q "shared/searxng/limiter.toml" "$d" \
-    grep -q "/opt/searxng/config/limiter.toml" "$REPO_ROOT/scripts/deploy.sh" \
+        || { echo "deploy-remote.sh does not copy shared/searxng/limiter.toml"; return 1; }
-        || { echo "deploy.sh does not place limiter.toml at /opt/searxng/config"; return 1; }
+    grep -q "/opt/searxng/config/limiter.toml" "$d" \
+        || { echo "deploy-remote.sh does not place limiter.toml at /opt/searxng/config"; return 1; }
 }