Update AGENTS.md with target-specific branching and service status

- vllm-awq-qwen36-dflash-ampere.sh: AWQ-4bit Qwen3.6-35B-A3B with DFlash
  speculative decoding, optimized for Ampere hardware
  - Model: cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit (AWQ Marlin)
  - Draft: z-lab/Qwen3.6-35B-A3B-DFlash
  - --max-model-len 32768
  - --max-num-batched-tokens 1536
  - --cpu-offload-gb 40 (experts)
  - --reasoning-parser qwen3
  - --tool-call-parser qwen3_xml
  - Port: 34401
  - Note: prefix caching disabled (Mamba cache align conflict)

- scripts/validate-vllm.sh: Per-stream latency tests
  - Separate tests for 1, 2, 3, 4 streams
  - Different prompts per stream to avoid caching effects
  - Reports elapsed ms per stream count

.AGENTTRASH/frank-default-small.env.llama.cpp

@@ -1,71 +0,0 @@
-# Tunables for frank-default-small (Tiel-Coder-35B-A3B UD-Q6_K_XL,
-# qwen35moe hybrid MoE, Dankserver, RTX 3090).
-#
-# Runtime: upstream llama.cpp (latest master, CUDA build with
-# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86) on RTX 3090
-# 24 GB; Ryzen 7 7700X; 62 GB RAM.  Replaced the ik_llama.cpp Ornith
-# CRACK Q8_0 deployment on 2026-09-01.
-#
-# Model: peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF, UD-Q6_K_XL quant
-# (30 GB).  Architecture: qwen35moe — 40 layers, full_attention_interval=4
-# (10 attention + 30 SSM/Mamba), 256 experts / 8 used, 2 KV heads × 256
-# dim, embedding 2048, rope dimension_count 64, rope freq_base 10 000 000.
-# Text-only (no mmproj).  Built-in chat template in the GGUF.
-#
-# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
-# Non-expert weights (~1.6 GB) fit entirely on the 24 GB card.
-#
-# KV cache: Q8_0 for both K and V.  With CTX_SIZE=200700 (66 900 per slot
-# × 3 parallel) and 10 attention layers, the KV cache is ~2 GB — the
-# requested 2 GB VRAM buffer.
-#
-# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
-# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
-# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
-
-# Model file (UD-Q6_K_XL, 30 GB, text-only MoE hybrid).
-MODEL_DIR=/home/dank/models/tiel-coder-35b
-MODEL_FILE=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf
-
-# Upstream llama.cpp llama-server (CUDA build, latest master).
-# Built with -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86
-# using the staged CUDA 13.2 toolkit at /home/dank/cuda13-stage.
-LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
-
-ALIAS=frank-default-small
-BIND_ADDR=0.0.0.0
-LISTEN_PORT=7770
-
-# Total context budget across all parallel slots.  200700 / 3 = 66 900
-# per slot.  The model's context_length is 262 144, so 66 900 is well
-# within range.
-CTX_SIZE=200700
-PARALLEL=3
-
-# Expert offload: all MoE expert weights pinned to CPU RAM.
-CPU_MOE=1
-
-# All non-expert layers on GPU (auto-fit will determine the exact count;
-# with --cpu-moe the non-expert footprint is ~1.6 GB, leaving ample room).
-N_GPU_LAYERS=999
-
-# KV cache: Q8_0 for both K and V.  ~2 GB total for 200700 ctx × 3 slots
-# × 10 attention layers × (2 KV heads × 256 dim × 1 byte) × 2 (K+V).
-CACHE_TYPE_K=q8_0
-CACHE_TYPE_V=q8_0
-
-VERBOSITY=2
-
-# Prefill batch sizing.
-BATCH_SIZE=4096
-UBATCH_SIZE=1024
-
-# Sampling.
-TEMPERATURE=1.0
-TOP_K=20
-TOP_P=0.95
-MIN_P=0.0
-
-# Compute threads lowered so other processes keep a core.
-THREADS=7
-THREADS_HTTP=8

.AGENTTRASH/frank-default-small.service.llama.cpp

@@ -1,16 +0,0 @@
-[Unit]
-Description=llama.cpp Tiel-Coder-35B-A3B Q6_K_XL qwen35moe hybrid MoE (dankserver, frank-default-small)
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User=dank
-Group=dank
-Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-default-small.env"
-ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
-Restart=on-failure
-RestartSec=10
-
-[Install]
-WantedBy=multi-user.target

.AGENTTRASH/serve-frank-small.sh.llama.cpp

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-# Serve peculiar-ragdoll/Tiel-Coder-35B-A3B (UD-Q6_K_XL) as
-# frank-default-small via stock llama.cpp on Dankserver (RTX 3090).
-#
-# Runtime: upstream llama.cpp (latest master, CUDA build with
-# -DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86).
-# The model is a 35B-total / 8-active MoE hybrid (qwen35moe arch) with
-# SSM/Mamba layers (full_attention_interval=4: 10 attention + 30 SSM
-# layers out of 40).  256 experts, 8 used per token.
-#
-# Expert offload: --cpu-moe keeps all 256 expert weights in CPU RAM.
-# Non-expert weights (attention, SSM, shared expert, embeddings, output)
-# are ~1.6 GB and fit entirely on the 24 GB RTX 3090.
-#
-# KV cache: Q8_0 for both K and V.  With CTX_SIZE=200700 (66 900 per
-# slot × 3 parallel) and 10 attention layers (2 KV heads × 256 dim),
-# the KV cache is ~2 GB — the requested 2 GB VRAM buffer.
-#
-# Model is loaded with mmap (default).  The 30 GB Q6_K_XL file is
-# memory-mapped; only accessed pages enter RAM.
-#
-# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
-# overrides that path.
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
-if [[ -f "$ENV_FILE" ]]; then
-    set -a
-    # shellcheck disable=SC1090
-    source "$ENV_FILE"
-    set +a
-fi
-
-# --- defaults (used when the env file does not set them) -----------------
-: "${MODEL_DIR:=/home/dank/models/tiel-coder-35b}"
-: "${MODEL_FILE:=Tiel-Coder-35B-A3B-UD-Q6_K_XL.gguf}"
-: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
-: "${ALIAS:=frank-default-small}"
-: "${BIND_ADDR:=0.0.0.0}"
-: "${LISTEN_PORT:=7770}"
-: "${CTX_SIZE:=200700}"
-: "${PARALLEL:=3}"
-: "${CACHE_TYPE_K:=q8_0}"
-: "${CACHE_TYPE_V:=q8_0}"
-: "${N_GPU_LAYERS:=999}"
-: "${BATCH_SIZE:=4096}"
-: "${UBATCH_SIZE:=1024}"
-: "${VERBOSITY:=2}"
-: "${CPU_MOE:=1}"
-: "${TEMPERATURE:=1.0}"
-: "${TOP_K:=20}"
-: "${TOP_P:=0.95}"
-: "${MIN_P:=0.0}"
-: "${THREADS:=7}"
-: "${THREADS_HTTP:=8}"
-
-MODEL="$MODEL_DIR/$MODEL_FILE"
-
-if [[ ! -f "$MODEL" ]]; then
-    echo "missing required file: $MODEL" >&2
-    exit 1
-fi
-if [[ ! -x "$LLAMA_SERVER" ]]; then
-    echo "missing llama-server binary: $LLAMA_SERVER" >&2
-    exit 1
-fi
-
-exec "$LLAMA_SERVER" \
-    --model "$MODEL" \
-    $([ "$CPU_MOE" = "1" ] && printf -- "--cpu-moe") \
-    --cache-type-k "$CACHE_TYPE_K" \
-    --cache-type-v "$CACHE_TYPE_V" \
-    --ctx-size "$CTX_SIZE" \
-    --parallel "$PARALLEL" \
-    --gpu-layers "$N_GPU_LAYERS" \
-    --batch-size "$BATCH_SIZE" \
-    --ubatch-size "$UBATCH_SIZE" \
-    --temp "$TEMPERATURE" \
-    --top-k "$TOP_K" \
-    --top-p "$TOP_P" \
-    --min-p "$MIN_P" \
-    --threads "$THREADS" \
-    --threads-http "$THREADS_HTTP" \
-    --alias "$ALIAS" \
-    --host "$BIND_ADDR" \
-    --port "$LISTEN_PORT"

.githooks/pre-commit

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-# Repository pre-commit hook: run only tests selected by staged paths.
-set -euo pipefail
-
-REPO_ROOT="$(git rev-parse --show-toplevel)"
-exec "$REPO_ROOT/scripts/test-changed.sh"

.gitignore

@@ -1,29 +1,6 @@
-# Local environment files — never commit real values
-.env
-*.env
-!.env.example
-# LLM fleet tunables (paths and numbers, never credentials)
-!scripts/*.env
-
-# pi subagent artifacts
 .pi-subagents/
-
-# doc-audit scratch (transient, synthesized into docs/ then trashed)
-.docaudit/
-
-# Agent trash
 .AGENTTRASH/
-
-# Python
 __pycache__/
 *.pyc
-
-# Virtual environments
 .venv/
-
-# Obsidian editor state
 .obsidian/
-
-# Editor/OS noise
-*.swp
-.DS_Store

AGENTS.md

@@ -1,476 +1,103 @@
-# AGENTS.md — serverAppConfigs Agent Rules
+# vLLM Scripts — Agent Instructions
 
-## Overview
+## Project Overview
+Scripts for running large language models via vLLM on two local machines:
+- **dankserver**: NVIDIA GPU + System RAM (frank-default-small)
+- **dankclaw**: AMD Strix Halo Shared Memory (frank-default-large)
 
-This repository is the single source of truth for the two homelab servers. It
-tracks both the Podman container stacks and the LLM inference fleet. One repo,
-one checkout per host, one deploy script.
+## Directory Structure
+```
+vllmScripts/
+├── AGENTS.md          # This file — agent instructions
+├── README.md          # Human-readable project overview
+├── deploy.sh          # Automated deployment script
+├── scripts/           # Executable vLLM launch scripts
+│   ├── validate-vllm.sh  # Validation tool
+│   └── *.sh             # Inference server scripts
+├── tests/             # Test suite
+│   └── test_validator.sh
+├── docs/              # Knowledge base
+│   ├── README.md
+│   └── *.md           # Research notes
+└── failedAttempts/    # Archived scripts (for reference)
+```
 
-- **Dankserver** (192.168.0.235) — media/library stack (RomM, Copyparty,
-  Calibre-Web, the *arr stack, Jellyfin) as bare containers, plus the
-  `frank-default-small` LLM service.
-- **Dankclaw** (192.168.0.143) — AI/search/development/observability stack
-  (SearXNG via Gluetun, ComfyUI, OpenWebUI, Hermes, Gitea, Grafana, Prometheus,
-  Loki, Alloy, Pi-hole) plus the `frank-default-large` LLM service. Caddy on
-  Dankclaw reverse-proxies both hosts.
+## Running Tests
+```bash
+# Test validator on dankserver
+bash tests/test_validator.sh dankserver
 
-**2026-08-22 merge:** the former `vllmScripts` repository (LLM fleet scripts,
-units, env tunables, and runbook) is merged into this repo. Its full git
-history is preserved (two-parent import commit under the retired `llm/`
-prefix, reorganized in follow-up commits). The retired target checkouts
-(`/home/dank/vLLM` on Dankserver, `/home/dank/vllmScripts` on Dankclaw) are
-archived. All services now run from `/home/dank/serverAppConfigs` on each
-host.
+# Test validator on dankclaw
+bash tests/test_validator.sh dankclaw
 
-### LLM fleet topology (2026-08-22)
+# Run all tests
+bash tests/test_validator.sh dankserver && bash tests/test_validator.sh dankclaw
+```
 
-| Target | Runtime | Service | Model | Port |
-|--------|---------|---------|-------|------|
-| Dankserver | llama.cpp (upstream b10121) | frank-default-small | HauhauCS Qwen3.6-35B-A3B-Uncensored-Aggressive Q6_K_P + ngram-mod, 36 MoE experts on CPU | 7770 |
-| Dankclaw | charlie12345/ROCmFPX llama.cpp | frank-default-large | kingjones Qwen3.8 Uncensored Q6_0_ROCmFPX_AGENT (standalone MTP draft head, Turbo4 V cache, 262K context / 1 slot), reasoning on | 7770 |
+## Deployment
+```bash
+# Deploy all scripts to all targets
+bash deploy.sh all
 
-History: the 2026-08-17 redo retired the pipeline-parallel split and the
-FastFlowLM NPU endpoint. The 2026-08-18 swap retired the vLLM service on
-Dankclaw. On 2026-08-21 a DFlash2 Vulkan deployment briefly replaced CIRU,
-then was reverted the same day for speed. On 2026-08-22 the service briefly
-swapped to the Kairic Edge IU4 runtime, then was reverted to v3 the same day
-(operator decision). On 2026-08-25 the service switched to the kingjones
-Qwen3.8-27B-Uncensored-Q6_0-ROCmFPX_AGENT model (standalone MTP draft,
-262144 context, reasoning on). On 2026-09-01 both services were swapped:
-Dankserver switched from ik_llama.cpp Ornith CRACK Q8_0 to stock llama.cpp
-Tiel-Coder-35B (qwen35moe hybrid MoE), and Dankclaw switched from the Q6_0
-AGENT model to the Flash-Next 56B MoE (qwen4exp, PR #27742 merged into
-charlie's ROCmFPX fork, --fit off, 128k context). Runbooks:
-[docs/LLM/index.md](docs/LLM/index.md).
-Retired scripts live in `old/` (reference only) and `.AGENTTRASH/`.
+# Deploy specific script to dankserver
+bash deploy.sh dankserver vllm-qwen36.sh
 
-## Repository Layout
-
-- `dankserver/` — compose.yaml + Caddyfile + .env.example for the media stack
-- `dankclaw/` — compose.yaml + Caddyfile + .env.example for the AI stack
-- `shared/` — homepage, Prometheus, Loki, Alloy, SearXNG configs
-- `scripts/` — `deploy.sh`, `deploy-plan.py`, and `deploy-remote.sh`
-  (service-scoped deployer), LLM serve scripts, systemd units, env tunables,
-  and bench harnesses
-- `old/` — retired PP/NPU-era LLM scripts (reference only)
-- `docs/` — knowledge base (Obsidian vault). `docs/LLM/` is the LLM fleet
-- `tests/` — bats test suite
-
-## Documentation Authoring
-
-The `docs/` vault is the project knowledge base (Obsidian, wikilinks) and is an
-OKF (Open Knowledge Format) knowledge bundle. Follow the OKF spec for every doc
-you create or edit.
-
-- **Canonical spec:** `~/.pi/agent/docs/references/okf-spec.md` — the OKF v0.1
-  spec in the **system-config knowledge base** at `~/.pi/agent/docs`. The repo's
-  own `docs/references/okf-spec.md` is deprecated/removed; use the system one. The
-  spec index and maintenance guide are
-  `~/.pi/agent/docs/references/index.md` and
-  `~/.pi/agent/docs/references/okf-maintenance.md`.
-- **Read before you touch:** open the relevant `index.md` first
-  (`docs/index.md` → `docs/LLM/index.md`). Read the whole target file before
-  editing it.
-- **One concept per file**, with YAML frontmatter that has a non-empty `type:`
-  field (`reference` / `guide` / `configuration` / `research`) plus `description`,
-  `tags`, and `timestamp`. Split broad topics; keep pages ~1–500 lines.
-- **Progressive disclosure:** an `index.md` at each level links its children.
-  Every `docs/**/*.md` (except `index.md`) must be linked from
-  `docs/index.md` with `[[Title]]` — the docs-completeness test checks this.
-- **Cross-link** siblings as `[[Title]]` wikilinks; cite external claims under a
-  `# Citations` section.
-- **Always update docs** when a task changes behavior, then run
-  `./tests/run.sh` (the docs-completeness seam is in there) before commit.
+# Deploy specific script to dankclaw
+bash deploy.sh dankclaw vllm-thinkingcap.sh
+```
 
 ## Git Workflow
 
-All edits flow through git. Do not edit files on the remote machine directly.
+### Branching Strategy
+- `main`: Stable, working scripts only
+- `dankserver/*`: Server-specific development
+  - `dankserver/qwen36-awq`: Qwen3.6 AWQ experiments
+  - `dankserver/*`: Feature branches for dankserver
+- `dankclaw/*`: Claw-specific development
+  - `dankclaw/*`: Feature branches for dankclaw
 
-### Protocol
+### Worktree Pattern
+For parallel development:
+```bash
+# Create worktree for feature
+git worktree add ../vllm-feature feature/qwen36-awq
 
-1. **Source the config from the target machine** — fetch the live
-   compose/Caddyfile/serve script from the server where it runs
-2. **Place it in this repo** — commit the fetched version (redact secrets
-   into `.env`)
-3. **Edit here** — make all changes locally in the working tree
-4. **Run the tests** — `./tests/run.sh` must pass
-5. **Commit and push** — push to origin (Gitea on Dankclaw)
-6. **Pull on the target** — `git pull` on the remote machine
-7. **Deploy** — `scripts/deploy.sh` pulls, plans, and dispatches only the
-   changed service(s), then checks the selected service health
-
-## Deployment
-
-Use `scripts/deploy.sh` to deploy committed changes to a target. The planner
-maps changed paths to individual services; the remote runner restarts only the
-selected service and records its successful revision outside the checkout.
-
-```
-./scripts/deploy.sh dankclaw
-./scripts/deploy.sh dankserver
-./scripts/deploy.sh dankclaw --service searxng
-./scripts/deploy.sh dankserver --service frank-default-small
-./scripts/deploy.sh dankclaw --dry-run
+# When done, merge back
+git worktree remove ../vllm-feature
 ```
 
-A documentation-only change is a no-op. The LLM services are selected only by
-their own unit/env/serve files or the shared chat template, so unrelated
-container changes cannot interrupt inference. `frank-default-npu` is relink-only
-when explicitly selected.
-
-Set `FRANK_SUDO_PW` for headless runs on a host without passwordless sudo
-(remote sudo reads it from stdin via `sudo -S`). The value is never stored
-in this repo.
-
-```mermaid
-graph LR
-    A[Edit locally] --> B[Tests pass]
-    B --> C[Commit]
-    C --> D[Push to origin]
-    D --> E[deploy.sh computes service plan]
-    E --> F[Target pulls with ff-only]
-    F --> G[deploy-remote.sh dispatches selected service]
-    G --> H[Per-service health check]
-```
-
-## Boot Persistence & Self-Healing (MANDATORY)
-
-Every Podman container governed by this project that is successfully deployed
-and proven working MUST have a self-healing, boot-persistent Quadlet unit. A
-container that runs only from `compose.yaml` (or a manual `podman run`) dies
-silently at reboot: rootless Podman does not auto-start bare containers, and
-`restart: unless-stopped` does not survive the explicit stop that shutdown
-performs. Do not report such a service as "up" without its unit.
-
-Each container's unit must provide:
-
-- **Boot persistence** — `[Install] WantedBy=default.target`, installed to
-  `~/.config/containers/systemd/<name>.container` (generating
-  `<name>.service`) and enabled in the user manager
-  (`systemctl --user enable <name>.service`). Linger must be on for the deploy
-  user (`loginctl show-user dank -p Linger` → `Linger=yes`) so user units start
-  at boot with no login session.
-- **Self-healing** — `[Service] Restart=always` (or `Restart=on-failure`) with
-  a short `RestartSec`, so a crash or OOM brings the container back without
-  operator action. Add a quadlet health check where the service has one.
-- **Repo provenance** — the `.container` file lives in this repo
-  (`dankclaw/<svc>/quadlet/` or `dankserver/<svc>/quadlet/`), is installed by
-  the service-scoped deployer, and is covered by the quadlet test seam. The
-  host-side unit is generated state; the repo file is the source of truth.
-
-Verification before a service is declared deployed:
-`systemctl --user is-enabled <name>.service` → `enabled`,
-`systemctl --user is-active <name>.service` → `active`, and the container
-answers its health check. A running container with no unit is a gap: write the
-unit, test it, deploy it.
-
-Incident that established this rule (2026-08-26, Dankclaw reboot): Caddy ran
-for weeks as a bare compose container and never came back after reboot; the
-Pi-hole unit existed but crash-looped on a sysctl conflict (a later-sorted
-drop-in reset `net.ipv4.ip_unprivileged_port_start` to 80), taking LAN DNS
-down with it.
-
-## Testing
-
-Tests are mandatory before commit. `bats` must be installed on the
-development machine.
-
-```
-./tests/run.sh
-```
-
-Test seams: compose validity, env consistency (no leaked secrets), Caddyfile
-validity, quadlet units, docs completeness, script syntax (all of `scripts/`,
-including the LLM serve scripts), LLM unit integrity (units point at files in
-this repo), LLM secret hygiene, deployment flow, and service-scoped deployment
-planning/dispatch. See [docs/Architecture/Testing Policy.md](docs/Architecture/Testing%20Policy.md).
-
-## Secret Rules
-
-- Never commit real secrets. The live Gitea database password and the live
-  sudo password are test-enforced as leak checks.
-- `.env` files are git-ignored. Commit only `.env.example` with placeholders.
-  The LLM env tunables in `scripts/` are tracked by exception (`!scripts/*.env`).
-  They hold paths and numbers, never credentials.
-- Do not document passwords in this repo, including sudo passwords. SSH keys
-  are the auth mechanism. Headless sudo passes the password via the
-  `FRANK_SUDO_PW` environment variable. The value never enters the repo.
-
-## Deletion & Trash Policy (MANDATORY)
-
-Never run `rm` to delete files. The command deletes permanently. Move every
-item you delete into trash instead. The trash keeps the deletion recoverable
-and reviewable.
-
-1. In-repo files: move the item to `.AGENTTRASH/` beside its directory.
-2. Home and other paths: create a dated staging directory
-   `~/.AGENTTRASH/YYYYMMDD-topic` and move each item there.
-3. List the staged items and their sizes for the user.
-4. Wait for explicit user permission before you empty the trash.
-
-Rules:
-
-- `rm`, `rm -rf`, and `git clean -f` are forbidden without explicit user
-  permission.
-- `git * --no-verify` is forbidden.
-- A moved item stays in the trash until the user approves its purge.
-- The trash is a temporary hold, not a long-term archive.
-
-## Benchmarking Policy (MANDATORY)
-
-**STOP the production service BEFORE running any benchmark on a target.**
-These servers serve real users (about 3 people). Running a bench while the
-service is up starves the GPU and corrupts results. Restart the service when
-the bench finishes.
-
-ALWAYS bench using llama-bench for llama.cpp or vllm bench for vllm runtimes.
-DO NOT ROLL YOUR OWN BENCH UNLESS YOU CANNOT TEST THE DESIRED PARAMETER WITH THE OFFICIAL BENCHES. ASK THE USER BEFORE DEVIATING FROM THIS DIRECTIVE.
-
-1. Stop the service before any benchmark:
-   `sudo systemctl stop frank-default-small` (Dankserver) or
-   `sudo systemctl stop frank-default-large` (Dankclaw).
-2. Run the bench (`llama-bench` for llama.cpp / vllm bench for vLLM).
-   Scratch ports are not used for tests or benches on the LLM hosts — a
-   test process cannot share the single GPU with the running service anyway
-   (the model + KV fill the card), so changes are validated directly against
-   the service itself.
-3. Restart the service when the bench completes: `sudo systemctl start <svc>`.
-4. Verify health before reporting:
-   `curl http://192.168.0.235:7770/health` or
-   `curl http://192.168.0.143:7770/health`.
-
-## Pi Model Config (`~/.pi/agent/models.json`)
-
-The Pi agent reaches the fleet through `/home/dank/.pi/agent/models.json`.
-This file lives outside the repository. Each provider is one host endpoint.
-Each model `id` is the served alias. The two aliases are
-`frank-default-small` (Dankserver `:7770`) and `frank-default-large`
-(Dankclaw `:7770`).
-
-The model `id` values are stable. The file
-`/home/dank/.pi/agent/settings.json` references them. Two resolution rules
-apply (verified in `@earendil-works/pi-coding-agent` `model-resolver.js`):
-
-- The default model resolves as a `defaultProvider` + `defaultModel` pair.
-  The `defaultProvider` must name the provider that serves the `defaultModel`.
-- A subagent override uses a bare model `id`. A bare `id` resolves only when
-  it is unique across all providers.
-
-When you rename a service or move a model to a new host, update
-`models.json`. Update the provider key, the model `id`, `contextWindow`, and
-`input`. If the default model moved hosts, also update `defaultProvider` in
-`settings.json`. Set `contextWindow` to the per-slot context (both hosts
-`262144`). The full schema is in the agent knowledge base page
-`~/.pi/agent/docs/pi/models.md`.
-
-## Runtime & Troubleshooting
-
-### Qwen3.8-Flash-Next ROCmFP4 STRIX_LEAN llama.cpp (Dankclaw, ACTIVE)
-
-The large service serves
-`kingjones777/Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN-GGUF` (public, 98.5 GiB)
-from `/home/dank/models/kingjones-Qwen3.8-Flash-Next-ROCmFP4-STRIX_LEAN/`
-with a charlie12345/ROCmFPX `llama-server` built at
-`/home/dank/GitPackages/ROCmFPX/build-qwen38` (PR #27742 qwen4exp architecture
-merged). The model is a 56B MoE (512 experts, 10 used), 48 layers (12 attn +
-36 SSM, full_attention_interval=4), QSA indexer (4 heads, top_k=2048),
-hyper-connections (count=4, low_rank=320), PLE (layer 1). Q4_0_ROCMFP4 quant.
-It runs f16 KV cache, 131072 context / 1 slot, `--fit off` (auto-fit segfaults
-on qwen4exp hyper-connection config), and reasoning on (Sharp template, budget
--1, `--reasoning-format deepseek`; chat kwargs keep `preserve_thinking` +
-`preserve_reasoning`). Env `scripts/frank-default-large.env`, serve
-`scripts/serve-frank-large.sh`.
-
-**Serving status (2026-09-01):** deployed the Flash-Next 56B MoE model. The
-previous kingjones Q6_0 AGENT model (27B, standalone MTP, 262144 context) is
-retired. The Q8_0 AGENT experiment is documented in
-`docs/LLM/kingjones-q8agent-rocmfpx.md`; it was retired because it crashed in
-`ggml_compute_forward_set_rows` and did not provide a working service.
-
-### Tiel-Coder-35B-A3B llama.cpp (Dankserver, ACTIVE)
-
-The small service serves `peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF` (public,
-30 GB, UD-Q6_K_XL quant) from `/home/dank/models/tiel-coder-35b/` with
-upstream llama.cpp (latest master, CUDA build with
-`-DGGML_CUDA_FA_ALL_QUANTS=ON -DCMAKE_CUDA_ARCHITECTURES=86`). The model is a
-qwen35moe hybrid MoE: 40 layers (10 attention + 30 SSM/Mamba,
-full_attention_interval=4), 256 experts / 8 used, 2 KV heads × 256 dim. It
-runs Q8_0 KV cache, 200700 total context (66900/slot × 3 parallel), `--cpu-moe`
-for expert offload, `-ngl 999` (non-expert on GPU), and mmap. Env
-`scripts/frank-default-small.env`, serve `scripts/serve-frank-small.sh`.
-
-**Serving status (2026-09-01):** deployed the Tiel-Coder-35B model. The
-previous ik_llama.cpp Ornith CRACK Q8_0 deployment is retired.
-
-### charlie12345/ROCmFPX ROCmI4 W4A4 llama.cpp (Dankclaw, prior v3, 2026-08-18 to 2026-08-25)
-
-The v4 runtime serves `cafonz/Qwen3.8-27B-ROCmI4-MTP-GGUF` (`eemin` namespace,
-public) from `/home/dank/rocmi4-qwen38/ROCmFPX/build-rocmi4-w4a4/bin/llama-server`
-(charlie12345/ROCmFPX commit `c49ebdbd` built with `-DGGML_HIP_ROCMI4_W4A4=ON`
-against system ROCm 7.2.4 / amdclang 22). The model is `Q4_0_ROCMI4` with an
-embedded NextN/MTP head (no separate draft file); the gfx1151 native IU4 WMMA
-instruction batch-verifies proposals. Profile: strict Qwen MTP
-(`--spec-mtp-strict-qwen`, n_max 16, p_min 0.60, backend sampling, all layers
-on ROCm0), `-t 16 -tb 32`, batch 512 / ubatch 256 (ubatch swappable via the
-ubatch bench on 8k context), f16 KV, full 262144 context / 1 slot, TEMPERATURE
-0.7 / TOP_P 0.95, reasoning on (deepseek, budget -1), `--no-mmap`. Two-file
-payload (~14.4 GiB) in `/home/dank/models/eemin-Qwen3.8-27B-ROCMI4-MTP/`. Full
-runbook:
-[docs/LLM/eemin-rocmi4-mtp.md](docs/LLM/eemin-rocmi4-mtp.md).
-
-
-### Kairic Edge llama.cpp (Dankclaw, RETIRED experiment 2026-08-22)
-
-The Kairic Edge llama-server serves `jcbtc/Qwen3.8-27B-IU4-Kairic-Edge`
-(pinned tag `kairic-edge-qwen38-27b-v1`). It was deployed and reverted to v3
-the same day (operator decision). The runtime routes qualified prompt and
-verification shapes through the RDNA 3.5 native IU4 matrix instruction.
-
-**Release gap — CK `config.h`:** the pinned CK commit ships only
-`config.h.in`. The release script and guide never generate
-`third_party/composable_kernel/include/ck/config.h`. Generate it via a CK
-CMake configure and copy it in before building (full runbook in
-[docs/LLM/kairic-edge.md](docs/LLM/kairic-edge.md)). The file is untracked
-in CK git. A `git clean` removes it. Local rebuilds are byte-identical
-(`d7625a0c...`).
-
-**Target greedy argmax fast path:** the pinned runner sets
-`LLAMA_TARGET_GREEDY_ARGMAX_FASTPATH=1`. The fast path rejects non-greedy
-requests (400) and the card's own chat sampling is non-greedy, so the
-production attempt ran it at 0. It still rejects reasoning budgets unless
-compatibility mode is used. Do not re-deploy Kairic without re-testing chat
-first.
-
-**Rollback target:** the v3 model files remain in
-`/home/dank/models/CIRU-Qwen3.8-27B-ActiveFPX-v3/` and the v3 binary in
-`/home/dank/ciru-qwen38/ROCmFPX/build-promptforge/bin/llama-server`.
-Restored to v3 on 2026-08-22 (commit `c60156f`).
-
-### DFlash2 llama.cpp (Dankclaw, retired 2026-08-21)
-
-A DFlash2 Vulkan deployment briefly replaced CIRU on 2026-08-21, then was
-reverted the same day for speed. Its serve script, env, and the build
-worktree are archived in `.AGENTTRASH/` and documented in
-[docs/LLM/dflash2-large-vulkan.md](docs/LLM/dflash2-large-vulkan.md).
-
-### FastMTP patched llama.cpp (Dankserver, retired 2026-08-18)
-
-Historical: the Qwen3.8-27B IQ4_XS FastMTP service was retired on 2026-08-18
-(files in `.AGENTTRASH/`). The build notes are kept for reference. The
-HauhauCS FastMTP draft sidecar required a llama.cpp build from commit
-`4df29be4f4c3673f428170fda944a5b19f743bb8` plus the
-`HauhauCS-FastMTP-llama.cpp.patch` (from the HF repo). The build lives at
-`/home/dank/llama.cpp-fastmtp/build` on Dankserver. Loading the draft with a
-stock binary fails with `expected 5120, 248320, got 5120, 32768`. Do not
-rebuild from another commit without re-applying the patch.
-
-**REQUIRED build flags:** `-DGGML_CUDA_FA=ON -DGGML_CUDA_FA_ALL_QUANTS=ON`.
-Without `GGML_CUDA_FA_ALL_QUANTS` the q8_0 KV cache + flash attention falls
-back to a slow path and prefill collapses ~12x (measured 2026-08-17: 95 t/s
-vs 1300+ t/s at an 8K prompt on the 3090). See the HauhauCS HF repo README
-for the base recipe.
-
-### Env var naming rule — HIP runtime collision
-
-The ROCm HIP runtime reads the bare environment variables `TEMP`, `HOST`,
-and `GPU_MAX_HEAP_SIZE`. The serve scripts export the env file with
-`set -a`. A tunable named `TEMP` leaks into HIP. On 2026-08-13 `TEMP=1.0`
-made HIP use the path `1.0` for its JIT kernel cache, then segfaulted every
-model load. Commit `227ce01` fixed it by renaming the sampling variable to
-`TEMPERATURE`.
-
-Never name a tunable `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`. Check for
-collisions with this command:
-
-```
-strings /opt/rocm-*/lib/libamdhip64.so.* | grep -xE "TEMP|HOST|GPU_MAX_HEAP_SIZE"
-```
-
-### Crash-loop diagnostic procedure
-
-When a llama service segfaults during model load, follow these steps.
-
-```mermaid
-graph TD
-    A[Service SEGVs at load] --> B[Read journal + kernel segfault line]
-    B --> C{Segfault in libamdhip64?}
-    C -- yes --> D[Bisect script vs raw binary]
-    C -- no --> E[Check config, memory, model files]
-    D --> F[Run gdb for backtrace]
-    F --> G[Find root cause]
-```
-
-1. Read the service journal with `journalctl -u <service> -n 100`. Then read
-   the kernel segfault line with `journalctl -k`. A segfault at address 8 in
-   `libamdhip64` means a NULL unref in the HIP runtime.
-2. Run the serve script directly. Then run the raw binary with the same
-   flags. A different outcome isolates the env file as the cause.
-3. Run gdb for the backtrace with this command:
-   ```
-   gdb -batch -ex run -ex "bt 25" --args bash scripts/serve-frank-large.sh
-   ```
-   `llama-server` buffers its timed stdout. On SIGSEGV the log tail is lost.
-   Trust gdb, not the log.
-4. Kernel spam about `amdgpu: SVM mapping failed` is a symptom, not the root
-   cause. Investigate other causes before assuming memory pressure.
-
-### Operational quirks
-
-- Both hosts allow passwordless sudo for the deploy user (`(ALL) NOPASSWD:
-  ALL`, verified 2026-08-22). Use plain `sudo` in committed scripts. Do not
-  use `echo password | sudo -S` in committed scripts. The `FRANK_SUDO_PW`
-  env var covers headless runs on a host without that.
-- Never run `pkill -f` with a pattern that appears in your own command line.
-  It kills your ssh session. Use `pkill -x <exact-name>`.
-- Model load is slow: Dankserver loads the 30.6 GB Q6_K_P through RAM
-  (`--no-mmap`, MoE experts on CPU) in a few minutes cold. Dankclaw loads the
-  ~30 GiB Q8_0 AGENT payload (main + MTP draft + mmproj) a few minutes cold
-  with a fresh KV cache; ~18 s with a warm page cache. Health returns 503 while loading. Do not kill the
-  process. `deploy.sh` waits up to 20 minutes on both targets (see
-  `LLM_WAIT`).
-- A client abort and a server release look different in the journal. A
-  client abort logs `W srv stop: cancel task, id_task = N`. A natural stop
-  (EOS or limit) logs only `slot release` with `truncated = 0`. The server
-  never preempts a running generation. When all slots are busy, new tasks
-  defer into a queue. See the 2026-08-13 incident in
-  [docs/LLM/deployment.md](docs/LLM/deployment.md).
-- ComfyUI on Dankclaw runs CPU-only (`--cpu`). ROCm wheels do not support
-  gfx1151. Do not use CUDA images.
-- SearXNG outbound traffic must stay on the Gluetun tunnel. Do not move
-  its ports off the gluetun service.
-- Retired hardware history: the Dankserver 35B cuBLAS crash and the DFlash
-  premature-EOS bug are historical. See
-  [docs/LLM/dankserver-vram-growth.md](docs/LLM/dankserver-vram-growth.md)
-  and [docs/LLM/qwen3.8-spec-matrix.md](docs/LLM/qwen3.8-spec-matrix.md).
-  The FastFlowLM NPU endpoint is retired. See
-  [docs/LLM/FastFlowLM NPU Inference.md](docs/LLM/FastFlowLM%20NPU%20Inference.md).
-
-### Dankclaw memory model
-
-The GPU budget on Dankclaw is the rocm-smi figure: 64 GiB VRAM reported by
-rocm-smi (SVM-backed). The docs' older 96 GiB figure was the addressable
-SVM pool. The GPU node reports `local_mem_size 0`, so all allocations are
-GTT/SVM-backed. Trust rocm-smi and the runtime. Do not use `free` or
-`dmidecode` for the GPU budget. They show the CPU-side carve-out. This is
-intentional. Do not change it.
-
-## Why
-
-Direct edits on the remote drift from version control. Future agents will not
-know what changed. Git is the single source of truth. The two workloads share
-two hosts, one Caddy, and one Gitea. One repo keeps them in one history.
-
-> User request → Fetch from remote → Commit here → Edit → Test → Commit → Push → Pull on remote → Dispatch changed service(s) → Health
-
-## Summary
-
-| Context | Action |
-|---------|--------|
-| Editing a compose/Caddyfile/serve script | **Work through git** — fetch, commit, edit, test, push, pull, deploy. Never edit on the remote directly. |
-| Config exists only on remote | **Pull it into the repo first** — commit the fetched version (redacted) before making changes. |
-| Unstaged local changes | **Commit or discard** — check `git status` before starting work. Ask the user if unsure. |
-| Deploying a committed change | **Run `scripts/deploy.sh <target>`** — it pushes, pulls with fast-forward-only, plans changed paths, dispatches selected services, and checks their health. See [docs/LLM/deployment.md](docs/LLM/deployment.md). | | Container without a Quadlet unit | **Give it one** — self-healing, boot-persistent `.container` unit via git (see Boot Persistence & Self-Healing). |
-| Benchmarking | **Stop the service first**, restart, verify health. See the Benchmarking Policy above. |
-| Deleting a file | **Move it to trash** — `.AGENTTRASH/` (in-repo) or `~/.AGENTTRASH/YYYYMMDD-topic`. Never `rm`. |
-| Tests exist | **Run them** — verify tests pass after every change. Do not commit failing tests. |
-| Real secrets | **Never commit** — use host `.env` files. `.env.example` holds placeholders only. `FRANK_SUDO_PW` covers headless sudo. |
+### Commit Guidelines
+- Use descriptive commit messages: `git commit -m "Add AWQ DFlash script"`
+- Commit before deploying: `git add -A && git commit -m "Update for deployment"`
+- Push to remote: `git push origin <branch>`
+
+### Deployment Flow
+1. Make changes locally
+2. Run tests: `bash tests/test_validator.sh dankserver`
+3. Commit: `git add -A && git commit -m "Description"`
+4. Deploy: `bash deploy.sh dankserver`
+5. Verify: `bash scripts/validate-vllm.sh -t dankserver`
+
+## Remote Servers
+
+### dankserver
+- SSH: `dank@dankserver`
+- Scripts: `~/vllm/`
+- Service: `vllm-awq-qwen36.service`
+- Model: frank-default-small (AWQ 4bit)
+- Port: 34401
+- Status: Disabled (pending bug fixes)
+
+### dankclaw
+- SSH: `dank@dankclaw`
+- Scripts: `~/vllm/`
+- Service: `vllm-thinkingcap.service`
+- Model: frank-default-large (FP8)
+- Port: 3090
+- Status: Running (you are the deployed inference)
+
+## Troubleshooting
+- Service failed: `systemctl status <service>`
+- Container stuck: `docker stop <name> && docker rm <name>`
+- OOM error: Reduce `--max-model-len` or increase `--cpu-offload-gb`
+- Prefix caching conflict: Remove `--enable-prefix-caching` from script

README.md

@@ -1,63 +1,105 @@
-# Server App Configs
+# vLLM Scripts
 
-The single repository for the two homelab servers: the Podman container
-stacks **and** the LLM inference fleet. One repo, one checkout per host,
-one deploy script.
-
-> 2026-08-22: the former `vllmScripts` repository (LLM fleet) is merged
-> into this repo with its full git history. See AGENTS.md for the merge
-> note.
+Scripts for running large language models via vLLM on two local machines.
 
 ## Targets
 
-| Host | User | IP | Podman stack | LLM service | LLM port |
-|------|------|----|--------------|-------------|----------|
-| dankserver | dank | 192.168.0.235 | media and library | frank-default-small (llama.cpp, Qwen3.6-35B-A3B) | 7770 |
-| dankclaw | dank | 192.168.0.143 | AI, search, development, observability | frank-default-large (CIRU ROCmFPX, Qwen3.8-27B) | 7770 |
+| Host | User | Codename | Architecture | Service Status |
+|------|------|----------|-------------|----------------|
+| dankserver | dank | DankServer | NVIDIA GPU + System RAM | `vllm-awq-qwen36.service` (failed) |
+| dankclaw | dank | DankClaw | AMD Strix Halo (Shared Memory) | `vllm-thinkingcap.service` (failed) |
 
-## Active Services
+## File Locations
 
-| Service | Host | Hostname | Port |
-|---------|------|----------|------|
-| RomM | dankserver | romm.dankserver.local | 8085 |
-| Copyparty | dankserver | files.dankserver.local | 3923 |
-| Calibre-Web | dankserver | books.dankserver.local | 8083 |
-| frank-default-small | dankserver | — (bare endpoint) | 7770 |
-| SearXNG | dankclaw | search.dankclaw.local | 8081 |
-| ComfyUI | dankclaw | comfyui.dankclaw.local | 8188 |
-| OpenWebUI | dankclaw | chat.dankclaw.local | 3000 |
-| Hermes | dankclaw | hermes.dankclaw.local | 8090 |
-| Gitea | dankclaw | gitea.dankclaw.local | 3001 |
-| Grafana | dankclaw | grafana.dankclaw.local | 3002 |
-| Prometheus | dankclaw | prometheus.dankclaw.local | 9090 |
-| frank-default-large | dankclaw | — (bare endpoint) | 7770 |
+### Local (vllmScripts/)
+- `scripts/` — Active inference scripts
+- `tests/` — Test suite
+- `docs/` — Knowledge base
+- `failedAttempts/` — Archived scripts
+
+### Remote (dankserver)
+- `~/vllm/` — Deployed inference scripts
+- `~/vllm/failedAttempts/` — Archived scripts
+- `~/vLLM/` — Legacy project directory (vllm-qwen36.sh)
+
+### Remote (dankclaw)
+- `~/vllm/` — vLLM source code (not inference scripts)
+- `~/vLLM/` — Inference scripts (if deployed)
+
+## Models
+
+| Model | Target | Format |
+|-------|--------|--------|
+| Qwen3.6 35B A3B FP8 | dankserver | FP8 |
+| Qwen3.6 27B FP8 | dankclaw | FP8 |
+| Qwen3.6 35B A3B AWQ 4bit | dankserver | AWQ 4bit |
+| Qwen3.6 35B A3B DFlash | dankserver | Draft model |
 
 ## Structure
 
-- `dankserver/` — compose.yaml + Caddyfile + .env.example for the Dankserver stack
-- `dankclaw/` — compose.yaml + Caddyfile + .env.example for the Dankclaw stack
-- `shared/` — homepage, prometheus config, deploy helpers
-- `scripts/` — `deploy.sh` and `deploy-remote.sh` (service-scoped deployment), the planner, LLM serve scripts, systemd units, env tunables, and bench harnesses
-- `old/` — retired PP/NPU-era LLM scripts (reference only)
-- `docs/` — project knowledge base (Obsidian vault, wikilinks). `docs/LLM/` is the LLM fleet
-- `tests/` — bats test suite (see [[Testing Policy]])
-
-## Deployment
-
-See [docs/Architecture/Deployment Flow.md](docs/Architecture/Deployment%20Flow.md)
-and [docs/LLM/deployment.md](docs/LLM/deployment.md). Run
-`./scripts/deploy.sh <dankclaw|dankserver>` after committing changes. It
-plans and deploys only the changed services. Use `--service NAME` for an
-explicit single-service retry; inference is restarted only when its own files
-(or the shared chat template) changed.
+- `docs/` — Project knowledge base with research notes on vLLM configuration, hardware specifics, and deployment notes
+- `scripts/` — Executable vLLM launch scripts
+- `vllm-qwen36.sh` — FP8 inference script for dankserver
+- `vllm-awq-qwen36-dflash.sh` — AWQ 4bit with DFlash speculative decoding (docker)
+- `vllm-awq-qwen36-dflash-ampere.sh` — AWQ 4bit with DFlash speculative decoding (Ampere-optimized, systemd)
 
 ## Knowledge Base
 
-The `docs/` directory is an Obsidian vault. Start at
-[docs/index.md](docs/index.md). The LLM fleet knowledge base lives at
-[docs/LLM/index.md](docs/LLM/index.md).
+See [[docs/README.md]] for the project knowledge base index.
 
-## Testing
+## Lessons Learned
 
-Run `./tests/run.sh` before every push. See
-[docs/Architecture/Testing Policy.md](docs/Architecture/Testing%20Policy.md).
+### AWQ + DFlash on Ampere Hardware (2026-08-08)
+
+**What worked:**
+- `cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit` loads successfully on Ampere GPUs
+- AWQ Marlin kernel engaged automatically for weight conversion
+- DFlash draft model `z-lab/Qwen3.6-35B-A3B-DFlash` detected as EAGLE model
+- torch.compile with cudagraphs works with `--max-num-batched-tokens 1536`
+
+**What didn't work:**
+- `--enable-prefix-caching` conflicts with Mamba cache "align" mode — causes `AssertionError` in `unify_kv_cache_spec_page_size`
+- `--max-model-len 262144` causes OOM during AWQ→Marlin weight conversion on 24GB GPU
+- Nightly vLLM `--speculative-config` requires `num_speculative_tokens` when model is provided
+- `--speculative-model` flag removed in nightly; use `--speculative-config` JSON instead
+
+**Working configuration:**
+- `--max-model-len 32768` (32k context)
+- `--cpu-offload-gb 40` with `--cpu-offload-params "experts"`
+- `--gpu-memory-utilization 0.95`
+- `--performance-mode throughput`
+- `--enable-chunked-prefill`
+- `--max-num-batched-tokens 1536`
+- Thread binding: `OMP_NUM_THREADS=8`, `VLLM_CPU_OMP_THREADS_BIND=8`
+
+**Performance observations (no speculative decoding):**
+| Streams | Prompt Speed | Completion Speed | Total Time |
+|---------|-------------|-----------------|------------|
+| 1 | 4.27 tok/s | 21.42 tok/s | 11,951ms |
+| 2 | 2.44 tok/s | 12.25 tok/s | 20,891ms |
+| 3 | 1.35 tok/s | 6.76 tok/s | 37,849ms |
+| 4 | 1.43 tok/s | 7.19 tok/s | 35,606ms |
+
+Note: Speeds decrease with more streams due to KV cache contention.
+
+**Still needs work:**
+- DFlash speculative decoding integration (KV cache assertion bug in nightly)
+- Full context length support (32k is temporary workaround for OOM)
+
+## Validation
+
+Use `scripts/validate-vllm.sh` to test inference servers:
+
+```bash
+# Test dankserver (default)
+bash scripts/validate-vllm.sh -t dankserver
+
+# Test dankclaw
+bash scripts/validate-vllm.sh -t dankclaw
+
+# Custom host/port
+bash scripts/validate-vllm.sh -H 192.168.1.100 -P 8001 -M my-model
+
+# Run tests
+bash tests/test_validator.sh dankserver
+```

dankclaw/.env.example

@@ -1,61 +0,0 @@
-# Dankclaw stack secrets and configuration.
-# Copy to .env and fill real values. Never commit real secrets.
-
-# SearXNG
-SEARXNG_SECRET=changeme
-SEARXNG_BASE_URL=http://search.franksdank.cloud/
-
-# Gluetun VPN (see https://github.com/qdm12/gluetun-wiki)
-# Single-peer WireGuard model. Fill from your provider's WireGuard config.
-VPN_SERVICE_PROVIDER=protonvpn
-VPN_TYPE=wireguard
-WIREGUARD_PRIVATE_KEY=changeme
-WIREGUARD_ADDRESSES=0.0.0.0/32
-WIREGUARD_PUBLIC_KEY=changeme
-WIREGUARD_ENDPOINT_IP=0.0.0.0
-WIREGUARD_ENDPOINT_PORT=51820
-WIREGUARD_ALLOWED_IPS=0.0.0.0/0,::/0
-WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL=25s
-
-# Gitea database
-GITEA_DB_PASSWORD=changeme
-
-# Grafana admin
-GRAFANA_ADMIN_USER=admin
-GRAFANA_ADMIN_PASSWORD=changeme
-
-# Caddy (the homelab's single reverse proxy, on this host)
-# bcrypt hash (caddy hash-password). deploy.sh renders it into
-# /home/dank/Containers/caddy/Caddyfile.
-# CADDY_BASICAUTH_HASH: the shared family password, still used by the
-#   vhosts that carry basicauth rather than Authentik forward_auth
-#   (grafana, pihole). Username: dankuser.
-CADDY_BASICAUTH_HASH=changeme
-# (The homepage's per-user hashes were retired: the homepage and its /admin/
-#  page are now behind Authentik forward_auth, not basicauth.)
-
-# Hermes -> Signal (signal-cli daemon on the host, port 8080)
-# E.164 number of the linked Signal account (your primary number, for Note-to-Self).
-SIGNAL_ACCOUNT=+15551234567
-# Comma-separated E.164 numbers allowed to message the agent.
-SIGNAL_ALLOWED_USERS=+15551234567
-
-# Hermes web UI (Hermex server) login password. Required: the webui binds to
-# loopback and is reached only through the SSH tunnel, but Hermex still signs
-# in with this password. Set a strong value in the live .env.
-HERMES_WEBUI_PASSWORD=changeme
-
-# Porkbun API (franksdank.cloud + franksdank.casa zones). Caddy uses these
-# for DNS-01 Let's Encrypt wildcard cert issuance. Porkbun API keys are
-# account-wide; rotate them here rather than in the dashboard.
-# PORKBUN_API_SECRET_KEY is the name the caddy-dns/porkbun module expects.
-PORKBUN_API_KEY=changeme
-PORKBUN_API_SECRET_KEY=changeme
-
-# Authentik (SSO / OIDC provider)
-# AUTHENTIK_PG_PASS: the PostgreSQL password for authentik-postgres.
-#   Generate: openssl rand -base64 32
-AUTHENTIK_PG_PASS=changeme
-# AUTHENTIK_SECRET_KEY: the server's secret key (64+ chars).
-#   Generate: openssl rand -base64 48
-AUTHENTIK_SECRET_KEY=changeme

dankclaw/Caddyfile

@@ -1,286 +0,0 @@
-# Homelab Caddyfile — the single Caddy for both hosts.
-#
-# Caddy runs as a container on dankclaw (192.168.0.143, this repo's
-# compose.yaml service 'caddy') and fronts every service on both hosts:
-#   - dankclaw services: by container name (compose network) or by the
-#     published LAN port (quadlet services on isolated networks)
-#   - dankserver services (192.168.0.235): by published LAN port
-#
-# Naming: flat franksdank.cloud scheme (see docs/Infrastructure/LAN-DNS.md).
-# DNS for these names is served by the Pi-hole quadlet on this host
-# (dankclaw/pihole/addn-hosts — every name resolves to 192.168.0.143)
-# and handed to LAN clients by the SURFboard G36 router.
-#
-# Authentication: Caddy forward_auth (Authentik SSO) protects the vhosts that
-# do not have their own application-level auth (homepage, SearXNG, Hermes,
-# Prometheus, RomM, Copyparty). The basicauth hash is a placeholder; deploy.sh
-# renders the real hash (CADDY_BASICAUTH_HASH from the host .env, git-ignored)
-# into /home/dank/Containers/caddy/Caddyfile, which the container mounts.
-# Apps with their own auth (Grafana, Gitea, OpenWebUI, Jellyfin, *arr, Pi-hole)
-# are not wrapped in forward_auth — they handle their own login.
-#
-# TLS: real Let's Encrypt certs. Site addresses carry no http:// prefix,
-# so Caddy serves HTTPS (and auto-redirects HTTP). Certs are issued via
-# dns-01 through the Porkbun module built into the franksdank/caddy-porkbun
-# image (caddy/Containerfile). That needs no inbound ports and works from the
-# LAN, where Pi-hole resolves these names to 192.168.0.143.
-
-{
-	# Admin stays ON: it binds to 127.0.0.1:2019 inside the container and the
-	# port is not published, so nothing outside the container can reach it.
-	# It powers the config file watcher, so a rendered Caddyfile change
-	# hot-reloads without a container restart. 'admin off' once left a broken
-	# basicauth hash sitting on disk unapplied for hours (2026-08-22).
-	email admin@franksdank.cloud
-
-	# Solve ACME challenges for every site via Porkbun DNS (no port 80/443
-	# reachability needed). Credentials come from the container environment
-	# (compose.yaml, from the host .env — never committed).
-	acme_dns porkbun {
-		api_key {env.PORKBUN_API_KEY}
-		api_secret_key {env.PORKBUN_API_SECRET_KEY}
-	}
-}
-
-# --- Bare-IP access (health checks, convenience) ---
-:80 {
-	redir https://franksdank.cloud/ 302
-}
-
-# --- Homepage (Authentik SSO) ---
-franksdank.cloud www.franksdank.cloud {
-	root * /srv/homepage
-
-	route {
-		# Authentik outpost (login UI + session management).
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-
-		# Forward auth: check the Authentik session before serving the page.
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-
-		# Operator-only admin page. Auth is checked on the original path,
-		# before any rewrite, so the path matcher is reliable. 'handle'
-		# preserves order.
-		handle /admin* {
-			try_files /admin.html
-			file_server
-		}
-
-		# Shared data + assets: fetched by both the user page and the admin
-		# page. With forward_auth, any authenticated user can access them
-		# (the browser's Authentik session covers these fetches).
-		@data path /services.json /status.json /app.js /style.css /deployed.js
-		handle @data {
-			file_server
-		}
-
-		# Everything else (the user page).
-		handle {
-			try_files /index.html
-			file_server
-		}
-	}
-}
-
-# --- Legacy name: the old homepage name redirects to the naked domain ---
-dankclaw.franksdank.cloud {
-	redir https://franksdank.cloud/ 302
-}
-
-# --- Legacy name: the media host has no Caddy of its own anymore ---
-dankserver.franksdank.cloud {
-	redir https://franksdank.cloud/ 302
-}
-
-# --- Authentik (SSO / OIDC provider) ---
-auth.franksdank.cloud {
-	reverse_proxy authentik-server:9000
-}
-
-# ═══════════════════════════════════════════════════════════════
-# Services on this host (dankclaw, 192.168.0.143)
-# ═══════════════════════════════════════════════════════════════
-
-# --- SearXNG (quadlet; behind the Gluetun tunnel, published 8081) ---
-search.franksdank.cloud {
-	route {
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-		reverse_proxy 192.168.0.143:8081
-	}
-}
-
-# --- ComfyUI ---
-comfyui.franksdank.cloud {
-	reverse_proxy comfyui:8188
-}
-
-# --- OpenWebUI (quadlet on slirp4netns; published 3005) ---
-chat.franksdank.cloud {
-	reverse_proxy 192.168.0.143:3005
-}
-
-# --- Hermes ---
-hermes.franksdank.cloud {
-	route {
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-		reverse_proxy hermes:8090
-	}
-}
-
-# --- Gitea (quadlet; published 3000) ---
-gitea.franksdank.cloud {
-	reverse_proxy 192.168.0.143:3000
-}
-
-# --- Grafana ---
-grafana.franksdank.cloud {
-	basicauth {
-		dankuser REPLACE_WITH_HASH
-	}
-	reverse_proxy grafana:3000
-}
-
-# --- Prometheus ---
-prometheus.franksdank.cloud {
-	route {
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-		reverse_proxy prometheus:9090
-	}
-}
-
-# --- Loki ---
-loki.franksdank.cloud {
-	reverse_proxy loki:3100
-}
-
-# --- Pi-hole (quadlet on slirp4netns; published 8083) ---
-pihole.franksdank.cloud {
-	basicauth {
-		dankuser REPLACE_WITH_HASH
-	}
-	reverse_proxy 192.168.0.143:8083
-}
-
-# ═══════════════════════════════════════════════════════════════
-# Services on dankserver (192.168.0.235) — published LAN ports
-# ═══════════════════════════════════════════════════════════════
-
-# --- Plex (existing, kept) ---
-# http:// on purpose: franksdank.casa is not hosted in the same Porkbun
-# account as franksdank.cloud, so the DNS-01 provider cannot issue a
-# Let's Encrypt cert for it. It stays plain-HTTP legacy; the canonical
-# name is plex.franksdank.cloud (HTTPS, below).
-http://plex.franksdank.casa {
-	reverse_proxy 192.168.0.235:32400 {
-		transport http {
-			tls_insecure_skip_verify
-		}
-	}
-}
-
-# PMS requires TLS from every non-loopback client: plain-HTTP off-loopback
-# is read and silently closed (diagnosed 2026-08-26), which surfaced as an
-# empty-body 502 here. Dial the upstream over TLS; PMS's self-signed cert is
-# skipped on this pinned-LAN hop only (client-facing LE TLS is untouched).
-# The web UI lives at /web/; / returns the XML MediaContainer index.
-plex.franksdank.cloud {
-	reverse_proxy 192.168.0.235:32400 {
-		transport http {
-			tls_insecure_skip_verify
-		}
-	}
-}
-
-# --- Calibre-Web ---
-books.franksdank.cloud {
-	reverse_proxy 192.168.0.235:8083
-}
-
-# --- RomM ---
-romm.franksdank.cloud {
-	route {
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-		reverse_proxy 192.168.0.235:8085
-	}
-}
-
-# --- Copyparty ---
-files.franksdank.cloud {
-	route {
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-		reverse_proxy 192.168.0.235:3923
-	}
-}
-
-# --- Jellyfin ---
-jellyfin.franksdank.cloud {
-	reverse_proxy 192.168.0.235:8096 {
-		transport http {
-			versions h2c 1.1
-		}
-	}
-}
-
-# --- *arr Stack ---
-radarr.franksdank.cloud {
-	reverse_proxy 192.168.0.235:7878
-}
-
-sonarr.franksdank.cloud {
-	reverse_proxy 192.168.0.235:8989
-}
-
-lidarr.franksdank.cloud {
-	reverse_proxy 192.168.0.235:8686
-}
-
-readarr.franksdank.cloud {
-	reverse_proxy 192.168.0.235:8787
-}
-
-prowlarr.franksdank.cloud {
-	reverse_proxy 192.168.0.235:9696
-}
-
-qbittorrent.franksdank.cloud {
-	reverse_proxy 192.168.0.235:8080
-}
-
-# --- Optional: Jackett ---
-# jackett.franksdank.cloud {
-# 	reverse_proxy 192.168.0.235:9117
-# }
-
-# --- Optional: Whisparr ---
-# whisparr.franksdank.cloud {
-# 	reverse_proxy 192.168.0.235:6969
-# }

dankclaw/Containerfile.comfyui

@@ -1,24 +0,0 @@
-# ComfyUI on the gfx1151 (AMD Strix Halo / Radeon 8060S) ROCm torch build.
-#
-# Base image: AMD's gfx1151-tagged vLLM image. It is the only verified build
-# where torch.cuda.is_available() is True on this APU (device capability
-# (11,5) = gfx1151). It already ships the matching ROCm torch family
-# (torch 2.9.1+rocm, torchvision 0.24.0+rocm, torchaudio 2.9.0+rocm), so we
-# must NOT let pip reinstall them: ComfyUI's requirements.txt lists an unpinned
-# `torch`, which would otherwise clobber the gfx1151 kernels with a default build.
-FROM docker.io/rocm/vllm:rocm7.12.0_gfx1151_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0
-
-# Bake the ComfyUI code into the image (rebuild the image to update the code).
-RUN git clone --depth 1 https://github.com/comfyanonymous/ComfyUI /opt/ComfyUI
-WORKDIR /opt/ComfyUI
-
-# Pre-create the user-data dirs. The host mounts its collections over them
-# (see the comfyui service in compose.yaml).
-RUN mkdir -p /opt/ComfyUI/models /opt/ComfyUI/input /opt/ComfyUI/output /opt/ComfyUI/custom_nodes
-
-# Install ComfyUI's dependencies, but skip the torch family (already present
-# above as the gfx1151 ROCm builds). torchsde and the rest are kept.
-RUN grep -viE '^(torch|torchvision|torchaudio)([^a-z]|$)' requirements.txt > /tmp/req-notorch.txt \
-    && pip install --no-cache-dir -r /tmp/req-notorch.txt
-
-EXPOSE 8188

dankclaw/alloy/Containerfile

@@ -1,18 +0,0 @@
-# Alloy + a static busybox.
-#
-# grafana/alloy has a shell but no wget/curl, so the compose healthcheck
-# cannot run anything inside the container. This image drops a static
-# busybox into it for the healthcheck command only. All Alloy behavior is
-# unchanged.
-#
-# Build on dankclaw:
-#   podman build -t franksdank/alloy:latest -f dankclaw/alloy/Containerfile .
-
-FROM docker.io/library/alpine:latest AS tools
-# busybox-static is statically linked: the target images have no dynamic
-# loader (loki is distroless), so the alpine default /bin/busybox (musl
-# dynamic) would fail with "no such file or directory".
-RUN apk add --no-cache busybox-static
-
-FROM docker.io/grafana/alloy:latest
-COPY --from=tools /bin/busybox.static /usr/local/bin/busybox

dankclaw/caddy/Containerfile

@@ -1,22 +0,0 @@
-# Caddy with the Porkbun DNS provider (github.com/caddy-dns/porkbun).
-#
-# The porkbun module lets Caddy solve Let's Encrypt DNS-01 challenges
-# directly through the Porkbun API, so the wildcard cert for
-# *.franksdank.cloud / *.franksdank.casa can be issued and renewed without
-# any port 80/443 reachability from the internet (see the Caddyfile in
-# ../Caddyfile and docs/Infrastructure/LAN-DNS.md).
-#
-# Build on dankclaw (podman rootless, same as every other container here):
-#   podman build -t franksdank/caddy-porkbun:latest -f dankclaw/caddy/Containerfile .
-#
-# The first build runs an xcaddy Go compilation and takes a few minutes.
-# Rebuild only when Caddy or the porkbun module must be updated.
-
-FROM docker.io/library/caddy:2-builder AS builder
-
-RUN xcaddy build \
-    --with github.com/caddy-dns/porkbun \
-    --output /usr/bin/caddy-porkbun
-
-FROM docker.io/library/caddy:2
-COPY --from=builder /usr/bin/caddy-porkbun /usr/bin/caddy

dankclaw/caddy/env.example

@@ -1,7 +0,0 @@
-# Caddy environment (placeholders only). The deploy writes the live file at
-# /home/dank/Containers/caddy/env from the checkout's .env; the quadlet reads
-# it via EnvironmentFile. Consumed by the Caddyfile's acme_dns block
-# (DNS-01 wildcard cert issuance). PORKBUN_API_SECRET_KEY is the name the
-# caddy-dns/porkbun module expects.
-PORKBUN_API_KEY=changeme
-PORKBUN_API_SECRET_KEY=changeme

dankclaw/caddy/quadlet/caddy.container

@@ -1,54 +0,0 @@
-# Quadlet unit: canonical Caddy reverse proxy + ACME (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/caddy.container
-# Unit name: caddy.service (container name: caddy)
-#
-# The homelab's single Caddy: it fronts every service on both hosts and issues
-# Let's Encrypt wildcard certs via DNS-01 (Porkbun). It left compose.yaml on
-# 2026-08-26 after a reboot exposed that the bare compose container never came
-# back: rootless Podman has no boot-time auto-start for bare containers, and
-# `restart: unless-stopped` does not survive the explicit stop that shutdown
-# performs. See AGENTS.md "Boot Persistence & Self-Healing" and
-# docs/Infrastructure/Caddy.md.
-#
-# It joins the existing compose network (dankclaw_default) so the Caddyfile's
-# container-name upstreams (authentik-server, grafana, ...) still resolve; if
-# no compose service has created that network yet at boot, quadlet creates it
-# first and compose reuses it by name later. The image is built locally from
-# dankclaw/caddy/Containerfile and exists only on this host, so Pull=never: a
-# boot must not stall on a registry that does not have it. Porkbun credentials
-# come from /home/dank/Containers/caddy/env (deploy writes it from the
-# checkout's .env; secrets never enter the repo).
-
-[Unit]
-Description=Caddy reverse proxy + ACME (Porkbun DNS-01) container
-After=network-online.target
-Wants=network-online.target
-
-[Container]
-ContainerName=caddy
-Image=franksdank/caddy-porkbun:latest
-Pull=never
-Network=dankclaw_default
-PublishPort=0.0.0.0:80:80/tcp
-PublishPort=0.0.0.0:443:443/tcp
-EnvironmentFile=/home/dank/Containers/caddy/env
-Volume=/home/dank/Containers/caddy/Caddyfile:/etc/caddy/Caddyfile:ro
-Volume=/home/dank/Containers/caddy/data:/data
-Volume=/home/dank/Containers/caddy/config:/config
-# The homepage: the host's /srv/homepage (deploy-managed, not the repo's
-# shared/homepage).
-Volume=/srv/homepage:/srv/homepage:ro
-# The bare-IP catch-all answers 302 (curl -f passes on <400); this proves
-# Caddy itself is serving without touching any upstream.
-HealthCmd=curl -sf -o /dev/null http://localhost:80/
-HealthInterval=30s
-HealthTimeout=10s
-HealthRetries=3
-
-[Service]
-Restart=always
-RestartSec=5
-
-[Install]
-WantedBy=default.target

dankclaw/comfyui/Containerfile

@@ -1,55 +0,0 @@
-# ComfyUI on ROCm for the AMD Strix Halo APU (gfx1151).
-#
-# The official `rocm/comfyui` and pytorch.org ROCm wheels compile kernels for
-# datacenter archs (gfx942/gfx950) and omit gfx1151. This base — AMD's
-# gfx1151-tagged PyTorch build (torch 2.9.1+rocm7.12.0rc1) — is the verified
-# working GPU path on Dankclaw (2026-08-14: avail=True, Radeon 8060S, cap (11,5)).
-#
-# The base is heavier than a dedicated ComfyUI image (it carries vLLM), but it
-# is the only prebuilt image that ships gfx1151 torch kernels. See
-# docs/Research/ComfyUI ROCm and Vulkan.md.
-
-FROM docker.io/rocm/vllm:rocm7.12.0_gfx1151_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0
-
-# ComfyUI source + runtime requirements (non-GPU deps install from PyPI).
-ENV PYTHONUNBUFFERED=1 \
-    COMFYUI_DIR=/opt/ComfyUI
-
-WORKDIR /opt
-
-# git ships in the base; clone the stable ComfyUI tag.
-RUN git clone --depth 1 --branch v0.3.6 \
-        https://github.com/Comfy-Org/ComfyUI.git "$COMFYUI_DIR"
-
-WORKDIR "$COMFYUI_DIR"
-
-# Install ComfyUI Python requirements. Do NOT reinstall torch/torchvision —
-# keep the gfx1151-enabled torch from the base. The requirements pin torch,
-# so install everything else from a deps-only pass.
-RUN /opt/python/bin/pip install --no-cache-dir \
-        -r requirements.txt \
-        --no-deps 2>/dev/null || true
-
-# Ensure the managed deps that are missing get installed without touching torch.
-# (First pass is best-effort; a reinstall of torch is deliberately avoided.)
-RUN /opt/python/bin/python -m pip install --no-cache-dir \
-        requests==2.32.3 \
-        aiohttp==3.10.10 \
-        yarl==1.13.1 \
-        pillow==10.4.0 \
-        numpy>=1.24 \
-        huggingface-hub>=0.24 \
-        einops \
-        comfyui-frontend-package>=1.0 \
-        model-management>=1.0.3 \
-        opaque>=0.0.10 \
-        pydantic>=2.6 \
-        websockets>=15 \
-        imageio
-
-# Run ComfyUI as the host user so bind mounts /opt/comfyui are writable.
-USER 1000:1000
-
-EXPOSE 8188
-
-ENTRYPOINT ["/opt/python/bin/python", "main.py", "--listen", "0.0.0.0", "--port", "8188"]

dankclaw/compose.yaml

@@ -1,317 +0,0 @@
-# Dankclaw stack — AI, search, development & observability services.
-# Deploy: cd /home/dank/serverAppConfigs && podman-compose -f dankclaw/compose.yaml up -d
-# See ../docs/Dankclaw/ and ../docs/Infrastructure/ for service documentation.
-
-services:
-  # --- SearXNG stack (gluetun + searxng + valkey) --------------------------
-  # These three services are now boot-persistent Quadlet units, not compose
-  # services. They live in searxng/quadlet/ and install to
-  # ~/.config/containers/systemd/ on the host. searxng reuses gluetun's netns
-  # (Network=gluetun.container) so its traffic exits the WireGuard tunnel.
-  # Deploy with: scripts/deploy-searxng-quadlet.sh dankclaw
-  # See ../docs/Dankclaw/SearXNG.md, Gluetun.md, Valkey.md.
-
-  # --- AI image generation (ROCm GPU backend, gfx1151) ---------------------
-  # Built from Containerfile.comfyui on AMD's gfx1151 torch base. The GPU is
-  # reached via the /dev/kfd + /dev/dri passthrough (checked host-side; the run
-  # user is already in the render group). Only the 'video' group is added:
-  # rootless podman cannot map the render gid into the container namespace.
-  # Models + data persist under /home/dank/comfyui-data. See ../docs/Dankclaw/ComfyUI.md.
-  comfyui:
-    build:
-      context: .
-      dockerfile: Containerfile.comfyui
-    image: comfyui-rocm:gfx1151
-    container_name: comfyui
-    restart: unless-stopped
-    command: ["python", "main.py", "--listen", "0.0.0.0", "--port", "8188"]
-    ports:
-      - "8188:8188"
-    devices:
-      - /dev/kfd:/dev/kfd
-      - /dev/dri:/dev/dri
-    group_add:
-      - video
-    volumes:
-      - /home/dank/comfyui-data/models:/opt/ComfyUI/models
-      - /home/dank/comfyui-data/input:/opt/ComfyUI/input
-      - /home/dank/comfyui-data/output:/opt/ComfyUI/output
-      - /home/dank/comfyui-data/custom_nodes:/opt/ComfyUI/custom_nodes
-    healthcheck:
-      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8188/')"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 180s
-
-  # --- Chat UI (open-webui) ------------------------------------------------
-  # Now a boot-persistent Quadlet unit (quadlet, slirp4netns, port 3005), not
-  # a compose service. See open-webui/quadlet/open-webui.container and
-  # ../docs/Dankclaw/OpenWebUI.md.
-
-  hermes:
-    image: docker.io/nousresearch/hermes-agent:latest
-    container_name: hermes
-    restart: unless-stopped
-    # The image's default s6 entrypoint runs the interactive TUI, which exits
-    # without a TTY and crash-loops the container. Override it to run the
-    # headless gateway (the documented Docker mode). The gateway drops to the
-    # unprivileged hermes user (uid 10000), so the data dir must be writable by it.
-    entrypoint: ["bash", "-c", "hermes gateway run --force -v"]
-    environment:
-      HERMES_DATA_DIR: /data
-      # Point hermes's config/.env at the mounted data dir (persistent).
-      HERMES_HOME: /data
-      # Signal adapter -> the host signal-cli daemon. The slirp host alias
-      # (host.containers.internal) can drop at runtime; the host LAN IP is the
-      # stable container->host path. 127.0.0.1 inside the container is not the host.
-      SIGNAL_HTTP_URL: "http://192.168.0.143:8080"
-      SIGNAL_ACCOUNT: "${SIGNAL_ACCOUNT}"
-      SIGNAL_ALLOWED_USERS: "${SIGNAL_ALLOWED_USERS}"
-    ports:
-      - "8090:8090"
-    volumes:
-      - /home/dank/hermes-data:/data
-    # No healthcheck: 'hermes gateway run' starts the messaging gateway (Signal
-    # SSE), not the :8090 web dashboard, so there is no HTTP endpoint to probe.
-    # The container's running state + the daemon's SSE connection are the signal.
-
-  # --- Hermes web dashboard (management UI: config, API keys, sessions) -----
-  # A separate process from the Signal gateway (the gateway entrypoint does not
-  # start the dashboard). Bound to the LAN with basic auth: the username + a
-  # scrypt password_hash live in the git-ignored /home/dank/hermes-data config.
-  # Port 9119 is published so a browser on the LAN can reach it.
-  hermes-dashboard:
-    image: docker.io/nousresearch/hermes-agent:latest
-    container_name: hermes-dashboard
-    restart: unless-stopped
-    entrypoint: ["bash", "-c", "hermes dashboard --host 0.0.0.0 --port 9119 --no-open --skip-build"]
-    environment:
-      HERMES_DATA_DIR: /data
-      HERMES_HOME: /data
-    ports:
-      - "9119:9119"
-    volumes:
-      - /home/dank/hermes-data:/data
-
-  # --- Hermes web UI (Hermex server; phone-facing API for the agent) --------
-  # Runs the open-source hermes-webui (github.com/nesquena/hermes-webui) in
-  # foreground mode: bootstrap.py creates .venv inside the mounted checkout,
-  # installs WebUI + agent deps, then execs server.py as a single foreground
-  # process. It reads HERMES_HOME=/data, so it drives the SAME agent state
-  # (memory, skills, sessions) as the gateway container.
-  # Loopback-only publish: the SSH tunnel (phone -> Dankclaw:22) is the sole
-  # ingress. The password gates Hermex logins and lives in the host .env.
-  hermes-webui:
-    image: docker.io/nousresearch/hermes-agent:latest
-    container_name: hermes-webui
-    restart: unless-stopped
-    entrypoint: ["bash", "-c", "python3 /webui/bootstrap.py --foreground"]
-    environment:
-      HERMES_HOME: /data
-      # Container-internal bind: 0.0.0.0 so Podman's port forwarder (which
-      # dials the container's bridge IP) can reach it. The *host-side*
-      # publish below stays on 127.0.0.1, so loopback-only ingress holds.
-      HERMES_WEBUI_HOST: 0.0.0.0
-      HERMES_WEBUI_PORT: "8787"
-      HERMES_WEBUI_PASSWORD: "${HERMES_WEBUI_PASSWORD}"
-      # Headless host: never show the first-run wizard (a profile already
-      # exists in /data from the gateway).
-      HERMES_WEBUI_SKIP_ONBOARDING: "1"
-    ports:
-      - "127.0.0.1:8787:8787"
-    volumes:
-      - /home/dank/hermes-data:/data
-      # rw on purpose: bootstrap.py creates /webui/.venv in the repo root.
-      - /home/dank/hermes-webui:/webui
-    healthcheck:
-      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8787/health')"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      # First boot pip-installs WebUI + agent deps into a fresh venv.
-      start_period: 600s
-
-  # --- Git hosting (Gitea) -------------------------------------------------
-  # Now boot-persistent Quadlet units (private bridge network, DNS enabled),
-  # not compose services. See gitea/quadlet/ and ../docs/Dankclaw/Gitea.md.
-
-  # --- Observability -------------------------------------------------------
-  prometheus:
-    image: docker.io/prom/prometheus:latest
-    container_name: prometheus
-    restart: unless-stopped
-    # Rootless: container root maps to the invoking user (dank), which
-    # owns /opt/prometheus/data. The image's default user (nobody) maps to
-    # a subuid that cannot write it.
-    user: "0:0"
-    command:
-      - --config.file=/etc/prometheus/prometheus.yml
-      - --storage.tsdb.path=/prometheus
-      - --storage.tsdb.retention.time=30d
-      - --web.enable-lifecycle
-    ports:
-      - "9090:9090"
-    volumes:
-      - /opt/prometheus/config:/etc/prometheus
-      - /opt/prometheus/data:/prometheus
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  grafana:
-    image: docker.io/grafana/grafana:latest
-    container_name: grafana
-    restart: unless-stopped
-    # Rootless: container root maps to the invoking user (dank), which
-    # owns /opt/grafana/data. The image's default user (472) maps to a
-    # subuid that cannot write it.
-    user: "0:0"
-    environment:
-      GF_SERVER_ROOT_URL: "https://grafana.franksdank.cloud"
-      GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}"
-      GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
-    ports:
-      - "3002:3000"
-    volumes:
-      - /opt/grafana/data:/var/lib/grafana
-      - /opt/grafana/provisioning:/etc/grafana/provisioning:ro
-    depends_on:
-      - prometheus
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  loki:
-    # Local build: grafana/loki + static busybox (healthcheck).
-    image: franksdank/loki:latest
-    container_name: loki
-    restart: unless-stopped
-    # Rootless: container root maps to the invoking user (dank), which
-    # owns /opt/loki/data. The image's default user (10001) maps to a
-    # subuid that cannot write it.
-    user: "0:0"
-    command: ["-config.file=/etc/loki/loki-config.yaml"]
-    ports:
-      - "3100:3100"
-    volumes:
-      - /opt/loki/config:/etc/loki
-      - /opt/loki/data:/loki
-    healthcheck:
-      test: ["CMD", "/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:3100/ready"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  alloy:
-    # Local build: grafana/alloy + static busybox (healthcheck).
-    image: franksdank/alloy:latest
-    container_name: alloy
-    restart: unless-stopped
-    volumes:
-      - /opt/alloy/config:/etc/alloy
-      - /run/systemd/journal/socket:/run/systemd/journal/socket:ro
-      - /var/log:/var/log:ro
-      - /var/lib/containers:/var/lib/containers:ro
-    depends_on:
-      - loki
-    healthcheck:
-      test: ["CMD", "/usr/local/bin/busybox", "wget", "--spider", "-q", "http://localhost:12345/-/healthy"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  node-exporter:
-    image: docker.io/prom/node-exporter:latest
-    container_name: node-exporter
-    restart: unless-stopped
-    network_mode: host
-    pid: host
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    command:
-      - --path.procfs=/host/proc
-      - --path.sysfs=/host/sys
-      - --path.rootfs=/rootfs
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9100/metrics"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  # --- Authentik (SSO / OIDC provider) -------------------------------------
-  # 3-service stack: PostgreSQL + server + worker. Redis was removed in
-  # 2025.10 (sessions/caching moved to PostgreSQL). The Caddyfile's
-  # forward_auth vhosts proxy to authentik-server:9000 (compose network).
-  # See ../docs/Infrastructure/Authentik.md.
-  authentik-postgres:
-    image: docker.io/library/postgres:16-alpine
-    container_name: authentik-postgres
-    restart: unless-stopped
-    environment:
-      POSTGRES_DB: authentik
-      POSTGRES_USER: authentik
-      POSTGRES_PASSWORD: ${AUTHENTIK_PG_PASS}
-    volumes:
-      - /home/dank/authentik/postgres:/var/lib/postgresql/data
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d authentik -U authentik"]
-      interval: 30s
-      retries: 5
-      start_period: 20s
-
-  authentik-server:
-    image: ghcr.io/goauthentik/server:2026.8.0
-    container_name: authentik-server
-    restart: unless-stopped
-    command: server
-    environment:
-      AUTHENTIK_POSTGRESQL__HOST: authentik-postgres
-      AUTHENTIK_POSTGRESQL__NAME: authentik
-      AUTHENTIK_POSTGRESQL__USER: authentik
-      AUTHENTIK_POSTGRESQL__PASSWORD: ${AUTHENTIK_PG_PASS}
-      AUTHENTIK_SECRET_KEY: ${AUTHENTIK_SECRET_KEY}
-    ports:
-      - "9000:9000"
-    volumes:
-      - /home/dank/authentik/data:/data
-    shm_size: 512mb
-    depends_on:
-      authentik-postgres:
-        condition: service_healthy
-    healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9000/-/health/')"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-      start_period: 30s
-
-  authentik-worker:
-    image: ghcr.io/goauthentik/server:2026.8.0
-    container_name: authentik-worker
-    restart: unless-stopped
-    command: worker
-    environment:
-      AUTHENTIK_POSTGRESQL__HOST: authentik-postgres
-      AUTHENTIK_POSTGRESQL__NAME: authentik
-      AUTHENTIK_POSTGRESQL__USER: authentik
-      AUTHENTIK_POSTGRESQL__PASSWORD: ${AUTHENTIK_PG_PASS}
-      AUTHENTIK_SECRET_KEY: ${AUTHENTIK_SECRET_KEY}
-    user: "0:0"
-    volumes:
-      - /home/dank/authentik/data:/data
-    depends_on:
-      authentik-postgres:
-        condition: service_healthy
-
-  # Caddy (the homelab's single reverse proxy) left compose on 2026-08-26:
-  # it is boot-persistent via its Quadlet unit (dankclaw/caddy/quadlet/
-  # caddy.container), installed by deploy_caddy in scripts/deploy-remote.sh.
-  # The rendered Caddyfile, certs, and config still live under
-  # /home/dank/Containers/caddy (rootless-writable).

dankclaw/gitea/.env.example

@@ -1,4 +0,0 @@
-# Gitea stack secrets. Copy to /opt/gitea/.env on dankclaw (mode 0600).
-# The live password never enters this repo.
-
-GITEA_DB_PASSWORD=changeme

dankclaw/gitea/quadlet/gitea-postgres.container

@@ -1,31 +0,0 @@
-# Quadlet unit: Gitea PostgreSQL (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/gitea-postgres.container
-# Unit name: gitea-postgres.service (podman 5.x drops the container- prefix)
-#
-# The DB password lives in /opt/gitea/.env on the host (mode 0600).
-# This file holds no secrets. See docs/Dankclaw/Gitea.md.
-# The [Service] EnvironmentFile= line lets systemd expand
-# ${GITEA_DB_PASSWORD} in the generated ExecStart.
-
-[Unit]
-Description=Gitea PostgreSQL container
-After=network.target
-
-[Container]
-ContainerName=gitea-postgres
-Network=gitea.network
-Image=docker.io/library/postgres:16-alpine
-EnvironmentFile=/opt/gitea/.env
-Environment=POSTGRES_USER=gitea
-Environment=POSTGRES_PASSWORD=${GITEA_DB_PASSWORD}
-Environment=POSTGRES_DB=gitea
-Volume=/opt/gitea/data/gitea-postgres:/var/lib/postgresql/data
-
-[Service]
-EnvironmentFile=/opt/gitea/.env
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/gitea/quadlet/gitea-server.container

@@ -1,44 +0,0 @@
-# Quadlet unit: Gitea web server (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/gitea-server.container
-# Unit name: gitea-server.service (podman 5.x drops the container- prefix)
-#
-# The [Service] EnvironmentFile= line lets systemd expand
-# ${GITEA_DB_PASSWORD} in the generated ExecStart.
-# The DB password lives in /opt/gitea/.env on the host (mode 0600).
-# This file holds no secrets. See docs/Dankclaw/Gitea.md.
-#
-# Ports (live): 3000 = HTTP UI/API, 2222 = git SSH. The docs table in
-# Dankclaw Stack.md historically said 3001; the live host port is 3000.
-
-[Unit]
-Description=Gitea web server container
-Wants=gitea-postgres.service
-After=gitea-postgres.service network.target
-
-[Container]
-ContainerName=gitea-server
-Network=gitea.network
-Image=docker.io/gitea/gitea:latest
-EnvironmentFile=/opt/gitea/.env
-Environment=DATABASE_TYPE=postgres
-Environment=DATABASE_HOST=gitea-postgres
-Environment=DATABASE_NAME=gitea
-Environment=DATABASE_USER=gitea
-Environment=DATABASE_PASSWD=${GITEA_DB_PASSWORD}
-Environment=DATABASE_PORT=5432
-Environment=USER_UID=1000
-Environment=USER_GID=1000
-PublishPort=0.0.0.0:3000:3000
-PublishPort=2222:22
-Volume=/opt/gitea/data/gitea-gitea:/data
-Volume=/etc/timezone:/etc/timezone:ro
-Volume=/etc/localtime:/etc/localtime:ro
-
-[Service]
-EnvironmentFile=/opt/gitea/.env
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/gitea/quadlet/gitea.network

@@ -1,10 +0,0 @@
-# Quadlet network: private bridge for the Gitea stack (rootless).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/gitea.network
-# Unit name: gitea-network.service (podman network name: systemd-gitea)
-#
-# DNS is enabled by default for quadlet networks. This matters: the
-# automatic "podman" default network has dns_enabled=false, so the
-# gitea-postgres and gitea-server container names would not resolve.
-
-[Network]

dankclaw/hermes/config.yaml.example

@@ -1,58 +0,0 @@
-# Hermes configuration backup / non-secret baseline.
-#
-# Source on Dankclaw: /home/dank/hermes-data/config.yaml
-# This file is a sanitized Git backup, not a complete replacement for the
-# live Hermes state. Do not add dashboard password hashes, phone numbers,
-# OAuth credentials, API keys, sessions, memories, or generated state here.
-# The live directory is bind-mounted into the containers as /data.
-
-model:
-  provider: dankclaw-large
-  base_url: http://192.168.0.143:7770/v1
-  api_key: not-needed
-  default: frank-default-large
-
-providers:
-  dankclaw-large:
-    api: http://192.168.0.143:7770/v1
-    discover_models: false
-    models:
-      frank-default-large:
-        context_length: 262144
-  dankserver-small:
-    api: http://192.168.0.235:7770/v1
-    discover_models: false
-    models:
-      frank-default-small:
-        context_length: 163840
-
-# Auxiliary routing from the live configuration. These use the existing
-# custom endpoint shape and are retained here as a settings backup.
-auxiliary:
-  vision: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  web_extract: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  compression: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  skills_hub: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  approval: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  mcp: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  title_generation: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  memory_query_rewrite: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  tts_audio_tags: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  triage_specifier: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  kanban_decomposer: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  profile_describer: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-  curator: {provider: custom, model: frank-default-small, base_url: http://192.168.0.235:7770/v1}
-
-dashboard:
-  # basic_auth intentionally omitted; its password_hash remains live-only.
-  font: atkinson-hyperlegible
-skills:
-  disabled: []
-onboarding:
-  seen:
-    busy_input_prompt: true
-_config_version: 34
-platforms:
-  signal:
-    enabled: true
-    # extra.http_url and account intentionally omitted from the backup.

dankclaw/loki/Containerfile

@@ -1,18 +0,0 @@
-# Loki + a static busybox.
-#
-# grafana/loki is distroless: no shell, no wget, so the compose healthcheck
-# cannot run anything inside the container. This image drops a static
-# busybox into it for the healthcheck command only. All Loki behavior is
-# unchanged.
-#
-# Build on dankclaw:
-#   podman build -t franksdank/loki:latest -f dankclaw/loki/Containerfile .
-
-FROM docker.io/library/alpine:latest AS tools
-# busybox-static is statically linked: the target images have no dynamic
-# loader (loki is distroless), so the alpine default /bin/busybox (musl
-# dynamic) would fail with "no such file or directory".
-RUN apk add --no-cache busybox-static
-
-FROM docker.io/grafana/loki:latest
-COPY --from=tools /bin/busybox.static /usr/local/bin/busybox

dankclaw/open-webui/quadlet/open-webui.container

@@ -1,50 +0,0 @@
-# Quadlet unit: open-webui (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/open-webui.container
-# Unit name: open-webui.service (container name: open-webui)
-#
-# Chat UI for the LLM fleet. It runs on the CPU image (dankclaw is CPU-only, so
-# the :cuda image would waste a large pull for a GPU it cannot use) and points
-# its OpenAI connection at the live llama.cpp backend on this host via the
-# host LAN IP (192.168.0.143:7770, model frank-default-large). It publishes 3005
-# (3000 is Gitea's HTTP port). Data persists under
-# /home/dank/Containers/open-webui/data. See docs/Dankclaw/OpenWebUI.md.
-#
-# Network history (2026-08-22): this unit used Network=slirp4netns. When the
-# slirp4netns helper process died, the container kept running but lost its tap
-# interface: a zombie that accepted TCP and reset every request, and chat
-# served empty responses for ~2 days with no error anywhere. It now runs on a
-# pinned bridge network (open-webui.network), which has no per-container
-# helper to die. The LLM is reached via the host LAN IP (192.168.0.143):
-# the bridge gateway lives in the rootless netns, not the host netns where
-# llama-server runs. The healthcheck probes /health (app liveness) and
-# requires a non-loopback interface (network liveness).
-
-[Unit]
-Description=open-webui chat UI container
-After=network.target
-
-[Container]
-ContainerName=open-webui
-Network=open-webui.network
-Image=ghcr.io/open-webui/open-webui:main
-PublishPort=3005:8080
-Environment=OPENAI_API_BASE_URL=http://192.168.0.143:7770
-Environment=ANONYMIZED_TELEMETRY=False
-Environment=DO_NOT_TRACK=True
-Volume=/home/dank/Containers/open-webui/data:/app/backend/data
-
-# App liveness (/health) AND network liveness (a non-loopback interface must
-# exist; a dead tap leaves only "lo" behind).
-HealthCmd=CMD-SHELL python -c 'import os,urllib.request,sys; urllib.request.urlopen("http://127.0.0.1:8080/health",timeout=5).read(); sys.exit(0 if any(i!="lo" for i in os.listdir("/sys/class/net")) else 1)'
-HealthInterval=30s
-HealthTimeout=10s
-HealthRetries=3
-HealthStartPeriod=300s
-
-[Service]
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/open-webui/quadlet/open-webui.network

@@ -1,31 +0,0 @@
-# Quadlet network: private bridge for open-webui (rootless).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/open-webui.network
-# Unit name: open-webui-network.service (podman network name: systemd-open-webui)
-#
-# Why a bridge instead of slirp4netns (2026-08-22 incident):
-# slirp4netns runs a user-space helper process per container. When that
-# helper died, the container kept running but lost its tap interface,
-# becoming a zombie that accepted TCP and reset every request. chat
-# served empty responses for ~2 days and nothing logged an error.
-# A kernel bridge has no per-container helper, so the failure mode
-# cannot occur.
-#
-# The subnet is pinned so the container's own IP (10.89.3.2) is
-# deterministic. Note the LLM backend is NOT reached via this gateway:
-# the gateway lives in the rootless netns, while llama-server runs in
-# the host netns. The host's LAN IP (192.168.0.143) reaches it through
-# the rootless NAT. host.containers.internal is a slirp4netns-only name
-# and does not resolve on a bridge.
-#
-# Subnet registry on dankclaw (rootless):
-#   10.88.0.0/16  default "podman" bridge
-#   10.89.0.0/24  gitea (systemd-gitea)
-#   10.89.1.0/24  compose stack (dankclaw_default)
-#   10.89.2.0/24  searxng (systemd-searxng)
-#   10.89.3.0/24  open-webui (systemd-open-webui)  <- this file
-
-[Network]
-Driver=bridge
-Subnet=10.89.3.0/24
-Gateway=10.89.3.1

dankclaw/pihole/addn-hosts

@@ -1,45 +0,0 @@
-# A records for the flat franksdank.name scheme (dnsmasq addn-hosts format:
-# <IP> <name> [<name> ...]). Served by the Pi-hole quadlet (FTL is a
-# dnsmasq fork): the quadlet mounts this file into /etc/pihole/hosts/ read-only.
-#
-# Caddy is consolidated on dankclaw (192.168.0.143): it fronts every service
-# on both hosts, so every name resolves to that one IP. Keep in sync with the
-# Caddyfile (tests/pihole.bats cross-checks this).
-#
-#   192.168.0.143 = dankclaw (runs the Caddy container + the AI/dev stack)
-#   192.168.0.235 = dankserver (media stack; reached by Caddy via published
-#                    LAN ports — no DNS records point here)
-
-# --- Homepage (naked domain + www) + legacy names ---
-192.168.0.143 franksdank.cloud
-192.168.0.143 www.franksdank.cloud
-192.168.0.143 dankclaw.franksdank.cloud
-192.168.0.143 dankserver.franksdank.cloud
-
-# --- Authentik (SSO / OIDC provider) ---
-192.168.0.143 auth.franksdank.cloud
-
-# --- Media services (dankserver, proxied) ---
-192.168.0.143 plex.franksdank.cloud
-192.168.0.143 plex.franksdank.casa
-192.168.0.143 books.franksdank.cloud
-192.168.0.143 romm.franksdank.cloud
-192.168.0.143 files.franksdank.cloud
-192.168.0.143 jellyfin.franksdank.cloud
-192.168.0.143 radarr.franksdank.cloud
-192.168.0.143 sonarr.franksdank.cloud
-192.168.0.143 lidarr.franksdank.cloud
-192.168.0.143 readarr.franksdank.cloud
-192.168.0.143 prowlarr.franksdank.cloud
-192.168.0.143 qbittorrent.franksdank.cloud
-
-# --- AI / search / dev services (dankclaw, local) ---
-192.168.0.143 search.franksdank.cloud
-192.168.0.143 comfyui.franksdank.cloud
-192.168.0.143 chat.franksdank.cloud
-192.168.0.143 hermes.franksdank.cloud
-192.168.0.143 gitea.franksdank.cloud
-192.168.0.143 grafana.franksdank.cloud
-192.168.0.143 prometheus.franksdank.cloud
-192.168.0.143 loki.franksdank.cloud
-192.168.0.143 pihole.franksdank.cloud

dankclaw/pihole/env.example

@@ -1,18 +0,0 @@
-# Pi-hole container environment (dankclaw).
-# Copy to /home/dank/Containers/pihole/env and fill real values.
-# The real env file is never committed (it holds the web UI password).
-
-# The host IP the container answers for (also shown in the web UI).
-ServerIP=192.168.0.143
-
-# Timezone for the web UI and logs.
-TZ=America/Los_Angeles
-
-# Pinned upstream resolvers. The slirp netns has no useful resolv.conf, so
-# pin them (mirrors the retired dnsmasq.conf server= lines).
-DNS1=1.1.1.1
-DNS2=1.0.0.1
-
-# Web UI password (the shared family password; the same credential as
-# CADDY_BASICAUTH_HASH in the host .env).
-WEBPASSWORD=changeme

dankclaw/pihole/quadlet/pihole.container

@@ -1,42 +0,0 @@
-# Quadlet unit: Pi-hole LAN DNS (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/pihole.container
-# Unit name: pihole.service (container name: pihole)
-#
-# The LAN's authoritative DNS for the flat franksdank.cloud / franksdank.casa
-# name scheme. It retired the dnsmasq quadlet (2026-08-24): Pi-hole FTL is a
-# dnsmasq fork, so the addn-hosts record file carries over unchanged and is
-# mounted into /etc/pihole/hosts/ (the FTL hostsdir) read-only. It serves one A
-# record per Caddy vhost and forwards everything else to the pinned upstream
-# resolvers (DNS1/DNS2 in the env file). The SURFboard G36 router hands this
-# server out to DHCP clients (DNS statically configured -> 192.168.0.143),
-# so no client needs per-device config.
-#
-# Runs on slirp4netns with port 53 (DNS) and 8083 (web UI) published to the
-# LAN (the same pattern as open-webui). Config + state mount from
-# /home/dank/Containers/pihole. The web UI password is WEBPASSWORD in the env
-# file. See docs/Infrastructure/LAN-DNS.md.
-
-[Unit]
-Description=Pi-hole LAN DNS container
-After=network-online.target
-Wants=network-online.target
-
-[Container]
-ContainerName=pihole
-Image=docker.io/pihole/pihole:latest
-Network=slirp4netns
-PublishPort=0.0.0.0:53:53/udp
-PublishPort=0.0.0.0:53:53/tcp
-PublishPort=0.0.0.0:8083:80/tcp
-EnvironmentFile=/home/dank/Containers/pihole/env
-Volume=/home/dank/Containers/pihole/etc-pihole:/etc/pihole
-Volume=/home/dank/Containers/pihole/etc-dockers:/etc/dockers
-Volume=/home/dank/Containers/pihole/addn-hosts:/etc/pihole/hosts/addn-hosts:ro
-
-[Service]
-Restart=always
-RestartSec=5
-
-[Install]
-WantedBy=default.target

dankclaw/pihole/sysctl/99-dnsmasq.conf

@@ -1,8 +0,0 @@
-# Rootless DNS must bind port 53: lower the unprivileged-port threshold so a
-# rootless container may publish 0.0.0.0:53 (80/443 stay allowed; 53 < 80).
-# Single source of truth for net.ipv4.ip_unprivileged_port_start on dankclaw;
-# deploy_pihole installs it to /etc/sysctl.d/ and applies it with
-# `sysctl --system` (the same code path boot uses via systemd-sysctl).
-# A later-sorted drop-in that reset this to 80 took LAN DNS down after the
-# 2026-08-26 reboot — deploy_pihole neutralizes such conflicts.
-net.ipv4.ip_unprivileged_port_start=53

dankclaw/searxng/quadlet/gluetun.container

@@ -1,39 +0,0 @@
-# Quadlet unit: Gluetun VPN tunnel (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/gluetun.container
-# Unit name: gluetun.service (container name: gluetun)
-#
-# Creates the WireGuard tunnel to ProtonVPN (US-WA). SearXNG reuses this
-# container's network namespace (see searxng.container), so its outbound search
-# traffic exits here. The WireGuard credentials live in /opt/searxng/.env
-# (mode 0600); this file holds no secrets. See docs/Dankclaw/Gluetun.md.
-
-[Unit]
-Description=Gluetun VPN tunnel container
-Wants=searxng-network.service
-After=searxng-network.service network.target
-
-[Container]
-ContainerName=gluetun
-Network=searxng.network
-Image=docker.io/qmcgaw/gluetun:latest
-AddCapability=NET_ADMIN
-AddDevice=/dev/net/tun:/dev/net/tun
-Environment=VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER}
-Environment=VPN_TYPE=${VPN_TYPE}
-Environment=WIREGUARD_PRIVATE_KEY=${WIREGUARD_PRIVATE_KEY}
-Environment=WIREGUARD_ADDRESSES=${WIREGUARD_ADDRESSES}
-Environment=WIREGUARD_PUBLIC_KEY=${WIREGUARD_PUBLIC_KEY}
-Environment=WIREGUARD_ENDPOINT_IP=${WIREGUARD_ENDPOINT_IP}
-Environment=WIREGUARD_ENDPOINT_PORT=${WIREGUARD_ENDPOINT_PORT}
-Environment=WIREGUARD_ALLOWED_IPS=${WIREGUARD_ALLOWED_IPS}
-Environment=WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL=${WIREGUARD_PERSISTENT_KEEPALIVE_INTERVAL}
-PublishPort=8081:8080
-
-[Service]
-EnvironmentFile=/opt/searxng/.env
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/searxng/quadlet/searxng-valkey.container

@@ -1,28 +0,0 @@
-# Quadlet unit: Valkey cache for SearXNG (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/searxng-valkey.container
-# Unit name: searxng-valkey.service (container name: searxng-valkey)
-#
-# The limiter's rate-limit state store. Joins the searxng network and is
-# reachable by the name "valkey" (settings.yml points there). Data persists in
-# /opt/searxng/valkey. See docs/Dankclaw/Valkey.md.
-
-[Unit]
-Description=SearXNG Valkey cache container
-Wants=searxng-network.service
-After=searxng-network.service network.target
-
-[Container]
-ContainerName=searxng-valkey
-Network=searxng.network
-NetworkAlias=valkey
-Image=docker.io/valkey/valkey:9-alpine
-Volume=/opt/searxng/valkey:/data
-
-[Service]
-EnvironmentFile=/opt/searxng/.env
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/searxng/quadlet/searxng.container

@@ -1,30 +0,0 @@
-# Quadlet unit: SearXNG search engine (rootless, boot-persistent).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/searxng.container
-# Unit name: searxng.service (container name: searxng)
-#
-# Reuses gluetun's network namespace (Network=gluetun.container) so its
-# outbound traffic exits the WireGuard tunnel. It publishes no port of its own;
-# the port lives on gluetun. It reaches valkey by name on the searxng network.
-# Config mounts from /opt/searxng/config. See docs/Dankclaw/SearXNG.md.
-
-[Unit]
-Description=SearXNG search engine container
-Wants=gluetun.service searxng-valkey.service
-After=gluetun.service searxng-valkey.service network.target
-
-[Container]
-ContainerName=searxng
-Network=gluetun.container
-Image=ghcr.io/searxng/searxng:latest
-Environment=SEARXNG_SECRET=${SEARXNG_SECRET}
-Environment=SEARXNG_BASE_URL=${SEARXNG_BASE_URL}
-Volume=/opt/searxng/config:/etc/searxng
-
-[Service]
-EnvironmentFile=/opt/searxng/.env
-TimeoutStartSec=900
-Restart=always
-
-[Install]
-WantedBy=default.target

dankclaw/searxng/quadlet/searxng.network

@@ -1,16 +0,0 @@
-# Quadlet network: private bridge for the SearXNG stack (rootless).
-#
-# Install path on dankclaw: ~/.config/containers/systemd/searxng.network
-# Unit name: searxng-network.service (podman network name: systemd-searxng)
-#
-# DNS is enabled by default for quadlet networks. This matters: the automatic
-# "podman" default network has dns_enabled=false, so container names would not
-# resolve. searxng (which reuses gluetun's netns) reaches valkey by the name
-# "valkey" on this network.
-#
-# The subnet is pinned so the value stays stable across reboots. It must match
-# an entry in shared/searxng/limiter.toml trusted_proxies. 10.89.2.0/24 is free
-# (gitea uses 10.89.0.0/24; the old compose net used 10.89.1.0/24).
-
-[Network]
-Subnet=10.89.2.0/24

dankserver/.env.example

@@ -1,24 +0,0 @@
-# Dankserver stack secrets and configuration.
-# Copy to .env and fill real values. Never commit real secrets.
-
-# RomM admin credentials (first boot)
-ROMM_USERNAME=changeme
-ROMM_PASSWORD=changeme
-ROMM_AUTH_SECRET_KEY=changeme
-
-# RomM database (MariaDB)
-ROMM_DB_USER=romm
-ROMM_DB_PASSWD=changeme
-MARIADB_ROOT_PASSWORD=changeme
-
-# RomM Redis
-# (no password needed for local Redis)
-
-# Copyparty
-COPAY_PASS=changeme
-
-# Jellyfin
-JELLYFIN_PUBLISH_SERVER_URL=http://192.168.0.235:8096
-
-# *arr Stack
-ARR_TZ=America/New_York

dankserver/compose.yaml

@@ -1,272 +0,0 @@
-# Dankserver stack — media & library services.
-# Deploy: cd /home/dank/serverAppConfigs && podman-compose -f dankserver/compose.yaml up -d
-# See ../docs/Dankserver/ for service documentation.
-
-networks:
-  appnet:
-    driver: bridge
-
-services:
-  # ═══════════════════════════════════════════════
-  # RomM — retro game manager (requires MariaDB + Redis)
-  # ═══════════════════════════════════════════════
-  mariadb:
-    image: docker.io/mariadb:11
-    container_name: romm-mariadb
-    restart: unless-stopped
-    environment:
-      MARIADB_ROOT_PASSWORD: "${MARIADB_ROOT_PASSWORD}"
-      MARIADB_DATABASE: romm
-      MARIADB_USER: "${ROMM_DB_USER}"
-      MARIADB_PASSWORD: "${ROMM_DB_PASSWD}"
-    volumes:
-      - /opt/romm/mariadb:/var/lib/mysql
-    healthcheck:
-      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-
-  redis:
-    image: docker.io/redis:7-alpine
-    container_name: romm-redis
-    restart: unless-stopped
-    volumes:
-      - /opt/romm/redis:/data
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-
-  romm:
-    image: ghcr.io/rommapp/romm:latest
-    container_name: romm
-    restart: unless-stopped
-    environment:
-      ROMM_USERNAME: "${ROMM_USERNAME}"
-      ROMM_PASSWORD: "${ROMM_PASSWORD}"
-      ROMM_AUTH_SECRET_KEY: "${ROMM_AUTH_SECRET_KEY}"
-      DB_HOST: mariadb
-      DB_NAME: romm
-      DB_USER: "${ROMM_DB_USER}"
-      DB_PASSWD: "${ROMM_DB_PASSWD}"
-      REDIS_HOST: redis
-      HASHEOUS_API_ENABLED: "true"
-    ports:
-      - "8085:8080"
-    volumes:
-      - /opt/romm/library:/romm/library
-      - /opt/romm/resources:/romm/resources
-      - /opt/romm/config:/romm/config
-    depends_on:
-      mariadb:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  # ═══════════════════════════════════════════════
-  # Copyparty — file sharing and streaming
-  # ═══════════════════════════════════════════════
-  copyparty:
-    image: ghcr.io/9001/copyparty:latest
-    container_name: copyparty
-    restart: unless-stopped
-    environment:
-      COPAY_PASS: "${COPAY_PASS}"
-      COPAY_MODE: rw
-    ports:
-      - "3923:3923"
-    volumes:
-      - /opt/copyparty/data:/data
-    healthcheck:
-      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3923/')"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  # ═══════════════════════════════════════════════
-  # Jellyfin — media server (movies, TV, music)
-  # ═══════════════════════════════════════════════
-  jellyfin:
-    image: jellyfin/jellyfin:latest
-    container_name: jellyfin
-    restart: unless-stopped
-    user: "1000:1000"
-    ports:
-      - "8096:8096/tcp"
-      - "8920:8920/tcp"
-      - "7359:7359/udp"
-      - "1900:1900/udp"
-    environment:
-      - JELLYFIN_PublishedServerUrl=http://192.168.0.235:8096
-      - DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
-    volumes:
-      - /opt/jellyfin/config:/config
-      - /opt/jellyfin/cache:/cache
-      - /mnt/media/movies:/data/movies:ro
-      - /mnt/media/shows:/data/shows:ro
-    devices:
-      - /dev/dri:/dev/dri
-    group_add:
-      - "992"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8096/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  # ═══════════════════════════════════════════════
-  # Calibre-Web — e-book web frontend
-  # ═══════════════════════════════════════════════
-  calibre-web:
-    image: lscr.io/linuxserver/calibre-web:0.6.27-ls397
-    container_name: calibre-web
-    restart: unless-stopped
-    environment:
-      PUID: "1000"
-      PGID: "1000"
-      TZ: Etc/UTC
-      OAUTHLIB_RELAX_TOKEN_SCOPE: "1"
-    ports:
-      - "8083:8083"
-    volumes:
-      - /opt/calibre-web/config:/config
-      - /opt/calibre-web/books:/books
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8083/"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-
-  # ═══════════════════════════════════════════════
-  # *arr Stack — automated media download
-  # ═══════════════════════════════════════════════
-  prowlarr:
-    image: lscr.io/linuxserver/prowlarr:latest
-    container_name: prowlarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/prowlarr/config:/config
-      - /mnt/media/downloads:/downloads:ro
-    ports:
-      - "9696:9696"
-    networks: [appnet]
-
-  qbittorrent:
-    image: lscr.io/linuxserver/qbittorrent:latest
-    container_name: qbittorrent
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/qbittorrent/config:/config
-      - /mnt/media/downloads:/downloads
-      - /mnt/media/downloads/incomplete:/downloads/incomplete
-    ports:
-      - "8080:8080"
-    networks: [appnet]
-
-  radarr:
-    image: lscr.io/linuxserver/radarr:latest
-    container_name: radarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/radarr/config:/config
-      - /mnt/media/movies:/movies
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "7878:7878"
-    networks: [appnet]
-
-  sonarr:
-    image: lscr.io/linuxserver/sonarr:latest
-    container_name: sonarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/sonarr/config:/config
-      - /mnt/media/tv:/tv
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "8989:8989"
-    networks: [appnet]
-
-  lidarr:
-    image: lscr.io/linuxserver/lidarr:latest
-    container_name: lidarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/lidarr/config:/config
-      - /mnt/media/music:/music
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "8686:8686"
-    networks: [appnet]
-
-  readarr:
-    image: lscr.io/linuxserver/readarr:latest
-    container_name: readarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/readarr/config:/config
-      - /mnt/media/books:/books
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "8787:8787"
-    networks: [appnet]
-
-  # ═══════════════════════════════════════════════
-  # Node Exporter — host metrics
-  # ═══════════════════════════════════════════════
-  node-exporter:
-    image: prom/node-exporter:latest
-    container_name: node-exporter
-    restart: unless-stopped
-    network_mode: host
-    pid: host
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    command:
-      - --path.procfs=/host/proc
-      - --path.sysfs=/host/sys
-      - --path.rootfs=/rootfs
-    healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9100/metrics"]
-      interval: 30s
-      timeout: 10s
-      retries: 3

deploy.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# deploy.sh — Deploy inference scripts to remote servers
+#
+# Usage: ./deploy.sh [TARGET] [SCRIPT]
+#   TARGET: dankserver, dankclaw, or all
+#   SCRIPT: Script name to deploy (default: all scripts in scripts/)
+
+TARGET="${1:-all}"
+SCRIPT="${2:-}"
+
+# Remote configuration
+declare -A REMOTE_HOSTS
+REMOTE_HOSTS=(
+    ["dankserver"]="dank@dankserver"
+    ["dankclaw"]="dank@dankclaw"
+)
+
+declare -A REMOTE_DIRS
+REMOTE_DIRS=(
+    ["dankserver"]="/home/dank/vllm"
+    ["dankclaw"]="/home/dank/vllm"
+)
+
+declare -A REMOTE_SERVICES
+REMOTE_SERVICES=(
+    ["dankserver"]="vllm-awq-qwen36.service"
+    ["dankclaw"]="vllm-thinkingcap.service"
+)
+
+deploy_to() {
+    local target="$1"
+    local remote="${REMOTE_HOSTS[$target]}"
+    local remote_dir="${REMOTE_DIRS[$target]}"
+    local service="${REMOTE_SERVICES[$target]}"
+    
+    echo "=== Deploying to $target ==="
+    
+    # Create remote directory if it doesn't exist
+    sshpass -p 'Assface.3241' ssh "$remote" "mkdir -p $remote_dir"
+    
+    # Deploy scripts
+    if [ -n "$SCRIPT" ]; then
+        if [ -f "scripts/$SCRIPT" ]; then
+            echo "  Deploying scripts/$SCRIPT..."
+            sshpass -p 'Assface.3241' scp "scripts/$SCRIPT" "$remote:$remote_dir/$SCRIPT"
+        else
+            echo "  ERROR: scripts/$SCRIPT not found"
+            return 1
+        fi
+    else
+        for script in scripts/*.sh; do
+            if [ -f "$script" ]; then
+                local name=$(basename "$script")
+                echo "  Deploying $name..."
+                sshpass -p 'Assface.3241' scp "$script" "$remote:$remote_dir/$name"
+            fi
+        done
+    fi
+    
+    # Make scripts executable
+    if [ -n "$SCRIPT" ]; then
+        sshpass -p 'Assface.3241' ssh "$remote" "chmod +x $remote_dir/$SCRIPT"
+    else
+        sshpass -p 'Assface.3241' ssh "$remote" "chmod +x $remote_dir/*.sh"
+    fi
+    
+    # Restart service if it exists
+    echo "  Restarting $service..."
+    sshpass -p 'Assface.3241' ssh "$remote" "echo 'Assface.3241' | sudo -S systemctl restart $service 2>/dev/null || echo 'Service $service not found or failed'"
+    
+    echo "  Deploy to $target complete"
+    echo ""
+}
+
+case "$TARGET" in
+    dankserver|dankclaw)
+        deploy_to "$TARGET"
+        ;;
+    all)
+        for target in "${!REMOTE_HOSTS[@]}"; do
+            deploy_to "$target"
+        done
+        ;;
+    *)
+        echo "ERROR: Unknown target $TARGET"
+        echo "Usage: $0 [dankserver|dankclaw|all] [script]"
+        exit 1
+        ;;
+esac
+
+echo "=== Deployment complete ==="

docs/.obsidian/app.json

@@ -1,8 +0,0 @@
-{
-  "alwaysUpdateLinks": true,
-  "newFileLocation": "folder",
-  "newFileFolderPath": "Dankclaw",
-  "attachmentFolderPath": "_attachments",
-  "useMarkdownLinks": false,
-  "showUnsupportedFiles": true
-}

docs/Architecture/Deployment Flow.md

@@ -1,128 +0,0 @@
----
-type: guide
-title: Deployment Flow
-description: Service-scoped Git deployment for the two homelab hosts.
-tags:
-  - deployment
-  - operations
-  - git
-timestamp: 2026-09-03
----
-
-# Deployment Flow
-
-Git is the single source of truth. `scripts/deploy.sh` pushes the committed
-`main` branch, pulls it on one target, and applies only the runtime services
-whose tracked inputs changed. A SearXNG change no longer recreates the
-inference service; an LLM change no longer restarts unrelated containers.
-
-## Workflow
-
-```mermaid
-graph LR
-    A[Edit in this repo] --> B[Run tests]
-    B --> C[Commit main]
-    C --> D[deploy.sh computes plan]
-    D --> E[Push main]
-    E --> F[Target pulls with ff-only]
-    F --> G[deploy-remote.sh dispatches selected services]
-    G --> H[Per-service health check]
-```
-
-1. Edit files locally in this repository.
-2. Run `./tests/run.sh`, then commit the complete change on `main`.
-3. Run `./scripts/deploy.sh <target>`.
-4. The planner maps changed paths to services. Compose YAML is compared by
-   service definition so an edit to one service does not restart its peers.
-5. The target pulls with `git pull --ff-only origin main` and runs the remote
-   dispatcher with the explicit plan.
-6. The dispatcher updates only selected Quadlet, compose, homepage, Caddy, or
-   systemd services and records each successful service revision outside Git.
-
-## Commands
-
-```bash
-./scripts/deploy.sh dankclaw
-./scripts/deploy.sh dankserver
-./scripts/deploy.sh dankclaw --service searxng
-./scripts/deploy.sh dankserver --service frank-default-small
-./scripts/deploy.sh dankclaw --dry-run
-```
-
-`--service` is repeatable and is an explicit retry/override. Without it, the
-planner compares the local `HEAD` to `origin/main`. A documentation-only
-commit produces an empty plan. The remote state directory can also replay a
-service whose earlier deployment failed, but it never expands a plan to a
-service on the other host.
-
-## Service managers
-
-| Service family | Runtime manager | Scope behavior |
-|---|---|---|
-| Dankclaw compose services | `podman-compose -f dankclaw/compose.yaml up -d --no-deps NAME` | Recreates only `NAME`; Caddy is handled separately because it fronts both hosts. |
-| SearXNG, Gluetun, Valkey, OpenWebUI, Pi-hole, Gitea database/server | user Quadlet + `systemctl --user` | Installs the selected unit and restarts only the required dependency-coupled units. Gitea's web proxy is the canonical Caddy compose service. |
-| `frank-default-large` | systemd + `systemctl` | Relinks and restarts only the large inference service; waits up to 20 minutes for `/health`. |
-| `frank-default-small` | systemd + `systemctl` | Relinks and restarts only the small inference service; waits up to 20 minutes for `/health`. |
-| Dankserver media inventory | host-managed bare containers | Compose is inventory only; the scoped deploy does not create duplicate containers. |
-
-The shared chat template maps to the inference service on the selected target.
-The inactive `frank-default-npu` unit is relink-only when explicitly selected;
-it is not restarted by an unrelated large-model deployment.
-
-## Caddy safety
-
-Caddy runs on Dankclaw and fronts both hosts, including Gitea. The former
-standalone `gitea-caddy` Quadlet is retired; the canonical compose `caddy`
-service is now the sole reverse-proxy owner of ports 80/443. A Caddy reload
-failure falls back to a restart of **only** the selected `caddy` container.
-Before deploying, verify that no retired proxy still owns either port.
-
-Homepage files, generated `deployed.js`, the status checker, and the rendered
-Caddyfile are updated only when their service is selected. A media-only
-Dankserver deployment does not touch Dankclaw's proxy or homepage.
-
-## Per-service state
-
-The remote runner writes one revision file per successfully completed service
-to:
-
-```text
-${XDG_STATE_HOME:-~/.local/state}/serverAppConfigs/deploy-state/
-```
-
-The directory is outside the checkout and is never committed. A failed handler
-is not marked successful. On the next ordinary deployment, the runner compares
-the recorded revision with the new checkout and retries only a still-changed
-service. This prevents a failed Caddy or LLM deployment from being silently
-forgotten while avoiding a host-wide restart.
-
-## Targets
-
-| Target | Host | Repository checkout |
-|---|---|---|
-| dankclaw | `dank@dankclaw` (`192.168.0.143`) | `/home/dank/serverAppConfigs` |
-| dankserver | `dank@dankserver` (`192.168.0.235`) | `/home/dank/serverAppConfigs` |
-
-SSH uses the existing key pair. No password is stored or documented in this
-repository. If sudo is needed on a headless target, `FRANK_SUDO_PW` is read
-from the environment and is never written to the repository.
-
-## Tests
-
-Run the full suite before pushing:
-
-```bash
-./tests/run.sh
-```
-
-The deployment seams are in `tests/deployment_scoping.bats` and
-`tests/deployment_flow.bats`. They cover path isolation, compose narrowing,
-Caddy safety, inference isolation, per-service state, and legacy wrapper
-routing. See [[Testing Policy]].
-
-## See Also
-
-- [[Testing Policy]]
-- [LLM deployment](../LLM/deployment.md)
-- [[Caddy]]
-- [[SearXNG]]

docs/Architecture/Network Architecture.md

@@ -1,63 +0,0 @@
-# Network Architecture
-
-One Caddy fronts both hosts. It runs as a container on Dankclaw (192.168.0.143) and terminates TLS for every service on the LAN. Pi-hole on Dankclaw answers the service names for LAN clients; Porkbun answers them for the WAN. The G36 router firewall governs external access. SearXNG outbound traffic exits through the Gluetun VPN container.
-
-## Topology
-
-```mermaid
-graph TB
-    subgraph WAN["G36 router 192.168.0.1"]
-        EXT[Internet]
-    end
-    EXT -->|port forwards| C
-    subgraph Dankclaw[192.168.0.143]
-        C[Caddy :80 :443]
-        D[Pi-hole :53]
-        C -->|comfyui:8188| F[ComfyUI]
-        C -->|192.168.0.143:3000| G[Gitea quadlet]
-        C -->|192.168.0.143:8081| SE[SearXNG quadlet]
-        C -->|192.168.0.143:3005| O[OpenWebUI quadlet]
-        SE -->|network_mode service| GL[Gluetun VPN]
-        GL -->|tunnel| EXT
-    end
-    subgraph Dankserver[192.168.0.235]
-        C -->|192.168.0.235:8085| R[RomM]
-        C -->|192.168.0.235:3923| P[Copyparty]
-        C -->|192.168.0.235:8083| W[Calibre-Web]
-        C -->|192.168.0.235:32400| PL[Plex host service]
-    end
-    CLIENT[LAN clients] -->|DHCP DNS| D
-```
-
-Dankserver runs no proxy of its own. Its services expose published LAN ports, and the Dankclaw Caddy routes the names to those ports.
-
-## Hostnames
-
-| Scope | Convention | Resolved by |
-|-------|------------|-------------|
-| LAN | `romm.franksdank.cloud` → 192.168.0.143 | Pi-hole (`addn-hosts`) |
-| WAN | `romm.franksdank.cloud` → 71.227.227.172 | Porkbun A record |
-
-Users type a name and nothing else. Caddy maps the name to the upstream. The split between the two resolvers removes the need for router NAT loopback. See [[LAN-DNS]] for the flow and the router hand-off.
-
-## TLS
-
-Caddy holds one Let's Encrypt certificate per vhost, issued and renewed through the DNS-01 challenge via the Porkbun module. No inbound port 80/443 is required. Details in [[Caddy]].
-
-## Gluetun Path (SearXNG only)
-
-SearXNG attaches to the Gluetun network namespace (`network_mode: service:gluetun`). Only SearXNG shares the tunnel. Everything else on Dankclaw uses the normal LAN route. This matches KB Pattern 2 in [[Gluetun]].
-
-## Access Policy
-
-- The **G36 router firewall** governs what the WAN can reach. Port forwards are manual router steps; the repo tracks no router state.
-- Caddy adds HTTP basic auth on the homepage and on sensitive services.
-- LAN subnet: `192.168.0.0/24`.
-
-## See Also
-
-- [[LAN-DNS]] — name resolution, Pi-hole, router hand-off
-- [[Caddy]] — vhost map, TLS, basic auth
-- [[Dankserver Stack]], [[Dankclaw Stack]]
-- [[Gluetun]]
-- Knowledge base: [[Gluetun]], [[SearXNG]]

docs/Architecture/Storage Policy.md

@@ -1,40 +0,0 @@
-# Storage Policy
-
-All persistent data uses **bind mounts**: a host directory mapped directly into the container. No named volumes. This matches the preference of this repo and the existing Gitea stack on Dankclaw.
-
-## Why Bind Mounts
-
-- The exact host location of every byte is visible and controllable.
-- Backups are a simple `tar` of known host directories.
-- The data survives container recreation and `podman-compose down`.
-- Ownership is explicit: everything runs as UID 1000 (`dank`) so host tools can read it.
-
-## Conventions
-
-| Stack | Host root |
-|-------|-----------|
-| Dankserver | `/opt/<service>/...` |
-| Dankclaw | `/opt/<service>/...` (Gitea keeps `/opt/gitea/data/...`) |
-
-## Ownership
-
-Containers run as `PUID=1000` / `PGID=1000`. Host directories must be owned by `dank`:
-
-```bash
-sudo chown -R 1000:1000 /opt/<service>
-```
-
-## Secrets
-
-Secrets never live in bind-mounted configs that get committed. They are:
-
-1. Set in `.env` files on the host (git-ignored, `.env.example` committed as template), or
-2. Injected as environment variables from the deploy script.
-
-The live Gitea database password exists only in `.env` on Dankclaw. A test enforces that it never appears in this repo.
-
-## See Also
-
-- [[Deployment Flow]]
-- [[Gitea]] — the existing bind-mount stack being migrated
-- [[Backblaze Integration]] — future backup target for these mounts

docs/Architecture/Testing Policy.md

@@ -1,45 +0,0 @@
-# Testing Policy
-
-This repository tests its configuration at defined seams. The tests run with `bats`; run them with `./tests/run.sh` before every push.
-
-## Seams Under Test
-
-| Seam | File | What it asserts |
-|------|------|-----------------|
-| Compose validity | `tests/compose_validity.bats` | YAML parses, services have images, no named volumes |
-| Env consistency | `tests/env_consistency.bats` | every `${VAR}` is defined in `.env.example`; no real secrets; the live gitea password never appears |
-| Caddyfile validity | `tests/caddyfile.bats` | per-host Caddyfile, `.local` convention, balanced braces, basic auth present |
-| Docs completeness | `tests/docs_completeness.bats` | every compose service has a docs page; index links every page |
-| Script syntax | `tests/scripts_syntax.bats` | `bash -n` clean, executable bits, shellcheck when available |
-| Deployment flow | `tests/deployment_flow.bats`, `tests/deployment_scoping.bats` | scoped dispatcher exists, targets are known, service plans isolate inference, Caddy ownership is guarded, and SSH reachability is checked (skips offline) |
-
-## Focused pre-commit testing
-
-For normal commits, use the focused wrapper:
-
-```bash
-./scripts/test-changed.sh
-```
-
-It reads staged paths with `git diff --cached` and runs only the mapped Bats
-files. The tracked `.githooks/pre-commit` hook invokes this wrapper. Enable the
-repository hook path once per checkout:
-
-```bash
-git config core.hooksPath .githooks
-```
-
-Use `--list` to inspect the selection without running tests, or `--paths FILE
-...` to preview a mapping. Shared test infrastructure and unmapped paths fail
-closed rather than silently skipping coverage. Run the full suite with
-`./tests/run.sh` when changing shared test helpers, the test runner, or when
-focused coverage is not sufficient.
-
-## Red-Green
-
-Tests are written first. A failing test means the seam is not satisfied; the config change that makes it pass is the only acceptable fix. No `--no-verify`, no deleting tests without approval.
-
-## See Also
-
-- [[Deployment Flow]]
-- [[index]]

docs/Dankclaw System Specs.md

@@ -0,0 +1,81 @@
+# Dankclaw System Specs
+
+## Server Overview
+
+Dankclaw is an AMD APU server for vLLM inference. The host runs Ubuntu 26.04 with an AMD Ryzen AI MAX+ 395 Strix Halo APU. The APU uses unified memory — CPU and GPU share the same physical RAM pool.
+## Hardware Specifications
+
+### CPU
+
+| Property | Value |
+|----------|-------|
+| Model | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S |
+| Architecture | x86_64 |
+| Cores | 16 (1 socket, 1 thread per core) |
+| Max MHz | 5187.50 |
+| Min MHz | 625.00 |
+| NUMA Nodes | 1 (node 0, 62909 MB) |
+
+### GPU (Integrated)
+
+| Property | Value |
+|----------|-------|
+| Model | AMD Radeon 8060S (Strix Halo) |
+| Device ID | 0x1586 |
+| Architecture | gfx1151 (Strix Halo) |
+| GPU-Visible Memory | 96 GiB (ROCm addressable range) |
+| Physical RAM | 30 GiB (shared CPU + GPU) |
+
+### ROCm
+
+| Property | Value |
+|----------|-------|
+| ROCk Module | Loaded |
+| Runtime Version | 1.18 |
+| ROCm Version | 7.2.4 (from AMD repos) |
+| Agent Name | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S |
+| GPU Arch | gfx1151 (Strix Halo) |
+| GPU Global Memory | 96 GiB (overcommitted, physical is 30 GiB) |
+
+### Memory
+
+| Property | Value |
+|----------|-------|
+| System RAM | 30 GiB (shared CPU + GPU) |
+| GPU-Visible (ROCm) | 96 GiB (overcommitted addressable range) |
+| Swap | 8.0 GiB |
+| /dev/shm | ~15 GiB (half of RAM) |
+
+### Storage
+
+| Filesystem | Size | Used | Avail | Use% |
+|------------|------|------|-------|------|
+| / (nvme0n1p2) | 1.9T | 391G | 1.4T | 22% |
+
+### OS and Software
+
+| Property | Value |
+|----------|-------|
+| OS | Ubuntu 26.04 LTS (Resolute Raccoon) |
+| Kernel | 7.0.0-29-generic |
+| Python | 3.14.4 (system), 3.12.13 (vLLM venv) |
+| vLLM | 0.26.0+rocm723 |
+| uv | 0.12.2 |
+
+## VLLM Relevance
+
+- Unified memory eliminates separate VRAM limits. The GPU uses system RAM.
+- ROCm reports 96 GiB GPU memory (overcommitted). Physical RAM is 30 GiB. vLLM sees 96 GiB and will try to use it. Set `--gpu-memory-utilization` conservatively (0.80-0.85) to avoid OOM when the OS needs memory.
+- Only 8 GiB swap. Set `--swap-space` conservatively (4-8 GiB).
+- ROCm 7.2.4 from AMD repos. vLLM ROCm support requires ROCm 6.3+.
+- Python 3.14.4 is very new. vLLM ROCm wheels target Python 3.12. Use a Python 3.12 venv.
+- Radeon 8060S falls under Ryzen AI MAX series (gfx1151). vLLM lists this as supported.
+- 1.4 TB free disk space is ample for model storage.
+- `libmpi_cxx.so.40` is missing from the system. Install `openmpi-bin` or provide a workaround for torch MPI dependency.
+
+## See Also
+
+- [[VLLM Overview]]
+- [[Shared Memory Architecture Dankclaw]]
+- [[RAM CPU Offloading in VLLM]]
+- [[Running Qwen3.6 27B FP8 on Dankclaw]]

docs/Dankclaw/ComfyUI.md

@@ -1,79 +0,0 @@
-# ComfyUI
-
-ComfyUI is a node-based AI image generation engine. Workflows are built from nodes that load models, sample latents, and post-process images; everything is scriptable through a web UI and an API.
-
-**Host:** Dankclaw — **Hostname:** `comfyui.dankclaw.local` — **Port:** 8188 — **Backend:** ROCm GPU (gfx1151) — **Docs:** https://github.com/Comfy-Org/ComfyUI
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `comfyui-rocm:gfx1151` — built from `dankclaw/Containerfile.comfyui` |
-| Base image | `docker.io/rocm/vllm:rocm7.12.0_gfx1151_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0` |
-| Port | 8188 (host) → 8188 (container) |
-| Backend | ROCm GPU (gfx1151 / Radeon 8060S). No `--cpu` flag. |
-| Data | `/home/dank/comfyui-data/{models,input,output,custom_nodes}` → `/opt/ComfyUI/...` |
-| Run | `podman compose -f dankclaw/compose.yaml up -d comfyui` |
-| Boot-persistent | user systemd unit `comfyui.service` (runs the compose `up` at boot) |
-
-## Why the GPU works here
-
-Dankclaw is an AMD Strix Halo APU (gfx1151). Standard PyTorch ROCm wheels do not ship gfx1151 kernels. So `torch.cuda.is_available()` returns `False` and ComfyUI falls back to CPU. AMD publishes a gfx1151-tagged build, and the image above is based on it. On this APU it reports `torch.cuda.is_available() = True`, device `Radeon 8060S`, capability `(11, 5)`. See [[ComfyUI ROCm and Vulkan]] for the full verification.
-
-## How the image is built
-
-`dankclaw/Containerfile.comfyui` starts from the AMD gfx1151 base image and bakes the ComfyUI code in. It installs ComfyUI's dependencies but skips the torch family (torch, torchvision, torchaudio). The base image already ships the matching ROCm builds. Installing an unpinned `torch` from `requirements.txt` clobbers the gfx1151 kernels with a default build, so the Containerfile filters those lines out.
-
-## GPU device access (rootless)
-
-The container reaches the GPU through the `/dev/kfd` and `/dev/dri` device passthrough. The permission is checked host-side. The run user is already a member of the `render` group on the host, so the device is readable.
-
-Only the `video` group is added to the container. The `render` group (gid 990) is not added. Rootless podman cannot map it into the container namespace (the subgid range is 100000+), so `--group-add render` fails to start the container.
-
-## Volumes
-
-| Container path | Host path |
-|----------------|-----------|
-| `/opt/ComfyUI/models` | `/home/dank/comfyui-data/models` |
-| `/opt/ComfyUI/input` | `/home/dank/comfyui-data/input` |
-| `/opt/ComfyUI/output` | `/home/dank/comfyui-data/output` |
-| `/opt/ComfyUI/custom_nodes` | `/home/dank/comfyui-data/custom_nodes` |
-
-The ComfyUI code is baked into the image (not mounted). Rebuild the image to update the code.
-
-## Custom nodes
-
-Custom nodes go in `/home/dank/comfyui-data/custom_nodes` (mounted at `/opt/ComfyUI/custom_nodes`). ComfyUI loads every subdirectory there on start. Because this path is a bind mount, it shadows anything baked into the image — so custom nodes are installed host-side, not in the Containerfile.
-
-**ComfyUI-Manager** is installed here. Install it with a one-time clone, then restart the container:
-
-```bash
-git clone https://github.com/ltdrdata/ComfyUI-Manager \
-  /home/dank/comfyui-data/custom_nodes/ComfyUI-Manager
-systemctl --user restart comfyui
-```
-
-ComfyUI-Manager self-installs its own pip dependencies on start (GitPython, PyGithub, matrix-nio, transformers, and more). To confirm it loaded, query `GET /manager/version` (returns a version like `V3.41`). Its pip installs live in the container's writable layer, so they survive a restart and a reboot but not an image rebuild; after a rebuild, Manager re-installs them on the next start.
-
-## Boot persistence
-
-The user systemd unit `~/.config/systemd/user/comfyui.service` runs `podman compose -f /home/dank/serverAppConfigs/dankclaw/compose.yaml up -d comfyui` at boot. It sets `WantedBy=default.target`, and user linger is on. The old CPU venv unit is preserved at `comfyui.service.venv.bak` as a fallback.
-
-## Healthcheck
-
-The web UI responds on `/`. ComfyUI also exposes `/system_stats`. On this APU, `/system_stats` reports `device=cuda:0 Radeon 8060S Graphics` (type `cuda`, the ROCm device).
-
-## Gotchas
-
-- The GPU path needs the device passthrough. Do not remove `/dev/kfd` or `/dev/dri` from the compose service.
-- Do not add the `render` group to the container. Rootless podman cannot map it. Add only `video`.
-- Do not let pip reinstall the torch family. The Containerfile filters it out on purpose.
-- The base image is about 32 GB. A code update means a rebuild, because the code is baked in.
-- The model collection starts empty. Add models to `/home/dank/comfyui-data/models` before generating.
-
-## See Also
-
-- [[Dankclaw Stack]]
-- [[ComfyUI ROCm and Vulkan]] — the live-verified GPU path research
-- [[Lemonade and sd.cpp]] — the ggml-family alternative
-- Knowledge base: [[ComfyUI]] (tools)

docs/Dankclaw/Dankclaw Stack.md

@@ -1,73 +0,0 @@
----
-type: reference
-title: Dankclaw Stack
-description: Service inventory and routing overview for the Dankclaw AI, development, and observability host.
-tags:
-  - dankclaw
-  - services
-  - containers
-  - caddy
-timestamp: 2026-08-26
----
-
-# Dankclaw Stack
-
-Dankclaw (192.168.0.143) runs the AI, search, and development stack. It
-hosts the LLM inference endpoint (`frank-default-large`, `:7770`, CIRU
-ROCmFPX llama.cpp), the agent tools, the privacy search, and the
-observability stack. The llama.cpp endpoint (`frank-default-small`,
-`:7770`) runs on Dankserver (192.168.0.235).
-
-```mermaid
-graph LR
-    U[User] --> C[Caddy :80/:443]
-    C --> O[OpenWebUI]
-    C --> S[SearXNG]
-    C --> G[Gitea]
-    C --> F[ComfyUI]
-    C --> D[Grafana]
-    S --> V[Gluetun VPN]
-    O --> L[llama.cpp dankserver :7770]
-    O --> H[Hermes]
-    O --> E[CIRU llama.cpp :7770]
-```
-
-## Services
-
-| Service | Hostname | Port | Docs |
-|---------|----------|------|------|
-| SearXNG | search.dankclaw.local | 8081 | [[SearXNG]] |
-| ComfyUI | comfyui.dankclaw.local | 8188 | [[ComfyUI]] |
-| OpenWebUI | chat.dankclaw.local | 3005 | [[OpenWebUI]] |
-| Hermes | hermes.dankclaw.local | 8090 | [[Hermes]] |
-| Gitea | gitea.dankclaw.local | 3000 | [[Gitea]] |
-| Grafana | grafana.dankclaw.local | 3002 | [[Grafana]] |
-| Prometheus | prometheus.dankclaw.local | 9090 | [[Prometheus]] |
-| Homepage | dankclaw.local | 80/443 | — |
-| frank-default-large (host service, Dankclaw) | — | 7770 | [[deployment]] |
-| frank-default-small (host service, Dankserver) | — | 7770 | [[deployment]] |
-
-The last two rows are bare-metal systemd services, not containers. This
-repo owns their scripts, units, and knowledge base (see
-[docs/LLM/index.md](../LLM/index.md)). Current mapping (2026-08-22):
-`frank-default-large` runs the CIRU ROCmFPX llama.cpp on Dankclaw;
-`frank-default-small` runs the upstream llama.cpp on Dankserver.
-
-## Boot Persistence
-
-The Gitea database and server, the SearXNG trio (gluetun, searxng, valkey), open-webui, Pi-hole, and Caddy run as Quadlet units, so they start at boot. Caddy left `compose.yaml` on 2026-08-26 after a reboot exposed that bare compose containers never come back; see [[Caddy]] for the unit and the incident. The former standalone Gitea Caddy unit is retired. User linger is on. ComfyUI runs as a systemd unit around a bare process. The remaining services in the table are defined in `compose.yaml` and are not yet boot-persistent — per AGENTS.md (Boot Persistence & Self-Healing) they each need a quadlet before they may be considered reliably deployed.
-
-Deploy the SearXNG trio with `scripts/deploy-searxng-quadlet.sh dankclaw`, open-webui with `scripts/deploy-open-webui-quadlet.sh dankclaw`, and Pi-hole with `scripts/deploy-pihole-quadlet.sh dankclaw`. The Gitea units install to `~/.config/containers/systemd/` (see [[Gitea]]); Caddy is deployed through `scripts/deploy.sh dankclaw --service caddy`.
-
-## Routing Rules
-
-- SearXNG outbound traffic exits through the [[Gluetun]] VPN container.
-- Every other service uses the normal LAN route.
-- The homepage lists all services with status.
-
-## See Also
-
-- [[index]] — repository index
-- [[Network Architecture]] — routing and proxy
-- [[Observability]] — Grafana, Prometheus, logs
-- [LLM fleet knowledge base](../LLM/index.md) — the model-serving endpoints on this host

docs/Dankclaw/Gitea.md

@@ -1,106 +0,0 @@
----
-type: configuration
-title: Gitea
-description: Gitea deployment on Dankclaw with its web UI reverse-proxied by the canonical Caddy service.
-tags:
-  - gitea
-  - dankclaw
-  - quadlet
-  - caddy
-timestamp: 2026-08-26
----
-
-# Gitea
-
-Gitea is a self-hosted Git service with issues, pull requests, and a web UI. On Dankclaw it runs as two Quadlet-managed containers in the rootless Podman session of user `dank`. The canonical Caddy on the same host reverse-proxies its web UI at `gitea.franksdank.cloud`.
-
-**Host:** Dankclaw — **LAN access:** `http://dankclaw:3000` (or `http://192.168.0.143:3000`) — **Proxy domain:** `gitea.franksdank.cloud` — **Docs:** https://docs.gitea.com
-
-## Containers
-
-| Container | Unit | Image | Purpose |
-|-----------|------|-------|---------|
-| `gitea-postgres` | `gitea-postgres.service` | `docker.io/library/postgres:16-alpine` | Database |
-| `gitea-server` | `gitea-server.service` | `docker.io/gitea/gitea:latest` | Web UI, API, git (HTTP :3000, SSH :2222) |
-
-The containers share the private bridge network `systemd-gitea`. The quadlet network file `gitea.network` enables DNS, so the containers resolve each other by name. The automatic `podman` default network has DNS disabled. Do not run the Gitea containers on it.
-
-## Hosted repositories
-
-| Repository | Purpose |
-|------------|---------|
-| `dank/serverAppConfigs` | this repo — Podman stacks + LLM fleet |
-| `dank/vllmScripts` | retired 2026-08-22 — merged into `dank/serverAppConfigs` (history preserved) |
-
-The LLM fleet knowledge base: [docs/LLM/index.md](../LLM/index.md)
-
-## Management
-
-Run all commands as `dank` on Dankclaw.
-
-| Action | Command |
-|--------|---------|
-| Start all | `systemctl --user start gitea-postgres gitea-server` |
-| Stop all | `systemctl --user stop gitea-server gitea-postgres` |
-| Restart Gitea only | `systemctl --user restart gitea-server` |
-| Status | `systemctl --user status gitea-postgres gitea-server` |
-| Container logs | `podman logs -f gitea-server` |
-| Unit logs | `journalctl --user -u gitea-server -f` |
-
-The units start in dependency order:
-
-```mermaid
-graph LR
-    N[gitea-network.service] --> P[gitea-postgres.service]
-    P --> S[gitea-server.service]
-```
-
-## Boot persistence
-
-User-level linger is enabled for `dank` (`loginctl enable-linger dank`). The units start at host boot without a login session. Each unit carries `[Install] WantedBy=default.target`. The podman generator applies it at boot; no symlink is needed.
-
-The containers use the quadlet fresh-start model. A stop removes the container (`podman run --rm --replace`). The next start recreates it. The bind-mounted data survives. The canonical Caddy is managed separately by the Dankclaw stack.
-
-## Secrets
-
-The database password lives in `/opt/gitea/.env` on the host (mode 0600). The quadlet files reference it with `EnvironmentFile=`. The password never enters this repository. Placeholders live in [`dankclaw/gitea/.env.example`](../../dankclaw/gitea/.env.example).
-
-Do not change the password in place. Change the PostgreSQL role first, then update `/opt/gitea/.env` and the `[database] PASSWD` line in the app.ini (see Gotchas), then restart the stack.
-
-## Configuration files
-
-| File | Location | Repo copy |
-|------|----------|-----------|
-| Quadlet units | `~/.config/containers/systemd/gitea-{postgres,server}.container`, `gitea.network` | [`dankclaw/gitea/quadlet/`](../../dankclaw/gitea/quadlet/) |
-| Caddyfile | `/home/dank/Containers/caddy/Caddyfile` | [`dankclaw/Caddyfile`](../../dankclaw/Caddyfile) |
-| Secrets | `/opt/gitea/.env` (0600) | [`dankclaw/gitea/.env.example`](../../dankclaw/gitea/.env.example) |
-
-After editing a repo copy, deploy through `scripts/deploy.sh dankclaw`. The Gitea database/server units are installed from this directory; the canonical Caddy and its rendered Caddyfile are deployed as the separate `caddy` service. Do not manually recreate the retired `gitea-caddy` unit.
-
-## Data volumes
-
-| Container path | Host path | Contents |
-|----------------|-----------|----------|
-| `/data` | `/opt/gitea/data/gitea-gitea` | Git repositories, app.ini, attachments |
-| `/var/lib/postgresql/data` | `/opt/gitea/data/gitea-postgres` | PostgreSQL data |
-
-Back up `/opt/gitea/data/` (tar) or `pg_dump` the database. See the knowledge base page [[Gitea]].
-
-## Gotchas
-
-- **app.ini wins over env vars.** Gitea writes `/data/gitea/conf/app.ini` (host: `/opt/gitea/data/gitea-gitea/gitea/conf/app.ini`) on first start. The `[database] HOST` and `PASSWD` lines persist and override the `DATABASE_HOST`/`DATABASE_PASSWD` env vars. When you rename the database container, edit the `HOST` line.
-- **The files in the data directory are owned by host UID 100999**, not `dank` (rootless ID mapping: container uid 1000 → host 100999). Edit them inside the container with `podman exec --user 1000 gitea-server ...`.
-- **The host auto-reboots.** `unattended-upgrades` reboots Dankclaw when a package needs it. The quadlet units make Gitea survive that. See the 2026-08-16 incident below.
-- **Port 3000 vs 3001.** The live host port is 3000. The full-stack [`dankclaw/compose.yaml`](../../dankclaw/compose.yaml) maps 3001. Align the two before deploying the full stack.
-- **Legacy compose retired.** `/opt/gitea/podman-compose.yml` is retired to `/opt/gitea/.deprecated/`. Do not run it. Its containers collide with the Quadlet units on ports 3000 and 2222.
-- **Canonical proxy.** `gitea.franksdank.cloud` is served by the Caddy service in `dankclaw/compose.yaml`, which proxies to `192.168.0.143:3000`. Do not recreate a Gitea-specific Caddy unit or publish another proxy on ports 80/443. During migration, the deployer stages any installed legacy unit under `~/.AGENTTRASH/YYYYMMDD-gitea-caddy/` before starting canonical Caddy.
-
-## Incident: 2026-08-16 — down after unattended reboot
-
-`unattended-upgrades` rebooted Dankclaw at 15:01 on 2026-08-16. The Gitea containers ran through podman-compose with `restart: unless-stopped` and no systemd units. Rootless podman does not auto-start containers at boot without units. Gitea stayed down until a manual start at 16:02. The fix: migrate the stack to quadlet units with linger (this page). After the fix, verify on the next reboot.
-
-## See Also
-
-- [[Dankclaw Stack]]
-- [[Postgres]]
-- Knowledge base: [[Gitea]] (tools)

docs/Dankclaw/Gluetun.md

@@ -1,74 +0,0 @@
-# Gluetun
-
-Gluetun is a lightweight VPN client container. It connects to a VPN provider (WireGuard or OpenVPN) and shares that tunnel with other containers that attach to its network namespace.
-
-**Host:** Dankclaw — **Docs:** https://github.com/qdm12/gluetun
-
-## Facts (from research, primary source)
-
-| Item | Value |
-|------|-------|
-| Image | `qmcgaw/gluetun:latest` |
-| Capabilities | `NET_ADMIN` required |
-| Device | `/dev/net/tun:/dev/net/tun` |
-| Ports published | searxng's 8081 → 8080 lives here |
-| Tunnel check | the `tun0` interface exists (the `:8000` server 404s on all paths) |
-
-## Required Environment
-
-```yaml
-environment:
-  VPN_SERVICE_PROVIDER: "${VPN_SERVICE_PROVIDER}"   # e.g. mullvad, protonvpn
-  VPN_TYPE: wireguard
-  WIREGUARD_PRIVATE_KEY: "${WIREGUARD_PRIVATE_KEY}" # from .env, never committed
-```
-
-- `VPN_TYPE` default is `openvpn`; WireGuard preferred when the provider supports it.
-- Provider-specific variables come from the Gluetun wiki; set them in `.env`.
-
-## Optional DNS (from research)
-
-| Variable | Default | Purpose |
-|----------|---------|---------|
-| `DNS_UPSTREAM_RESOLVER_TYPE` | `dot` | DNS over TLS |
-| `DNS_UPSTREAM_RESOLVERS` | `cloudflare` | upstream DoT providers |
-| `DNS_CACHING` | `on` | cache queries |
-| `BLOCK_MALICIOUS` | `on` | block malicious hostnames |
-
-Warning from the wiki: do NOT set `DNS_UPSTREAM_PLAIN_ADDRESSES` to a LAN/bridge address — DNS queries leak out of the tunnel.
-
-## Attaching a Container
-
-The searxng unit reuses gluetun's network namespace:
-
-```ini
-[Container]
-ContainerName=searxng
-Network=gluetun.container
-```
-
-`Network=gluetun.container` also adds a systemd dependency on gluetun, so searxng starts after the tunnel.
-
-## Boot Persistence
-
-gluetun runs as the `gluetun` quadlet unit (see [[SearXNG]] for the full trio). It sets `WantedBy=default.target` and user linger is on, so the tunnel starts at boot. The WireGuard credentials live in `/opt/searxng/.env` (mode 0600). Deploy with `scripts/deploy-searxng-quadlet.sh dankclaw`.
-
-## Verify the Tunnel
-
-```bash
-podman exec gluetun curl -s ifconfig.me
-```
-
-The returned IP must differ from the host IP.
-
-## Gotchas
-
-- The container that holds the published port must be the one with the ports mapping (gluetun).
-- `restart: always` on gluetun so the tunnel reconnects after VPN drops.
-- When gluetun restarts, attached containers inherit the new network state; searxng may need a restart to re-resolve DNS.
-
-## See Also
-
-- [[SearXNG]] — the consumer of the tunnel
-- [[Dankclaw Stack]]
-- Knowledge base: [[SearXNG]]

docs/Dankclaw/Hermes-Dashboard.md

@@ -1,41 +0,0 @@
-# Hermes-Dashboard
-
-The Hermes web dashboard is a browser UI for managing the agent. It shows config, API keys, and sessions. It is a management tool. It is not a chat window. Chat happens over Signal or the CLI. See [[Hermes]].
-
-**Host:** Dankclaw — **Port:** `9119` — **URL:** `http://192.168.0.143:9119`
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/nousresearch/hermes-agent:latest` (same as the gateway) |
-| Command | `hermes dashboard --host 0.0.0.0 --port 9119 --no-open --skip-build` |
-| Port | `9119` (published to the LAN) |
-| Auth | HTTP basic auth (username + scrypt password hash) |
-| Data | shares `/home/dank/hermes-data` with the gateway |
-
-## Why a separate container
-
-The gateway container runs `hermes gateway run` (the Signal platform). That entrypoint does not start the dashboard. The dashboard is a separate process. So it runs in its own compose service (`hermes-dashboard`). Both services mount the same data volume. Both restart at boot (`restart: unless-stopped`).
-
-## Authentication
-
-The dashboard refuses to bind to a non-loopback address without an auth provider. There is no unauthenticated public bind. This repo uses basic auth:
-
-- `dashboard.basic_auth.username` and `dashboard.basic_auth.password_hash` live in the live `/home/dank/hermes-data/config.yaml`; the tracked backup at `dankclaw/hermes/config.yaml.example` intentionally omits the password hash.
-- The stored value is a one-way scrypt hash. The plaintext password is never written to a file or the repo.
-- The hash is generated with `plugins.dashboard_auth.basic.hash_password`.
-
-## Access
-
-Open `http://192.168.0.143:9119` in a browser on the LAN. Log in with the configured username and password.
-
-## Gotchas
-
-- The dashboard and the gateway share one data volume. A config change made in the dashboard takes effect when the gateway restarts.
-- If the shared password is reused elsewhere, treat it as exposed. Rotate it and regenerate the hash.
-- `--skip-build` uses the prebuilt web assets in the image. Do not remove it unless the assets are rebuilt.
-
-## See Also
-
-- [[Hermes-WebUI]] — the phone-facing API server (Hermex)

docs/Dankclaw/Hermes-WebUI.md

@@ -1,114 +0,0 @@
----
-type: configuration
-title: Hermes-WebUI (Hermex server)
-description: The hermes-webui compose service on Dankclaw — the loopback-only API server that the Hermex iPhone app talks to over an SSH tunnel.
-tags:
-  - hermes
-  - hermes-webui
-  - hermex
-  - dankclaw
-  - ssh-tunnel
-timestamp: 2026-09-14
----
-
-# Hermes-WebUI (Hermex server)
-
-`hermes-webui` is the [hermes-webui](https://github.com/nesquena/hermes-webui)
-server (MIT, third-party) that the **Hermex** iPhone app connects to. It runs
-the Hermes agent in-process from the shared `HERMES_HOME`, so it is the *same*
-agent — same memory, skills, and sessions — as the [[Hermes]] gateway and the
-[[Hermes-Dashboard]].
-
-**Host:** Dankclaw — **Port:** `127.0.0.1:8787` (loopback only) — **Ingress:** SSH tunnel
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/nousresearch/hermes-agent:latest` (same as the gateway) |
-| Command | `python3 /webui/bootstrap.py --foreground` |
-| Source checkout | `/home/dank/hermes-webui` on Dankclaw (mounted at `/webui`) |
-| Port | `8787`, published to `127.0.0.1` only |
-| Auth | `HERMES_WEBUI_PASSWORD` (host `.env`; Hermex signs in with it) |
-| Data | shares `/home/dank/hermes-data` with the gateway and dashboard |
-| Health | `GET /health` (compose healthcheck + deployer) |
-
-## Why a container, not a host process
-
-The gateway's state files are owned by the container-mapped uid (`109999`,
-from rootless Podman's `1 → 100000/65536` subuid range). A fresh rootless
-namespace on Dankclaw maps identically (verified with `podman unshare
-cat /proc/self/uid_map`), so a second container using the same image reads
-and writes the shared volume with exactly the ownership the gateway already
-uses. A host-side process running as `dank` (uid 1000) could not read the
-gateway's `0600` state files.
-
-## How it works
-
-- `bootstrap.py` creates `/webui/.venv` on first boot, pip-installs the WebUI
-  and agent requirements, then **execs** `server.py` as one foreground process
-  (`--foreground`). systemd-free: Podman supervises it via
-  `restart: unless-stopped`.
-- `HERMES_WEBUI_SKIP_ONBOARDING=1` keeps the first-run wizard off; a profile
-  already exists in `/data` from the gateway.
-- The healthcheck `start_period` is 600 s because the very first boot spends
-  minutes pip-installing into a fresh venv.
-
-## Remote access (Hermex on iOS, from anywhere)
-
-Only port `22` is forwarded to Dankclaw at the router, so the webui is
-reached through an SSH local forward — no extra router ports, no third-party
-relay (no Tailscale, no Cloudflare):
-
-1. On the iPhone, open **Termius** (or any SSH client with local port
-   forwarding) and connect to `<dankclaw-public>:22` with the usual key.
-2. On that connection, add a **local forward**: `127.0.0.1:8787` →
-   `127.0.0.1:8787`. The forward runs inside the encrypted SSH session;
-   nothing else on the phone routes through it.
-3. In Hermex, set the server URL to `http://localhost:8787` and sign in with
-   the webui password.
-
-The tunnel lives as long as the SSH session on the phone — iOS does not allow
-a true background daemon, so connect before you use Hermex.
-
-```mermaid
-graph LR
-    A[Hermex app] -->|http://localhost:8787| B[SSH local forward on iPhone]
-    B ==>|encrypted SSH to public IP:22| C[Dankclaw sshd]
-    C --> D["hermes-webui 127.0.0.1:8787"]
-    D --> E["/home/dank/hermes-data (shared agent state)"]
-```
-
-## Deployment
-
-```
-./scripts/deploy.sh dankclaw --service hermes-webui
-```
-
-The planner maps changes to the `hermes-webui` compose block (and only that
-block) to this service, so gateway and inference restarts stay isolated.
-Before first deploy, set a strong `HERMES_WEBUI_PASSWORD` in the live
-`dankclaw/.env`.
-
-## Gotchas
-
-- The `/webui` mount is **rw on purpose**: `bootstrap.py` creates `.venv` in
-  the repo root. Do not flip it to `:ro`.
-- Never publish `8787` beyond loopback without keeping the password set —
-  the webui can run agent commands.
-- The webui and the gateway share one data volume. A config change made on one
-  side takes effect when the other restarts (same caveat as the dashboard).
-- `bootstrap.py` auto-installs a Hermes Agent if it cannot find one; in this
-  image the agent is already present, so that path should never fire. If the
-  log shows an install attempt, the image changed — re-verify the entrypoint.
-
-# Citations
-
-- hermes-webui README (env vars, `--foreground`, `/health`): https://github.com/nesquena/hermes-webui
-- Hermex setup model (server URL + password; Cloudflare/Tailscale are optional, not required): https://hermexapp.com/
-
-## See Also
-
-- [[Hermes]] — the agent and its Signal gateway
-- [[Hermes-Dashboard]] — the LAN browser management UI
-- [[Dankclaw Stack]]

docs/Dankclaw/Hermes.md

@@ -1,52 +0,0 @@
-# Hermes
-
-Hermes is the self-improving AI agent from Nous Research. It runs in a terminal, Telegram, Discord, Slack, WhatsApp, or Signal, and keeps a persistent learning loop: it creates skills from experience, searches past conversations, and builds a model of the user across sessions.
-
-**Host:** Dankclaw — **Hostname:** `hermes.dankclaw.local` — **Docs:** https://hermes-agent.nousresearch.com/docs
-
-**Persistent state:** `/home/dank/hermes-data` on Dankclaw, mounted as `/data` in
-both containers. The tracked, sanitized configuration baseline is
-[`dankclaw/hermes/config.yaml.example`](../../dankclaw/hermes/config.yaml.example).
-
-## Facts (from research, primary source)
-
-| Item | Value |
-|------|-------|
-| Image | built from `NousResearch/hermes-agent` (official Dockerfile in repo) |
-| License | MIT |
-| Model backends | Nous Portal, OpenRouter, OpenAI, or any custom endpoint |
-| Interfaces | TUI, Telegram, Discord, Slack, WhatsApp, Signal |
-| Memory | FTS5 session search, Honcho user modeling |
-| Skills | self-created, self-improving during use |
-
-## Configuration
-
-- Point Hermes at the local model endpoint or an external provider through its provider config.
-- The gateway process exposes one port for chat integration; the exact port is set in the Hermes config (`8090` convention in this repo).
-- The data volume persists skills, memory, and session history.
-- Hermes has two named local providers: `dankclaw-large` reaches
-  `http://192.168.0.143:7770/v1` and exposes `frank-default-large`;
-  `dankserver-small` reaches `http://192.168.0.235:7770/v1` and exposes
-  `frank-default-small`.
-- The live `config.yaml` contains secrets and user state and is not committed;
-  the tracked example is the sanitized backup of its non-secret settings.
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/data` | `/home/dank/hermes-data` |
-
-## Gotchas
-
-- Hermes is under active development; pin the image tag for reproducibility.
-- Model choice drives quality; the local `frank-default-large` (Dankclaw, 7770) is the intended backend.
-- Do not expose the gateway without auth — put it behind [[Caddy]] with basic auth.
-
-## See Also
-
-- [[Hermes-Dashboard]] — the LAN browser management UI
-- [[Hermes-WebUI]] — the Hermex server (phone-facing API, SSH-tunneled)
-- [[Dankclaw Stack]]
-- [[OpenWebUI]] — the chat UI that pairs with Hermes
-- Knowledge base: [[SearXNG]] (Hermes as the search agent)

docs/Dankclaw/OpenWebUI.md

@@ -1,75 +0,0 @@
-# OpenWebUI
-
-OpenWebUI is a self-hosted chat interface for LLMs. It connects to OpenAI-compatible endpoints (vLLM, llama.cpp, Ollama) and adds RAG, plugins, tool calling, and multi-user support.
-
-**Host:** Dankclaw — **Port:** 3005 (host) → 8080 (container) — **Boot-persistent:** Quadlet — **Docs:** https://github.com/open-webui/open-webui
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `ghcr.io/open-webui/open-webui:main` (CPU build; Dankclaw is CPU-only) |
-| Port | 3005 (host) → 8080 (container). Port 3000 is Gitea's HTTP, so it is not usable. |
-| Data volume | `/home/dank/Containers/open-webui/data` → `/app/backend/data` |
-| Default backend | CIRU llama.cpp on Dankclaw `:7770` = `frank-default-large` (via host LAN IP `192.168.0.143:7770`) |
-| Network | Pinned bridge `systemd-open-webui` (`10.89.3.0/24`, gateway `10.89.3.1`) — see [History](#history) |
-| Persistence | Quadlet unit `open-webui.service`; `[Install] WantedBy=default.target` |
-| Deploy | `scripts/deploy-open-webui-quadlet.sh dankclaw` |
-
-## The LLM backends
-
-The fleet runs two OpenAI-compatible endpoints. This mapping is the **live** state (verified 2026-08-22). Both services listen on `:7770`, distinguished by host. On 2026-08-22 `frank-default-large` moved from `:52625` to `:7770`.
-
-| Endpoint | Model name | Host | Capability |
-|----------|-----------|------|-----------|
-| `:7770` (CIRU llama.cpp) | `frank-default-large` | Dankclaw | completion |
-| `:7770` (llama.cpp) | `frank-default-small` | Dankserver | completion, multimodal |
-
-The default connection points at the local vLLM (`frank-default-large`), so chat stays on-host. To add the other backend, open **Admin → Connections** in the UI and point an OpenAI connection at `http://192.168.0.235:7770/v1` (key `none`).
-
-> Both endpoints are bare-metal systemd services managed by this repository (the former `vllmScripts` repo, merged in 2026-08-22). Model swaps, context sizing, and spec tuning happen here. See the [LLM knowledge base](../LLM/index.md).
-
-## Boot persistence (Quadlet)
-
-The unit lives at `dankclaw/open-webui/quadlet/open-webui.container`. On the host it installs to `~/.config/containers/systemd/open-webui.container`. The generator turns it into `open-webui.service`. Because the unit sets `[Install] WantedBy=default.target` and user linger is on, it starts at boot. `systemctl --user is-enabled open-webui.service` reports `generated` (the quadlet boot-start state).
-
-## Volumes
-
-| Container path | Host path |
-|----------------|-----------|
-| `/app/backend/data` | `/home/dank/Containers/open-webui/data` |
-
-The data dir holds the database, RAG indexes, and uploads. The deploy script creates it if missing and preserves it across reruns.
-
-## Healthcheck
-
-The quadlet carries a built-in healthcheck (30s interval, 3 retries, 300s start period). It checks two things from inside the container:
-
-1. **App liveness** — `GET http://127.0.0.1:8080/health` returns 200 (`{"status":true}`).
-2. **Network liveness** — a non-loopback interface exists in `/sys/class/net`. The dead-tap zombie state (History) leaves only `lo` behind, and this is the check that catches it.
-
-`podman ps` shows the container as `unhealthy` if either fails. Probe manually at `http://dankclaw:3005/health`.
-
-## Retired instances
-
-- The compose `open-webui` service (port 3000) was never run: 3000 is Gitea's HTTP port, so it could not bind.
-- The bare-container `open-webui` on Dankserver (port 3000) pointed at a dead backend (`:9090`) and was removed. Its data dir is preserved on Dankserver in case it is needed.
-- The bare-container `searxng` on Dankserver (port 8081) was removed. SearXNG now runs only on Dankclaw behind Caddy.
-
-## Gotchas
-
-- Use the non-CUDA image on this CPU-only host.
-- The default admin account is created on first login; secure it immediately.
-- If the backend is loading (3–8 minute model load), OpenWebUI shows connection errors until the model is ready.
-- The LLM is reached via the host LAN IP `192.168.0.143:7770`, not the bridge gateway: the gateway (`10.89.3.1`) lives in the rootless netns, while llama-server runs in the host netns, so the gateway refuses. `host.containers.internal` is slirp4netns-only and does not resolve on a bridge either. To reach the Dankserver backend, use the LAN address `192.168.0.235:7770`.
-
-## History
-
-- **2026-08-22 — slirp4netns → pinned bridge.** The unit ran on `Network=slirp4netns`. The slirp4netns helper process died (no OOM, no kill entry, no crash log found), and the container kept running with only `lo` in its netns: published port 3005 accepted TCP and reset every request, so chat served empty responses for ~2 days. Nothing logged an error. Moving to a kernel bridge removes the per-container helper entirely, and the healthcheck's interface check makes the zombie state visible if it ever returns.
-
-## See Also
-
-- [[Dankclaw Stack]]
-- [[Hermes]] — the agent OpenWebUI can talk to
-- [[SearXNG]] — the search engine on this host
-- [LLM knowledge base](../LLM/index.md) — the vLLM and llama.cpp endpoints

docs/Dankclaw/Postgres.md

@@ -1,70 +0,0 @@
-# Postgres
-
-PostgreSQL is the database backend for Gitea. It runs on the same podman network as Gitea and holds the repository metadata, users, and issue data.
-
-**Host:** Dankclaw — **Docs:** https://www.postgresql.org/docs
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/library/postgres:16-alpine` |
-| Port | 5432 (internal only) |
-| Data volume | bind mount at `/var/lib/postgresql/data` |
-
-## Configuration
-
-```yaml
-services:
-  postgres:
-    image: docker.io/library/postgres:16-alpine
-    restart: unless-stopped
-    environment:
-      POSTGRES_USER: gitea
-      POSTGRES_PASSWORD: "${GITEA_DB_PASSWORD}"
-      POSTGRES_DB: gitea
-    volumes:
-      - /opt/gitea/data/gitea-postgres:/var/lib/postgresql/data
-```
-
-## Gitea Connection
-
-Gitea reaches the database over the `systemd-gitea` network (quadlet-managed live, compose-managed after the full-stack deploy):
-
-```yaml
-environment:
-  DATABASE_TYPE: postgres
-  DATABASE_HOST: gitea-postgres
-  DATABASE_NAME: gitea
-  DATABASE_USER: gitea
-  DATABASE_PASSWD: "${GITEA_DB_PASSWORD}"
-```
-
-The live `app.ini` on the host overrides env vars. When you rename the database container, update its `[database] HOST` line. See [[Gitea]] Gotchas.
-
-## Migration
-
-The live Gitea stack on Dankclaw already runs this image with the data at `/opt/gitea/data/gitea-postgres`. The migration keeps the bind mount in place; see [[Gitea]].
-
-## Healthcheck
-
-Use `pg_isready`:
-
-```yaml
-healthcheck:
-  test: ["CMD-SHELL", "pg_isready -U gitea"]
-  interval: 30s
-  timeout: 5s
-  retries: 3
-```
-
-## Gotchas
-
-- The database password must match between the `postgres` service and Gitea's `DATABASE_PASSWD`; both come from the same `.env` variable.
-- Do not change the Gitea `SECRET_KEY` or `INTERNAL_TOKEN` after migration — encrypted data becomes unreadable.
-- Back up with `pg_dump`; see [[Gitea]] (knowledge base).
-
-## See Also
-
-- [[Gitea]]
-- [[Dankclaw Stack]]

docs/Dankclaw/SearXNG Providers.md

@@ -1,400 +0,0 @@
-# SearXNG Providers
-
-How Google and other upstream search engines get blocked behind the Gluetun
-VPN tunnel, and how to unblock them in SearXNG.
-
-**Host:** Dankclaw — **Docs:** https://docs.searxng.org
-
-SearXNG aggregates ~50 upstream engines (Google, Bing, DuckDuckGo, Brave,
-Wikipedia, …). When all of them are queried through **one** VPN exit IP — as on
-Dankclaw, where SearXNG reuses Gluetun's tunnel — the rate-sensitive engines
-start answering with reCAPTCHA or HTTP 429. SearXNG then *suspends* the offending
-engine for an hour to 15 days and silently drops it from results. That is the
-"Google (and sometimes a few others) went unavailable" symptom.
-
-This page is the fixbook. Read [[SearXNG]] for the stack; the sub-pages below
-are the detail if you need it.
-
-## Provider policy
-
-The default aggregation should maximize **independent, anonymous coverage**
-without turning one user query into a request storm. Disabled engines remain
-installed and selectable on demand; `inactive` is reserved for engines that
-should disappear completely.
-
-| Category | Default baseline | On-demand / deferred |
-|----------|------------------|----------------------|
-| General/web | Bing, DuckDuckGo, Wikipedia | Google CSE, Brave, Startpage, Qwant, Mojeek, Wiby, MWMBL after retesting |
-| Technical | GitHub, MDN, Stack Overflow, Ask Ubuntu, Superuser, Docker Hub, PyPI | Other technical indexes as needed |
-| Science | arXiv, PubMed, Semantic Scholar, OpenAlex-compatible sources | Google Scholar until independently tested |
-| News | Reuters, Bing News, DuckDuckGo News, Wikinews | Google News, Brave News, Startpage News |
-| Images | Openverse, Wikimedia Commons, Bing Images, Unsplash | Google/Brave/Startpage image engines |
-| Videos | YouTube, Vimeo, Dailymotion, Bing Videos | Brave video engines |
-
-This policy uses no API credentials and does not identify the operator to a
-search provider. It is deliberately conservative: a bundled engine is not
-necessarily healthy, anonymous, or suitable for every default query.
-
-## Root cause (one line)
-
-All SearXNG engine traffic shares **one ProtonVPN exit IP** with no rotation, so
-Google/DuckDuckGo see one datacenter IP spamming ~50 paginated engines and
-rate-limit it → CAPTCHA → engine **suspension** → empty results.
-
-## The suspension model (what "unavailable" actually is)
-
-An engine that returns 429 / reCAPTCHA / access-denied raises a named
-exception and is *suspended* (removed from result aggregation for a set time).
-Suspended engines are the usual cause of "gone":
-
-| Exception | Default suspension | Trigger |
-|-----------|--------------------|---------|
-| `SearxEngineTooManyRequests` | ~1 hour | HTTP 429 |
-| `SearxEngineAccessDenied` | ~1 day | 403 / access denied |
-| `SearxEngineCaptcha` | ~1 day | Google `/sorry/` or reCAPTCHA |
-| `recaptcha_SearxEngineCaptcha` | ~7 days | Google reCAPTCHA |
-| `cf_SearxEngineCaptcha` | ~15 days | Cloudflare edge CAPTCHA |
-| ordinary errors (timeout/parse) | 5–120 s | `ban_time_on_fail`/`max_ban_time_on_fail` |
-
-The values are `search.suspended_times` (+ `ban_time_on_fail`) in
-`settings.yml`. An engine that returns **0 results without an error** is *not*
-suspended — that is the separate "selector drift" problem (below).
-
-## Two failure modes
-
-| Mode | Symptom | Root cause | Fix family |
-|------|---------|-----------|-----------|
-| **CAPTCHA / 429 / access-denied** | engine flips `unresponsive`, then `suspended` for hours | blocked from the shared VPN exit IP | routing/proxies + request tuning (see below) |
-| **Selector drift** | engine returns 0 results, no error banner | Google renamed its minified CSS classes the engine's hardcoded XPath looks for | engine config (duplicate engine / JSON feed) |
-
-Most "Google is broken on a VPN host" cases are a **mix of both**.
-
-## Fix A — Engine config (settings.yml `engines:`)
-
-The current tracked policy enables Bing and DuckDuckGo for default general
-search, gives each weight 2, disables pagination, and allows 5 seconds for the
-VPN path. CAPTCHA/429-prone Google, Brave, and Startpage families are retained
-but disabled from default aggregation. This limits fan-out while preserving
-explicit/on-demand selection.
-
-Important: SearXNG's engines are configured in the **`engines:` list inside
-`settings.yml`**, not a separate `engines.yml` file. With
-`use_default_settings: true` (our setting) your `engines:` list is *merged* over
-the built-in engines by `name` — you do **not** set `use_default_settings: false`.
-
-To add, override, disable, or prune:
-
-```yaml
-# shared/searxng/settings.yml  (appended to the existing file)
-use_default_settings: true
-engines:
-  # prune a broken engine entirely
-  remove:
-    - # (example) wikipedia   # drop an engine that never recovers
-
-  # override + duplicate the Google engines so one survives the other's suspension
-  - name: google
-    weight: 3
-    disabled: false
-    timeout: 5.0                 # wait longer over the tunnel (global default is 2.0s)
-    retry_on_http_error: [403, 429]
-    display_error_messages: true
-
-  - name: google2                # fresh engine, same module, survives google's CAPTCHA ban
-    engine: google
-    categories: general
-    weight: 2
-    timeout: 5.0
-    disabled: false
-```
-
-Key per-engine keys (each overwrites the engine-module default; see
-`settings_engines.html`):
-
-| Key | Effect |
-|-----|--------|
-| `name` | merge key (lowercase, no underscore) |
-| `engine` | Python module (`google`, `json_engine`, `xpath`) |
-| `categories` | categories it serves (also a comma string) |
-| `timeout` | per-engine HTTP timeout, overrides `outgoing.request_timeout` |
-| `weight` | ranks results higher |
-| `disabled` | off by default, selectable in /preferences |
-| `inactive` | `disabled` **and** removed from the UI — use to fully drop a broken engine |
-| `retry_on_http_error` | retry on status, e.g. `true`, `403`, `[403, 429]` |
-| `display_error_messages` | show/hide the engine's error banner |
-| `proxies` | per-engine egress override (see Fix C) |
-| `language` | force region, e.g. `en` |
-
-**Best duplicate trick:** re-declare `google` under a new `name` in a
-*different* category. When the primary `google` gets CAPTCHA'd it suspends, but
-`google2` still returns results, and vice-versa.
-
-**Parser caveat (read before fiddling with `parsers`):** the modern `google`,
-`duckduckgo`, and `brave` engines ship a **custom `response()`** with hardcoded
-XPath and *ignore* the generic `parsers`/`p_parsers` table. So `parsers:
-[scrapy_html]` under `name: google` mostly does nothing. To make Google
-robust against selector drift, either update the upstream `google.py`
-selectors (upgrade SearXNG) or run a **`json_engine`** against a JSON feed.
-Verify `parsers` consumption on the pinned version before depending on it.
-
-**JSON-feed fallback:** decouple parsing from Google's HTML churn.
-
-```yaml
-engines:
-  - name: google-json
-    engine: json_engine
-    categories: general
-    paging: true
-    search_url: 'https://www.googleapis.com/customsearch/v1?key={api_key}&q={query}&cx={cx}'
-    results_query: results
-    title_query: title
-    url_query: link
-    content_query: snippet
-```
-
-## Fix B — Request tuning (settings.yml `outgoing:`)
-
-SearXNG already rotates a **random browser User-Agent per outgoing request**
-(`gen_useragent()` in `online.py`) — there is **no native `useragents:` mapping
-key**. The blogs that point at a `useragents:` table describe the old
-`asciimoo/searx`; current SearXNG rotates automatically. Keep relying on it.
-
-Tune the `outgoing` block and the suspend model to survive load:
-
-```yaml
-outgoing:
-  request_timeout: 4.0             # current Dankclaw value; modest VPN-latency allowance
-  max_request_timeout: 12.0
-  pool_connections: 100
-  pool_maxsize: 20
-  enable_http2: true
-  retries: 1                       # avoid amplifying 429/CAPTCHA responses from one exit IP
-  useragent_suffix: "ops@yourhomelab.example"   # so a per-engine blocker can reach you
-
-search:
-  ban_time_on_fail: 10
-  max_ban_time_on_fail: 180
-  suspended_times:
-    SearxEngineTooManyRequests: 300     # 429 → 5 min before it retries (tune down to recover faster)
-    SearxEngineCaptcha: 7200            # CAPTCHA → 2h before it retries
-```
-
-Notes:
-- **Auto-suspend is the real "rate limit".** The old per-engine `rate_limit` /
-  `time_between_requests` sleep field does **not** exist in current SearXNG —
-  that is a historical-searx concept. The functional equivalent is the suspend
-  schedule above (widen to back off before blocking, narrow to recover faster).
-- `request_timeout` interacts with the suspend model: too tight → engines look
-  "unresponsive"; too loose → slow pages. Dankclaw uses 4 seconds globally and
-  12 seconds as the cap; raise it only when measurements show real upstream
-  latency rather than 429/CAPTCHA responses.
-- Keep retries low: retrying a 429/CAPTCHA through the same VPN exit can make the
-  provider's rate limit worse. A retry count of 1 is a bounded compromise for a
-  transient network failure.
-- `safe_search: 0` when diagnosing — a too-strict global safe search can make an
-  engine return zero rows and *look* blocked.
-
-## Fix C — Routing & proxies (change which IP the upstreams exit as)
-
-Two independent levers: SearXNG's own upstream `proxies`, and the Gluetun VPN
-strategy. The decisive fact for our topology: a proxy that runs **inside the
-same Gluetun netns** exits from the *same* single IP, so round-robin among
-same-IP proxies **does not** unblock. The proxy must egress from a **different,
-clean IP**.
-
-```mermaid
-graph LR
-    SE[SearXNG] -->|default| GW[Gluetun #1 netns] -->|exit| IP1[ProtonVPN IP A — gets blocked]
-    SE -->|per-engine proxies| GW2[Gluetun #2 / dedicated IP] -->|exit| IP2[ProtonVPN IP B — clean]
-```
-
-**Option 1 — per-engine `proxies` (cheapest, settings.yml only).** Point the
-heaviest/blocked engines (Google and Google-hosted) at a proxy that exits from a
-clean IP:
-
-```yaml
-engines:
-  - name: google
-    proxies:
-      http:
-        - http://gluetun-upstream:8080
-      https:
-        - http://gluetun-upstream:8080
-        - socks5h://gluetun-upstream:8388
-  - name: google news
-    engine: google_news
-    proxies:
-      https:
-        - http://gluetun-upstream:8080
-```
-
-`using_tor_proxy` follows the same per-engine/global pattern.
-
-**Option 2 — dedicated second Gluetun for SearXNG's upstreams.** Run a second
-Gluetun on a different ProtonVPN server/geo, publish its built-in HTTP proxy
-(`8080`, `HTTPPROXY=on`), and point the Google engines at it. SearXNG inbound
-(still on the main host/Caddy) stays put; only *upstream* requests re-route.
-Gives a second isolated exit IP and reputation pool.
-- **Cons:** a second ProtenVPN device slot, a new container, port publishing
-  moves to Gluetun, and **split-brain DNS** (below).
-
-**Option 3 — dedicated ProtonVPN IP.** A static, exclusive exit IP that other
-subscribers can't pollute. Requires a **ProtonVPN Business plan** as of the
-sources reviewed (the long-requested *personal*-plan dedicated IP is still only
-on uservoice — flag: verify your plan). Wire it in Gluetun by pinning the node
-via `SERVER_HOSTNAMES`.
-
-**Option 4 — rotate the exit.** List `SERVER_COUNTRIES`/`SERVER_CITIES`/
-`SERVER_HOSTNAMES` so Gluetun picks from several physical exit servers. Caveat:
-rotating the **shared** Gluetun rotates the exit for *all* attached
-containers. To rotate only SearXNG's upstreams, use Option 1 → a dedicated
-gateway.
-
-> ⚠️ **Split-brain DNS:** a container sharing Gluetun's netns
-> (`Network=gluetun.container`) loses Docker's internal service-name DNS — it
-> can't resolve siblings by name. Use a static IP or `extra_hosts`/`~/.config/
-> containers/systemd/<unit>.container` `Environment` to inject names into the
-> tunnel's `/etc/hosts`.
-
-## Observed Dankclaw baseline (2026-08-25)
-
-The read-only audit found:
-
-- SearXNG image: `ghcr.io/searxng/searxng:latest`; embedded version `1.0.0`.
-- Gluetun uses `VPN_SERVICE_PROVIDER=custom` with a static WireGuard profile;
-  no `SERVER_COUNTRIES`, `SERVER_CITIES`, `SERVER_REGIONS`, or
-  `SERVER_HOSTNAMES` filters are configured.
-- The observed exit was `149.22.88.19` (Seattle, Washington, US).
-- Bing and DuckDuckGo returned results. Mojeek returned no results without an
-  upstream error.
-- `google cse` and Brave were suspended for too many requests; Startpage was
-  suspended for CAPTCHA; Qwant returned CAPTCHA. SearXNG logs also recorded
-  Google's "unusual traffic from your network" response.
-- `/healthz` returned `200 OK`; the SearXNG service and Gluetun service were
-  healthy. This isolates the primary failures to upstream provider handling,
-  not application availability.
-
-This deployment returns the legacy array form for `unresponsive_engines`:
-
-```json
-"unresponsive_engines": [
-  ["brave", "Suspended: too many requests"],
-  ["startpage", "Suspended: CAPTCHA"]
-]
-```
-
-Do not assume the newer object form (`engine`, `error_type`, `suspended`)
-until the live response has been checked. The direct published port is the
-useful diagnostic path because it exercises the ProtonVPN exit. The Caddy
-hostname resolved to the WAN address during the audit and timed out from the
-audit host, so it was not used as the baseline path.
-
-## Fix D — Verify an engine is unblocked
-
-Three layers, cheapest first. All queries should hit the path production uses
-(the **published Gluetun port → ProtonVPN exit**, not a direct host port).
-
-**1. Isolate one engine with `/json`.** Enable `json` in `search.formats`
-(otherwise `format=json` → 403), then query a single engine:
-
-```bash
-BASE="http://192.168.0.143:8081"   # published port (ProtonVPN exit)
-curl -s "$BASE/search?q=test&engines=google&format=json" \
-  | jq -r '.unresponsive_engines[]? | select(.engine=="google")'
-```
-
-Inspect:
-- `content[].engine` — which engine produced each result.
-- `unresponsive_engines[]` — inspect the live shape first. On Dankclaw it is
-  `[engine, message]`, for example `["google cse", "Suspended: too many requests"]`.
-  Newer builds may return objects containing `engine`, `error_type`, and
-  `suspended`.
-- `timings[].engine` — per-engine latency when the running build emits timings.
-
-Full status check (bash + jq):
-
-```bash
-BASE="http://192.168.0.143:8081"
-for e in google bing duckduckgo brave startpage; do
-  curl -s "$BASE/search?q=test&engines=$e&format=json" \
-    | jq -r --arg e "$e" '
-        (.unresponsive_engines // [])
-        | map(select(.engine==$e))
-        | if length==0 then "\($e): OK"
-          else .[] | "\($e): UNRESPONSIVE (\(.error_type), suspended=\(.suspended))" end'
-done
-```
-
-**2. `/config` REST endpoint** lists every engine + enabled/disabled +
-categories — an inventory of what is live. `?engines=<name>` isolates a single
-provider; `/debug` adds engine diagnostics.
-
-**3. `check.searx.space`** — the hosted official checker probes every engine of
-your instance with a **headless Firefox** (catches Cloudflare/CAPTCHA blocks a
-plain HTTP probe misses) and posts results on searx.space. Add your instance via
-a GitHub issue in `searxng/searx-instances`, then allow `check.searx.space` IPs
-to reach the SearXNG port.
-
-Local self-test CLI (`kvch/searx-checker`):
-
-```bash
-python3 checker/checker.py http://192.168.0.143:8081
-# google.OK  bing.OK  gitlab......ERROR
-```
-
-Verify through the **published Gluetun port** (realistic ProtonVPN-exit block
-signal), not a direct host port (which uses the host LAN IP and can be
-misleading).
-
-## Recommended path for Dankclaw (ranked)
-
-1. **Measure individual engines** — use `/json` + `?engines=<name>` and several
-   query classes before enabling a bundled engine in default aggregation.
-2. **Keep default fan-out small** — prefer independent anonymous providers and
-   disable pagination for rate-sensitive engines.
-3. **Promote by evidence** — raise `weight` only for engines that return useful
-   results consistently; weight is not a health check.
-4. **Avoid credentials** — do not add API-backed engines unless the provider is
-   genuinely free, anonymous, and compatible with our privacy requirements.
-5. **Verify first** — `/json` + `?engines=google` and the searx-checker to see
-   exactly which engines are down and why. Confirm the running SearXNG version
-   (check whether `parsers`/`rate_limit` exist in the pinned build).
-6. **Use conservative request tuning** — Dankclaw now allows 5 seconds per
-   upstream request, caps it at 15 seconds, and retries once. This addresses
-   modest VPN latency without multiplying 429/CAPTCHA traffic. Adjust only
-   after measuring the resulting error and latency profile.
-7. **If Google still re-blocks** — Fix C Option 1: point the Google engines at
-   a dedicated-IP proxy (Option 2 dedicated second Gluetun if reputation
-   isolation is needed). Avoid the same-netns-proxy trap.
-
----
-
-## Sources
-
-- [SearXNG settings: engines](https://docs.searxng.org/admin/settings/settings_engines.html) — per-engine keys.
-- [SearXNG settings: outgoing](https://docs.searxng.org/admin/settings/settings_outgoing.html) — `request_timeout`, pool, `retries`, `proxies`, `source_ips`.
-- [SearXNG settings: search](https://docs.searxng.org/admin/settings/settings_search.html) — `suspended_times`, `ban_time_on_fail`.
-- [SearXNG settings (merge + remove/keep_only)](https://docs.searxng.org/admin/settings/settings.html) — `use_default_settings` semantics.
-- [Google engine docs](https://docs.searxng.org/dev/engines/online/google.html) — `detect_google_sorry`, WML/Nokia XML, `max_page`.
-- [json_engine](https://docs.searxng.org/dev/engines/json_engine.html) — `json` parser config.
-- [SearXNG exceptions](https://docs.searxng.org/src/searx.exceptions.html) — exception types and suspend settings.
-- [Google engine stops yielding results #5286](https://github.com/searxng/searxng/blob/master/searx/engines/google.py) — reproducible "Google unavailable".
-- [DuckDuckGo CAPTCHA #4824](https://github.com/searxng/searxng/issues/4824) — per-engine CAPTCHA signature.
-- [Google engine actively blocking IPs #2515](https://github.com/searxng/searxng/issues/2515)
-- [answer-captcha from server's IP](https://docs.searxng.org/admin/answer-captcha.html) — block is tied to the egress IP.
-- [SearXNG rate limits / 429 (SSD Nodes)](https://www.ssdnodes.com/learn/searxng-rate-limits-and-429-errors) — suspension table, `/stats/errors`.
-- [How search engines detect scrapers (Crawlbase)](https://crawlbase.com/blog/how-search-engines-detect-scrapers/) — IP reputation + rate + CAPTCHA mechanics.
-- [check.searx.space](https://github.com/searxng/searx-space) + [kvch/searx-checker](https://github.com/kvch/searx-checker) — the checkers.
-- [searx-instances — how to be checked](https://github.com/searxng/searx-instances) — add your instance + allow checker IPs.
-- [ProtonVPN — Dedicated IP addresses](https://proton.me/support/dedicated-ips) + [Benefits](https://proton.me/support/dedicated-ips) — IP-reputation reasoning; **Business plan** (flag: personal dedicated IP still on uservoice).
-- [Gluetun wiki — ProtonVPN provider](https://github.com/qdm12/gluetun-wiki/blob/main/setup/providers/protonvpn.md) — `SERVER_HOSTNAMES` narrowest-filter risk.
-- [Gluetun wiki — Built-in Proxies](https://deepwiki.com/qdm12/gluetun-wiki/4.5-built-in-proxies) — HTTP `8080` + Shadowsocks `8388`.
-- [Gluetun wiki — Container networking](https://deepwiki.com/qdm12/gluetun-wiki/6.5-container-networking) — netns sharing, port publishing on Gluetun.
-- [Gluetun inter-network DNS failures (Jacob Jangles)](https://jacobjangles.com/posts/fixing-gluetuns-inter-network-dns-failures-in-docker/) — split-brain DNS + fix.
-
-## See Also
-
-- [[SearXNG]] — the stack, routing, limiter, health
-- [[Gluetun]] — the VPN tunnel (where the exit IP comes from)
-- [[Dankclaw Stack]] — the service the SearXNG stack sits in
-- [[Service Research Brief]] — the research that produced these pages

docs/Dankclaw/SearXNG.md

@@ -1,151 +0,0 @@
-# SearXNG
-
-SearXNG is a privacy-respecting metasearch engine. It aggregates results from many upstream search engines into one result set and serves both an HTML UI and a JSON API for agents.
-
-**Host:** Dankclaw — **Hostname:** `search.franksdank.cloud` (Caddy) — **Docs:** https://docs.searxng.org
-
-## Facts (from research, primary source)
-
-| Item | Value |
-|------|-------|
-| Image | `ghcr.io/searxng/searxng:latest` (or DockerHub mirror) |
-| Port | 8081 (host, on Gluetun) → 8080 (container) |
-| Config | `/etc/searxng/settings.yml` (bind mount) |
-| Cache | Valkey (replaces Redis) — `valkey/valkey:9-alpine` |
-| Formats | `html`, `json` (agent API) |
-
-## VPN Routing (critical)
-
-All SearXNG outbound traffic exits through the Gluetun VPN container. The searxng unit sets `Network=gluetun.container`, which makes it reuse gluetun's network namespace:
-
-```ini
-[Container]
-ContainerName=searxng
-Network=gluetun.container
-```
-
-Port publishing happens on the **gluetun** unit, not on searxng. See [[Gluetun]] (KB Pattern 2 is the one used here).
-
-## Boot Persistence (Quadlet)
-
-The trio runs as quadlet units, not podman-compose, so it starts at boot. The unit files live in `searxng/quadlet/` and install to `~/.config/containers/systemd/`.
-
-| Unit file | Service | Role |
-|-----------|---------|------|
-| `searxng.network` | `searxng-network.service` | DNS-enabled bridge, subnet pinned to 10.89.2.0/24 |
-| `gluetun.container` | `gluetun.service` | WireGuard tunnel; publishes 8081:8080 |
-| `searxng-valkey.container` | `searxng-valkey.service` | Valkey cache, alias `valkey` |
-| `searxng.container` | `searxng.service` | Reuses gluetun's netns; publishes no port |
-
-gluetun and valkey join the searxng network. searxng reuses gluetun's netns and reaches valkey by name on that network. Each unit sets `WantedBy=default.target`. User linger is on, so the units start at boot.
-
-Deploy with `scripts/deploy-searxng-quadlet.sh dankclaw`. It copies the units, daemon-reloads, enables them, and starts them in dependency order.
-
-## settings.yml Keys
-
-```yaml
-use_default_settings: true
-server:
-  secret_key: "${SEARXNG_SECRET}"   # from .env
-  base_url: "${SEARXNG_BASE_URL}"  # resolves to http://192.168.0.143:8081/
-  limiter: true
-  method: "POST"
-
-search:
-  default_lang: "en"
-  formats:
-    - html
-    - json
-
-valkey:
-  url: "valkey://searxng-valkey:6379/0"
-
-outgoing:
-  request_timeout: 5.0
-  max_request_timeout: 15.0
-  retries: 1                     # avoid amplifying upstream 429/CAPTCHA responses
-
-# Anonymous, bounded default aggregation. Disabled engines remain configured
-# for explicit/on-demand selection; see SearXNG Providers for the policy.
-engines:
-  - name: bing
-    disabled: false
-    weight: 2
-    paging: false
-    timeout: 5.0
-  - name: duckduckgo
-    disabled: false
-    weight: 2
-    paging: false
-    timeout: 5.0
-  - name: google cse
-    disabled: true
-  - name: brave
-    disabled: true
-  - name: startpage
-    disabled: true
-```
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/etc/searxng` | `/opt/searxng/config` |
-| `/etc/searxng/valkey` | (valkey container data) |
-
-## Healthcheck
-
-SearXNG responds on `/healthz` with 200 when ready.
-
-## Trusted Group (limiter.toml)
-
-The trusted group lives in `shared/searxng/limiter.toml`, not in `settings.yml`. The `settings.yml` file only turns the limiter on. The file deploys to `/opt/searxng/config/limiter.toml`, mounted at `/etc/searxng/limiter.toml`.
-
-Two lists control access:
-
-- `pass_ip` — an IP in this list gets unrestricted access and bypasses all bot detection.
-- `trusted_proxies` — if the request IP is in this list, SearXNG reads the real client IP from `X-Forwarded-For` (set by Caddy).
-
-The `pass_ip` list holds loopback (`127.0.0.1/32`, `::1`), the whole homelab LAN (`192.168.0.0/24`), and the host's private podman/slirp ranges (`10.88.0.0/16`, `10.89.0.0/16`). An IP in `pass_ip` skips every bot-detection check, including the rate limiter. `deploy.sh` copies the file into place on each deploy.
-
-Why so many ranges: rootless podman strips the client IP on published-port hits. A request to the published port reaches the container with a slirp/bridge source (`10.88.x`/`10.89.x`), never the real client IP. Only Caddy's `X-Forwarded-For` restores it. So a local subagent that hits the port directly must present a `pass_ip` source, or it is treated as an untrusted client. All these ranges are reachable only from the host or the LAN, never the internet.
-
-The rate limiter's JSON cap is `API_MAX = 4` requests per IP per hour. The constant is hardcoded in Python (`searx/botdetection/ip_limit.py`). It cannot be raised from `limiter.toml`. `pass_ip` is the only bypass. This is what lets 4 parallel subagents poll the JSON API at full rate without a 429.
-
-Reach SearXNG through Caddy (`search.dankclaw.local`) for browser use. A local subagent can hit the published port directly because its slirp source is now trusted. The searxng subnet 10.89.2.0/24 is in `trusted_proxies`, so Caddy's `X-Forwarded-For` is honored. The subnet is pinned in `searxng.network`, so it stays stable across reboots.
-
-## Unblocking Google & Search Providers
-
-The current request policy allows a modest amount of VPN latency (`4.0s` per
-request, `12.0s` maximum) while limiting retries to one. This can recover slow
-providers, but it cannot fix an upstream CAPTCHA or 429 caused by the shared
-VPN exit IP. SearXNG queries all ~50 upstream engines from one Gluetun VPN exit
-IP. The rate-sensitive engines (Google, DuckDuckGo) then answer with reCAPTCHA / 429.
-The default provider policy therefore favors anonymous Bing and DuckDuckGo,
-disables observed CAPTCHA/429 providers from normal aggregation, and disables
-pagination to reduce upstream request volume. SearXNG **suspends** offending
-engines for hours to days, which is what shows up as "Google went unavailable".
-This is upstream, not local.
-
-Fixes (cheapest first): tune the `outgoing` block (`request_timeout`,
-`retry_on_http_error`, `suspended_times`), duplicate the `google` engine under a
-fresh `name` so a CAPTCHA ban on one leaves the other returning results, and —
-if Google still re-blocks — route the Google engines at a clean-IP proxy via
-per-engine `proxies` in `settings.yml`. See [[SearXNG Providers]] for the full
-root cause, the suspend-model table, the `engines:` config, and the
-`/json` verification recipe.
-
-### Gotchas
-
-- The JSON API rate cap (4 per 3600s per IP) is hardcoded in Python, not settable in `limiter.toml`. The only bypass is `pass_ip`. Local subagents are covered because their slirp/loopback/LAN sources are in `pass_ip`.
-- The limiter stays armed for untrusted sources (a spoofed public IP still 429s). Do not disable it. Widen only `pass_ip` for sources you trust on this host.
-- The config file is root-owned (uid of the rootless podman namespace). Update it with a one-off `sudo cp`, then run `systemctl --user restart searxng.service` to reload.
-- `base_url` must match the public address or proxy rewrites break.
-- Keep the valkey cache container on the same podman network as searxng.
-
-## See Also
-
-- [[Dankclaw Stack]]
-- [[Gluetun]] — the VPN container
-- [[SearXNG Providers]] — unblocking Google & other upstream engines (VPN exit IP, engine config, proxies, verification)
-- Knowledge base: [[Gluetun]], [[SearXNG]], [[SearXNG Providers]], [[Hermes]]

docs/Dankclaw/Valkey.md

@@ -1,55 +0,0 @@
-# Valkey
-
-Valkey is the cache and limiter store for SearXNG. It replaces Redis as the default in the official SearXNG deployment.
-
-**Host:** Dankclaw — **Docs:** https://valkey.io
-
-## Facts (from research)
-
-| Item | Value |
-|------|-------|
-| Image | `valkey/valkey:9-alpine` |
-| Port | 6379 (internal, on the SearXNG network) |
-| Role | SearXNG limiter + result cache |
-
-## Why Valkey
-
-The official SearXNG docker-compose ships a `searxng-valkey` container. Valkey is the Redis fork under the Linux Foundation umbrella; the SearXNG project moved to it as the default cache.
-
-## Configuration
-
-The valkey unit is `searxng-valkey.container` (see [[SearXNG]] for the trio). It joins the searxng network and is reachable by the name `searxng-valkey` (container name) and the alias `valkey`:
-
-```ini
-[Container]
-ContainerName=searxng-valkey
-Network=searxng.network
-NetworkAlias=valkey
-Image=docker.io/valkey/valkey:9-alpine
-Volume=/opt/searxng/valkey:/data
-```
-
-SearXNG connects to it via `settings.yml`:
-
-```yaml
-valkey:
-  url: "valkey://searxng-valkey:6379/0"
-```
-
-It sets `WantedBy=default.target`. User linger is on, so it starts at boot.
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/data` | `/opt/searxng/valkey` |
-
-## Gotchas
-
-- The limiter relies on Valkey; without it SearXNG falls back to in-memory state and rate limiting is per-restart.
-- Keep Valkey on the same podman network as SearXNG.
-
-## See Also
-
-- [[SearXNG]] — the consumer
-- [[Dankclaw Stack]]

docs/Dankserver System Specs.md

@@ -0,0 +1,68 @@
+# Dankserver System Specs
+
+## Server Overview
+
+Dankserver is an NVIDIA GPU server for vLLM inference. The host runs Ubuntu 24.04 with an RTX 3090 and 62 GB of system RAM.
+
+## Hardware Specifications
+
+### CPU
+
+| Property | Value |
+|----------|-------|
+| Architecture | x86_64 |
+| Vendor | AuthenticAMD |
+| Cores | 8 (1 socket) |
+| NUMA Nodes | 1 (node 0, 63503 MB) |
+
+### GPU
+
+| Property | Value |
+|----------|-------|
+| Model | NVIDIA GeForce RTX 3090 |
+| VRAM | 24576 MiB (24 GB) |
+| Driver | 595.84 |
+| CUDA Version | 13.2 |
+| CUDA Toolkit | 12.0 (nvcc) |
+
+### Memory
+
+| Property | Value |
+|----------|-------|
+| System RAM | 62 GiB |
+| Swap | 127 GiB |
+| /dev/shm | 32 GiB |
+
+### Storage
+
+| Filesystem | Size | Used | Avail | Use% |
+|------------|------|------|-------|------|
+| / (nvme0n1p2) | 915G | 849G | 20G | 98% |
+| /mnt/entropy (dm-0) | 91T | 86T | 977G | 99% |
+
+### OS and Software
+
+| Property | Value |
+|----------|-------|
+| OS | Ubuntu 24.04.4 LTS (Noble Numbat) |
+| Kernel | 7.0.0-28-generic |
+| Python | 3.12.3 |
+
+### Active Services
+
+A llama-server process occupies approximately 20 GB of the RTX 3090 VRAM.
+
+## VLLM Relevance
+
+- 24 GB VRAM limits model size. Use FP8 quantization for large models.
+- 62 GB system RAM provides ample swap space for KV cache offloading.
+- 127 GB swap supports large `--swap-space` values for CPU KV cache paging.
+- Root disk has only 20 GB free. Store models on /mnt/entropy.
+- CUDA 12.0 and Python 3.12.3 are compatible with vLLM.
+
+## See Also
+
+- [[VLLM Overview]]
+- [[Hybrid CPU GPU Processing Dankserver]]
+- [[RAM CPU Offloading in VLLM]]
+- [[Running Qwen3.6 35B A3B FP8 on Dankserver]]

docs/Dankserver/ARR-Stack.md

@@ -1,369 +0,0 @@
-# *arr Stack
-
-The *arr suite is a collection of automated media management tools for Usenet and BitTorrent. For Dankserver — which runs Copyparty, Calibre-Web, RomM, and Jellyfin — the *arr stack provides automatic downloading and library management that Copyparty alone does not cover: quality profile enforcement, series/movie discovery, metadata enrichment, and automated file organization.
-
-The core chain is: **Prowlarr** (indexer API aggregator) → **Sonarr/Radarr/Lidarr/Readarr** (media-specific managers) → **qBittorrent** (download client) → **Jellyfin** (media server scan → auto-refresh).
-
-## Service Overview
-
-| Service | Role | Image | Port | Min RAM |
-|---------|------|-------|------|---------|
-| Prowlarr | Indexer management | `lscr.io/linuxserver/prowlarr:latest` | 9696 | 256 MB |
-| Radarr | Movie management | `lscr.io/linuxserver/radarr:latest` | 7878 | 512 MB |
-| Sonarr | TV show management | `lscr.io/linuxserver/sonarr:latest` | 8989 | 512 MB |
-| Lidarr | Music management | `lscr.io/linuxserver/lidarr:latest` | 8686 | 256 MB |
-| Readarr | Ebook/audiobook management | `lscr.io/linuxserver/readarr:latest` | 8787 | 256 MB |
-| qBittorrent | Torrent download client | `lscr.io/linuxserver/qbittorrent:latest` | 8080 | 512 MB |
-| Jackett | Indexer proxy (optional) | `lscr.io/linuxserver/jackett:latest` | 9117 | 256 MB |
-| Whisparr | NSFW movie management (optional) | `lscr.io/linuxserver/whisparr:latest` | 6969 | 256 MB |
-
-**Minimum total RAM:** ~3 GB for full stack (headroom recommended).
-
-## Architecture
-
-```mermaid
-graph LR
-    I[Indexers<br/>Torrents, Usenet] --> P[Prowlarr<br/>Indexer Manager]
-    P --> S[Sonarr<br/>TV]
-    P --> R[Radarr<br/>Movies]
-    P --> L[Lidarr<br/>Music]
-    P --> Rd[Readarr<br/>Books]
-    S --> Q[qBittorrent]
-    R --> Q
-    L --> Q
-    Rd --> Q
-    Q --> M1[/tv/]
-    Q --> M2[/movies/]
-    Q --> M3[/music/]
-    Q --> M4[/books/]
-    M1 --> J[Jellyfin]
-    M2 --> J
-    M3 --> J
-    M4 --> CW[Calibre-Web]
-```
-
-## Docker Compose
-
-```yaml
-networks:
-  appnet:
-    driver: bridge
-
-services:
-  prowlarr:
-    image: lscr.io/linuxserver/prowlarr:latest
-    container_name: prowlarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/prowlarr/config:/config
-      - /mnt/media/downloads:/downloads:ro
-    ports:
-      - "9696:9696"
-    networks: [appnet]
-
-  radarr:
-    image: lscr.io/linuxserver/radarr:latest
-    container_name: radarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/radarr/config:/config
-      - /mnt/media/movies:/movies
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "7878:7878"
-    networks: [appnet]
-
-  sonarr:
-    image: lscr.io/linuxserver/sonarr:latest
-    container_name: sonarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/sonarr/config:/config
-      - /mnt/media/tv:/tv
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "8989:8989"
-    networks: [appnet]
-
-  lidarr:
-    image: lscr.io/linuxserver/lidarr:latest
-    container_name: lidarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/lidarr/config:/config
-      - /mnt/media/music:/music
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "8686:8686"
-    networks: [appnet]
-
-  readarr:
-    image: lscr.io/linuxserver/readarr:latest
-    container_name: readarr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/readarr/config:/config
-      - /mnt/media/books:/books
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "8787:8787"
-    networks: [appnet]
-
-  qbittorrent:
-    image: lscr.io/linuxserver/qbittorrent:latest
-    container_name: qbittorrent
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/qbittorrent/config:/config
-      - /mnt/media/downloads:/downloads
-      - /mnt/media/downloads/incomplete:/downloads/incomplete
-    ports:
-      - "8080:8080"
-    networks: [appnet]
-
-  # Optional: Jackett for niche indexers
-  jackett:
-    image: lscr.io/linuxserver/jackett:latest
-    container_name: jackett
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/jackett/config:/config
-      - /opt/jackett/torrences:/torrences
-    ports:
-      - "9117:9117"
-    networks: [appnet]
-
-  # Optional: Whisparr for adult content
-  whisparr:
-    image: lscr.io/linuxserver/whisparr:latest
-    container_name: whisparr
-    restart: unless-stopped
-    environment:
-      - PUID=1000
-      - PGID=1000
-      - TZ=America/New_York
-      - UMASK_SET=022
-    volumes:
-      - /opt/whisparr/config:/config
-      - /mnt/media/whisparr-movies:/whisparr-movies
-      - /mnt/media/downloads:/downloads
-    ports:
-      - "6969:6969"
-    networks: [appnet]
-```
-
-## Caddy Configuration
-
-```caddy
-radarr.dankserver.local {
-    reverse_proxy localhost:7878
-}
-
-sonarr.dankserver.local {
-    reverse_proxy localhost:8989
-}
-
-lidarr.dankserver.local {
-    reverse_proxy localhost:8686
-}
-
-readarr.dankserver.local {
-    reverse_proxy localhost:8787
-}
-
-prowlarr.dankserver.local {
-    reverse_proxy localhost:9696
-}
-
-qbittorrent.dankserver.local {
-    reverse_proxy localhost:8080
-}
-
-# Optional: Jackett
-jackett.dankserver.local {
-    reverse_proxy localhost:9117
-}
-
-# Optional: Whisparr
-whisparr.dankserver.local {
-    reverse_proxy localhost:6969
-}
-```
-
-## Configuration Steps
-
-### Phase 1: Download Client
-
-1. Deploy qBittorrent first (or Transmission).
-2. In the qBittorrent Web UI, set **No auth mode** for LAN (Settings → Web UI).
-3. Create download categories: `movies`, `tv`, `music`, `books`.
-4. Change the default Web UI password.
-
-### Phase 2: Indexer Management
-
-1. Deploy Prowlarr.
-2. Add indexers (3–5 recommended for testing). Use the **Test** button on each.
-3. Add qBittorrent as a download client in Prowlarr.
-4. Verify search from the Prowlarr web UI.
-
-### Phase 3: Arr Apps (one at a time)
-
-1. Deploy Sonarr first (TV shows are the most common use case).
-2. Add Prowlarr as the indexer source (Settings → Indexers → Add Prowlarr).
-3. Add qBittorrent as the download client with categories.
-4. Add a test series. Wait for the episode to download. Verify in `/tv/`.
-5. Repeat for Radarr, Lidarr, Readarr.
-
-### Phase 4: Media Server Integration
-
-1. Configure Jellyfin libraries pointing to `/movies`, `/tv`, `/music`, `/books`.
-2. Set **On Download** notifications in each arr app → Jellyfin webhook URL.
-3. Test end-to-end: Add a movie in Radarr → download → verify in Jellyfin.
-
-## Quality Profiles
-
-Each arr app uses a hierarchical quality system:
-
-1. **Quality Profile** — Defines tiers (e.g., 2160p → 1080p BluRay → 720p).
-2. **Cutoff** — The quality target. Downloads at or above cutoff are accepted.
-3. **Custom Formats** — Additional filters (e.g., "Has HDR", "Is Director's Cut").
-4. **Search on Grab** — When content becomes available, all indexers are queried.
-
-### Radarr (Movies)
-
-```
-Profile: Movies 4K
-  Tiers:
-    1. 2160p BluRay REMUX
-    2. 2160p BluRay
-    3. 2160p WEB-DL
-    4. 1080p BluRay
-    5. 1080p WEB-DL
-    6. 720p
-  Cutoff: 1080p BluRay
-```
-
-### Sonarr (TV)
-
-```
-Profile: TV - Standard
-  Tiers: 2160p → 1080p → 720p
-  Min Size: 500 MB
-  Max Size: 50 GB
-  Cutoff: 1080p
-  Language Profile: English audio preferred
-```
-
-### Lidarr (Music)
-
-```
-Profile: Music - Standard
-  Tiers: Lossless (FLAC) → 320 kbps → 192 kbps
-  Cutoff: 320 kbps (or Lossless)
-```
-
-## Metadata Sources
-
-| App | Primary Source | Secondary |
-|-----|---------------|-----------|
-| Radarr | TMDB | TheMovieDB API |
-| Sonarr | TMDB | TVDB |
-| Lidarr | MusicBrainz | — |
-| Readarr | Goodreads, StoryGraph | — |
-| Whisparr | XH | TMDB |
-
-## *arr Stack vs Copyparty
-
-| Feature | Copyparty | *arr Stack |
-|---------|-----------|------------|
-| Auto-download from torrents | Basic (RSS + scripts) | Native, integrated |
-| Quality profiles | No | Multi-tier, custom formats |
-| Series/movie tracking | No | Watch lists, air dates |
-| Manual library management | Yes | Yes, metadata-driven |
-| Multi-user watchlists | No | Yes |
-| Complexity | Low | Medium |
-| Resource usage | Minimal | Moderate |
-
-Use Copyparty for curated file collections. Use *arr for automatic downloads. Use both together for maximum functionality — point *arr download directories at Copyparty's watched folders.
-
-## Disk Layout
-
-```
-/mnt/media/
-├── movies/         → Radarr root
-├── tv/             → Sonarr root
-├── music/          → Lidarr root
-├── books/          → Readarr root
-├── whisparr-movies/ → Whisparr root
-├── downloads/      → qBittorrent download dir
-│   ├── movies/
-│   ├── tv/
-│   ├── music/
-│   └── books/
-└── incomplete/     → qBittorrent incomplete downloads
-```
-
-## Gotchas
-
-- **Timezone mismatch:** All containers should use the same TZ. Sonarr's scheduler is timezone-sensitive.
-- **File permissions:** Set `umask=022` in qBittorrent. Ensure arr apps can read/move downloaded files.
-- **Network conflicts:** *arr apps use many ports. Check for conflicts with existing Dankserver services.
-- **Prowlarr API key rotation:** If rotated, update it in every arr app.
-- **qBittorrent Web UI password:** Change default on first run.
-- **Readarr stability:** Readarr is still in development. Do not rely on it for mission-critical collections.
-- **Whisparr separate Jellyfin library:** Keep adult content separate from regular media.
-- **RSS feed in qBittorrent:** Can auto-add torrents matching a pattern. Useful for podcast-like series.
-
-## Best Practices
-
-1. Use tags to route content to specific download clients or categories.
-2. Enable the Blocklist in arr apps to prevent re-downloading failed torrents.
-3. Set download limits in qBittorrent to avoid bandwidth saturation.
-4. Back up arr configs regularly (they are small SQLite DBs + JSON).
-5. Monitor Prowlarr indexer health — broken indexers hide new content.
-6. Start with default quality profiles. Add custom formats gradually.
-7. Back up arr configs: `./radarr/config`, `./sonarr/config`, etc.
-
-## See Also
-
-- [[Dankserver Stack]]
-- [[Jellyfin]] — media server that consumes *arr downloads
-- [[Copyparty]] — alternative file viewer
-- [[Calibre-Web]] — manual ebook management (Readarr auto-downloads into the same library)
-- [Sonarr Wiki](https://wiki.servarr.com/sonarr)
-- [Radarr Wiki](https://wiki.servarr.com/radarr)

docs/Dankserver/Calibre-Web.md

@@ -1,93 +0,0 @@
-# Calibre-Web
-
-Calibre-Web is a web application that browses, reads, and downloads eBooks from an existing Calibre library database. It serves as the web frontend for the Calibre e-book manager.
-
-**Host:** Dankserver — **Hostname:** `books.dankserver.local` — **Docs:** https://github.com/linuxserver/docker-calibre-web
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Latest version | 0.6.25-ls365 (live); 0.6.27-ls397 upstream |
-| Image | `lscr.io/linuxserver/calibre-web:latest` (live: 0.6.25-ls365) |
-| Base OS | Ubuntu Noble (24.04 LTS, since 2024-12-05) |
-| Port | 8083 (host) → 8083 (container) |
-| Default login | `admin` / `admin123` |
-| Docker mod | `linuxserver/mods:universal-calibre` (ebook conversion, x86-64 only) |
-| Volumes | `/config` (internal database + config), `/books` (existing Calibre library) |
-| Extra tools | `unrar` at `/usr/bin/unrar`, `kepubify` at `/usr/bin/kepubify` (pre-configured since 2025-01-07) |
-
-## CVEs Fixed in Latest Version
-
-| CVE | Issue |
-|-----|-------|
-| CVE-2025-7404 | External binary names (e.g. `unrar`) restricted to prevent RCE |
-| — | XSS in book comments imported via Calibre, now sanitized |
-
-Additional packages added: `libxfixes3` and `libasound2t64` to support epub-to-pdf conversion. The `--no-sandbox` flag is applied to qtwebengine to prevent crashes in Docker.
-
-## Configuration
-
-### Environment Variables
-
-| Variable | Value | Description |
-|----------|-------|-------------|
-| `PUID` | `1000` | User ID |
-| `PGID` | `1000` | Group ID |
-| `TZ` | `Etc/UTC` | Timezone |
-| `DOCKER_MODS` | `linuxserver/mods:universal-calibre` | Optional: enables ebook conversion tools |
-| `OAUTHLIB_RELAX_TOKEN_SCOPE` | `1` | Required for Google OAuth |
-
-### Setup Steps
-
-1. Point the container at the Calibre library directory (set the `/books` volume).
-2. Log in with `admin` / `admin123` and change the password immediately.
-3. In Basic Configuration → Calibre Database → set Location of Calibre database to `/books`.
-4. In Basic Configuration → External Binaries → set the directory to `/usr/bin/` for Calibre-Web >= 0.6.22.
-5. **kepubify is pre-configured** since version 0.6.27. No need to set the kepubify path manually.
-6. Set up Google OAuth if desired (requires `OAUTHLIB_RELAX_TOKEN_SCOPE=1`).
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/config` | `/opt/calibre-web/config` |
-| `/books` | The existing Calibre library directory (user-entered during setup) |
-
-## Docker Compose
-
-```yaml
-services:
-  calibre-web:
-    image: lscr.io/linuxserver/calibre-web:0.6.27-ls397
-    container_name: calibre-web
-    restart: unless-stopped
-    environment:
-      PUID: "1000"
-      PGID: "1000"
-      TZ: Etc/UTC
-      OAUTHLIB_RELAX_TOKEN_SCOPE: "1"
-    ports:
-      - "8083:8083"
-    volumes:
-      - /opt/calibre-web/config:/config
-      - /opt/calibre-web/books:/books
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8083/"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-```
-
-## Gotchas
-
-- **Wrong database path:** Login appears to succeed but the library is empty. Verify the Calibre database path in the admin page.
-- **Ebook conversion:** Requires the `universal-calibre` Docker mod AND the external binary path set to `/usr/bin/` (directory, not full path to `ebook-convert`).
-- **Google OAuth:** Set `OAUTHLIB_RELAX_TOKEN_SCOPE=1` or OAuth login fails.
-- **calibre 9.9.0 content server security fix:** A bug since v6.16.0 allows any user with server access to read non-Calibre files on the host. Upgrade to 9.9.0+ if serving content to untrusted users.
-
-## See Also
-
-- [[Dankserver Stack]]
-- [[Calibre]] — the underlying library engine (host tool, not a container)
-- [[ARR-Stack]] — Readarr for automatic ebook downloads into the Calibre library

docs/Dankserver/Calibre.md

@@ -1,69 +0,0 @@
-# Calibre
-
-Calibre is the e-book library manager that backs Calibre-Web. It manages metadata, formats, and conversions for the library that Calibre-Web serves.
-
-**Host:** Dankserver (library directory on host) — **Docs:** https://calibre-ebook.com
-
-## Role in this Stack
-
-Calibre itself is a desktop/GUI application, not a container service. This stack runs **Calibre-Web** in a container and points it at an existing Calibre library directory on the Dankserver host.
-
-Calibre is used to:
-- Import books into the library (drag-and-drop or CLI `calibredb`)
-- Edit metadata (author, title, series, tags, cover art)
-- Convert between formats (EPUB → MOBI, MOBI → EPUB, etc.)
-- Manage collections and reading lists
-
-The actual library files (`.epub`, `.mobi`, `.pdf`) and the Calibre database (`.db` file) live on the host filesystem. Calibre-Web reads and writes them through the mounted volume.
-
-## Current Version
-
-Calibre 9.2.1 is the version on the live host (9.9.0 is the upstream latest). Major features include:
-- Bookshelf view
-- Opt-in AI features
-- Content server security fix (CVE-2025-7404 equivalent): a bug since v6.16.0 allows any user with server access to read non-Calibre files on the host. Upgrade to 9.9.0+ if serving content to untrusted users.
-
-## Library Directory
-
-| Purpose | Host path (convention) |
-|---------|------------------------|
-| Calibre library | `/opt/calibre-web/books` (bind-mounted to `/books` in the Calibre-Web container) |
-
-## What the Container Needs
-
-- Read/write access to the library directory. Calibre-Web edits metadata through the database file.
-- Ownership matching `PUID=1000` / `PGID=1000` (the `dank` user on Dankserver).
-
-## Optional: Calibre Content Server
-
-Calibre includes a built-in HTTP content server (`calibre-server`) that can serve the library via OPDS feeds and a mobile-friendly web interface. This enables:
-- OPDS feeds for mobile reading apps (FBReader, Moon+ Reader)
-- Mobile-optimized UI via the `--mobile` flag
-- Direct Kindle push
-
-To run as a container:
-
-```yaml
-services:
-  calibre-server:
-    image: lscr.io/linuxserver/calibre:latest
-    container_name: calibre-server
-    restart: unless-stopped
-    environment:
-      PUID: "1000"
-      PGID: "1000"
-      TZ: Etc/UTC
-    command: calibre-server --daemonize --port 8080 --enable-auth --url-prefix /calibre /books
-    volumes:
-      - /opt/calibre-web/books:/books
-    ports:
-      - "8084:8080"
-```
-
-Note: The content server runs headless in `--daemonize` mode. It requires the full Calibre package, not a minimal server.
-
-## See Also
-
-- [[Calibre-Web]] — the containerized frontend
-- [[Dankserver Stack]]
-- [[ARR-Stack]] — Readarr for automatic ebook downloads

docs/Dankserver/Copyparty.md

@@ -1,106 +0,0 @@
-# Copyparty
-
-Copyparty is a lightweight file server and media streamer that runs as a single Python binary. It handles large payloads, media streaming, resumable uploads, directory browsing, and WebDAV from one process.
-
-**Host:** Dankserver — **Hostname:** `files.dankserver.local` — **Docs:** https://github.com/9001/copyparty
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Latest version | v1.20.21 (2026-08) |
-| Image | `ghcr.io/9001/copyparty:latest` (official Dockerfile in repo) |
-| Port | 3923 (host) → 3923 (container) |
-| Volume | bind mount the shared root directory |
-| Auth | Per-directory `.cpy` config files, HTTP basic auth, `dirkeys`, `filekeys` |
-| Streaming | HTTP range requests, HLS, WebDAV, WOPI (Office editing) |
-| Docker editions | `dj` (document + media), `iv` (image + video), `v` (video only) |
-
-## Configuration
-
-### Environment Variables
-
-| Variable | Description |
-|----------|-------------|
-| `COPAY_PASS` | Master password for the web UI |
-| `COPAY_MODE` | Set to `rw` for read-write shares |
-
-### Configuration File Options
-
-Copyparty reads its configuration from `initcfg` (or `config` in the `/data` directory). Key options:
-
-- `use-bwrap: n` — Disable bwrap sandboxing for ffmpeg (default in Docker). Set to `y` to enable.
-- `shadowing //NULL` — Full unmap of a volume from the filesystem.
-- Config env-var expansion: Use `${VAR}` syntax in `[global]` section for environment variable substitution.
-
-### Docker Editions
-
-Different editions bundle different libraries:
-
-| Edition | Bundled Libraries | Use Case |
-|---------|-------------------|----------|
-| `dj` | Document + image + video libraries | Full-featured server |
-| `iv` | Image + video libraries | Image and video focused |
-| `v` | Video only | Transcoding only, minimal footprint |
-
-The `iv` edition is no longer available for arm32/armv6. Use `dj` for maximum compatibility.
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/data` | `/opt/copyparty/data` |
-
-## Docker Compose
-
-```yaml
-services:
-  copyparty:
-    image: ghcr.io/9001/copyparty:latest
-    container_name: copyparty
-    restart: unless-stopped
-    environment:
-      COPAY_PASS: "${COPAY_PASS}"
-      COPAY_MODE: rw
-    ports:
-      - "3923:3923"
-    volumes:
-      - /opt/copyparty/data:/data
-    healthcheck:
-      test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3923/')"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-```
-
-## Healthcheck
-
-A simple HTTP check on port 3923 works. The root path returns HTTP 200 with the directory listing.
-
-## WOPI Integration
-
-Copyparty v1.20.9+ supports WOPI (Web Application Open Platform Interface) for editing Office documents in the web UI via Collabora Online. Enable with the `--wopi-accs` flag to limit WOPI access to specific paths.
-
-## Security Advisories
-
-| Version | CVE | Issue |
-|---------|-----|-------|
-| v1.20.19 | GHSA-phv8-wgjp-g4p9 | FTP upload outside volume |
-| v1.20.17 | GHSA-x5pq-m9p8-f4vx | dirkey/filekey confusion |
-| v1.20.9 | CVE-2026-27948 / GHSA-62cr-6wp5-q43h | XSS via file upload |
-
-Upgrade to v1.20.19+ to cover all known CVEs. The dirkey fix (v1.20.17) matters if using both `dirkeys` and `filekeys`.
-
-## Gotchas
-
-- **Large transfers:** Disable TLS inside the app and let Caddy terminate TLS. This avoids double encryption overhead.
-- **Dedicated share directory:** Do not give Copyparty the whole disk. Mount only the intended share path.
-- **Upload tokens scoped per folder:** Prevents accidental global write access.
-- **CSP nonces (v1.20.17+):** Security hardening for JavaScript execution. May break third-party JS plugins (`--js-browser` / `--html-head`).
-- **Custom thumbnail extractors (v1.20.19+):** New `thumbex` framework. Requires configuration in `initcfg`.
-
-## See Also
-
-- [[Dankserver Stack]]
-- [[Caddy]] — TLS termination and authentication in front
-- [[ARR-Stack]] — Copyparty as a viewer alongside automated download management

docs/Dankserver/Dankserver Stack.md

@@ -1,55 +0,0 @@
-# Dankserver Stack
-
-Dankserver (192.168.0.235) runs the media and library stack. Every service is reachable by hostname through the Caddy reverse proxy on the host.
-
-```mermaid
-graph LR
-    U[User] --> C[Caddy :80/:443]
-    C --> R[RomM]
-    C --> P[Copyparty]
-    C --> W[Calibre-Web]
-    C --> J[Jellyfin]
-    C --> S[Sonarr]
-    C --> D[Radarr]
-    C --> L[Lidarr]
-    C --> A[Readarr]
-    C --> O[Prowlarr]
-    W --> L2[Calibre library]
-```
-
-## Services
-
-| Service | Hostname | Port | Docs |
-|---------|----------|------|------|
-| RomM | romm.dankserver.local | 8085 | [[RomM]] |
-| Copyparty | files.dankserver.local | 3923 | [[Copyparty]] |
-| Calibre-Web | books.dankserver.local | 8083 | [[Calibre-Web]] |
-| Jellyfin | jellyfin.dankserver.local | 8096 | [[Jellyfin]] |
-| Sonarr | sonarr.dankserver.local | 8989 | [[ARR-Stack]] |
-| Radarr | radarr.dankserver.local | 7878 | [[ARR-Stack]] |
-| Lidarr | lidarr.dankserver.local | 8686 | [[ARR-Stack]] |
-| Readarr | readarr.dankserver.local | 8787 | [[ARR-Stack]] |
-| Prowlarr | prowlarr.dankserver.local | 9696 | [[ARR-Stack]] |
-| qBittorrent | qbittorrent.dankserver.local | 8080 | [[ARR-Stack]] |
-
-## Optional Services
-
-These services are included in the compose file but may be disabled.
-
-| Service | Hostname | Port | Docs |
-|---------|----------|------|------|
-| Jackett | jackett.dankserver.local | 9117 | [[ARR-Stack]] |
-| Whisparr | whisparr.dankserver.local | 6969 | [[ARR-Stack]] |
-
-## Removed Services
-
-The following services ran on Dankserver and are now retired from this stack:
-
-- OpenWebUI — moved to Dankclaw (see [[OpenWebUI]])
-- llama-swap — retired; no replacement
-- SearXNG — moved to Dankclaw (see [[SearXNG]])
-
-## See Also
-
-- [[index]] — repository index
-- [[Network Architecture]] — routing and proxy

docs/Dankserver/Entropy Volume.md

@@ -1,43 +0,0 @@
-# Entropy Volume
-
-The Dankserver entropy volume is a RAID-backed LUKS filesystem mounted at
-`/mnt/entropy`. The command-line helper reuses the host's desktop-integrated
-`udisks2`, `crypttab`, and `fstab` configuration; it does not store or handle
-the passphrase itself.
-
-## Command-line mount
-
-```bash
-/home/dank/serverAppConfigs/scripts/mount-entropy.sh
-```
-
-Run it as the normal desktop user (`dank`), not from a root shell. `udisksctl`
-will securely prompt for the LUKS passphrase. `sudo mount /mnt/entropy` then
-uses the existing `/etc/fstab` entry, preserving its options:
-
-```text
-nosuid,nodev
-```
-
-The existing mountpoint is owned by `dank:dankserver` with mode `0755`; the
-helper deliberately does not run `chmod` or `chown`, so it cannot silently
-change those permissions.
-
-## Device path
-
-The current storage chain is:
-
-```text
-/dev/md0 -> LUKS UUID cd86c728-459e-4f3a-bcc2-41ecf84685ec
-         -> ext4 label entropy
-         -> /mnt/entropy
-```
-
-The helper checks that `/dev/md0` exists and exits successfully if the volume
-is already mounted. It is intentionally specific to this host and volume:
-that avoids accidentally unlocking or mounting a different disk.
-
-## Related
-
-- [[Storage Policy]]
-- [[Dankserver Stack]]

docs/Dankserver/Jellyfin.md

@@ -1,173 +0,0 @@
-# Jellyfin
-
-Jellyfin is a free, open-source media server and suite of applications for organizing, managing, and streaming personal media collections (movies, TV shows, music, photos) to networked devices. It is a community fork of Emby, written in C# (.NET), and uses FFmpeg for transcoding.
-
-**Host:** Dankserver — **Hostname:** `jellyfin.dankserver.local` — **Docs:** https://jellyfin.org
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Latest version | Check https://jellyfin.org/downloads |
-| Image | `jellyfin/jellyfin:latest` (Docker Hub) or `ghcr.io/jellyfin/jellyfin` |
-| Base | Debian-based |
-| Primary port | 8096 (HTTP, web UI + API) |
-| HTTPS port | 8920 (optional) |
-| Client discovery | 7359/udp |
-| DLNA | 1900/udp (optional) |
-| GPU passthrough | `/dev/dri` (Intel QSV / AMD VA-API) or NVIDIA NVENC |
-| License | MIT |
-
-## Architecture
-
-```mermaid
-graph LR
-    U[User] --> C[Caddy :80/:443]
-    C --> J[Jellyfin :8096]
-    J --> Lib[/media/movies]
-    J --> Lib2[/media/shows]
-    J --> Lib3[/media/music]
-    J --> Conf[Jellyfin config]
-    J --> Cache[Jellyfin cache]
-```
-
-## Docker Compose
-
-```yaml
-services:
-  jellyfin:
-    image: jellyfin/jellyfin:latest
-    container_name: jellyfin
-    restart: unless-stopped
-    user: "1000:1000"
-    ports:
-      - "8096:8096/tcp"
-      - "8920:8920/tcp"
-      - "7359:7359/udp"
-      - "1900:1900/udp"
-    environment:
-      - JELLYFIN_PublishedServerUrl=http://192.168.0.235:8096
-      - DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true
-    volumes:
-      - /opt/jellyfin/config:/config
-      - /opt/jellyfin/cache:/cache
-      - /mnt/media/movies:/data/movies:ro
-      - /mnt/media/shows:/data/shows:ro
-    devices:
-      - /dev/dri:/dev/dri
-    group_add:
-      - "992"    # Host render group GID — verify with `getent group render`
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8096/health"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-```
-
-## Caddy Configuration
-
-```caddy
-jellyfin.dankserver.local {
-    reverse_proxy 127.0.0.1:8096
-
-    transport http {
-        versions h2c 1.1
-    }
-}
-```
-
-Set **Base URL** to `/` (or the path you configure) in Jellyfin Dashboard → Networking. Add `127.0.0.1` to **Known Proxies**.
-
-## Storage
-
-| Mount | Purpose | Size Estimate |
-|-------|---------|---------------|
-| `/opt/jellyfin/config` | SQLite database, settings, cached images | 10–100 GB |
-| `/opt/jellyfin/cache` | Transcoding segments, extracted posters | Self-cleaning per session |
-
-The database must be on local storage (SSD/NVMe preferred). Media libraries can be on NFS/mergerfs. If media is on NFS, set `DOTNET_SYSTEM_IO_DISABLEFILELOCKING=true`.
-
-## Authentication
-
-- **Built-in auth (default):** Per-user libraries, parental controls, max bitrate limits. Manage via Dashboard → Users.
-- **Caddy basic auth:** Single login layer. Does not integrate with per-user Jellyfin permissions.
-- **LDAP/OIDC:** Available via plugins (LDAP-Auth, Jellyfin SSO). Integrates with Authentik, Keycloak, etc.
-
-For Dankserver, built-in auth is recommended. Use Caddy basic auth on top for external access.
-
-## Metadata Scraping
-
-| Provider | Content | Type |
-|----------|---------|------|
-| TheMovieDb (TMDb) | Movies, TV shows, posters, fan art, overviews | Default |
-| OMDb API | Movies only, English metadata | Default |
-| TheTVDB | TV shows, episodes | Plugin |
-| fanart.tv | Extra artwork, banners, thumbnails | Plugin |
-| AniDB | Anime metadata | Plugin |
-| MusicBrainz | Music metadata | Plugin |
-| Open Subtitles | Subtitle downloads | Plugin |
-| Intro Skipper | TV show intro detection | Plugin |
-
-## Clients
-
-| Platform | App | Quality |
-|----------|-----|---------|
-| Web | Jellyfin Web | Excellent |
-| Android | Jellyfin (Google Play / F-Droid) | Excellent |
-| Android TV / Fire TV | Jellyfin | Good |
-| iOS / iPadOS | Swiftfin (third-party) | Good |
-| Desktop | Jellyfin Media Player (mpv-based) | Excellent |
-| Roku | Jellyfin (Channel Store) | Good |
-| Kodi | Jellyfin addon | Excellent |
-| Samsung/LG Smart TVs | TV browser | Fair |
-
-## Jellyfin vs Plex
-
-| Feature | Jellyfin | Plex |
-|---------|----------|------|
-| Cost | Free, no tiers | Free core; Plex Pass ($5/mo or $120 lifetime) |
-| License | MIT | Proprietary |
-| Account required | No | Yes |
-| Hardware transcoding | Free | Plex Pass required |
-| Mobile sync | Free | Plex Pass |
-| Live TV / DVR | Free | Plex Pass |
-| Client apps | Good, improving | Excellent, widest device support |
-
-Run Jellyfin alongside Plex for an incremental migration. Point Jellyfin at the same media folders as Plex. Watch history and metadata will not transfer cleanly.
-
-## Resource Requirements
-
-- **Idle / Direct Play:** ~200–500 MB RAM, <5% CPU. Negligible.
-- **1080p transcode (CPU):** 40–60% CPU on a modern multi-core CPU.
-- **1080p transcode (with QSV):** 3–5% CPU.
-- **4K HDR transcode (with QSV):** 5–15% CPU.
-- **Minimum host RAM:** 4 GB. 8 GB comfortable for transcoding.
-
-## Deployment Checklist
-
-1. Verify GPU on host: `ls -l /dev/dri/`, `lspci | grep -i vga`, `getent group render`.
-2. Install media drivers: `sudo apt install intel-media-va-driver-non-free vainfo intel-gpu-tools`.
-3. Create directories: `/opt/jellyfin/config`, `/opt/jellyfin/cache`.
-4. Write compose file, verify device and GID.
-5. Deploy: `podman compose up -d`.
-6. Test GPU passthrough: `podman exec jellyfin /usr/lib/jellyfin-ffmpeg/vainfo --display drm --device /dev/dri/renderD128`.
-7. Configure Caddy reverse proxy. Set Known Proxies in Jellyfin.
-8. Add media libraries in Dashboard.
-9. Install plugins: Open Subtitles, fanart.tv, Intro Skipper.
-10. Test playback on Android/Chrome.
-
-## Gotchas
-
-- **GPU passthrough silently failing:** `/dev/dri` must be passed and the correct render node used. Verify with `vainfo` inside the container.
-- **Wrong render group GID:** Common GID is 992, but varies per host. Run `getent group render` to find it.
-- **Subtitle burn-in triggers full transcode:** PGS (image-based) subtitles force full video transcoding. SRT/ASS (text-based) play fine.
-- **Database on NFS:** SQLite on network storage causes slow startup and corruption risk. Keep `/config` on local storage.
-- **Client bitrate limit too low:** Many clients default to 20 Mbps. Set Max Streaming Bitrate to Original or Maximum.
-- **Naming convention mismatches:** Non-standard folder/file names cause missing metadata. Use `Movie Title (Year)` for movies and `Show Name/Season 01/Show Name S01E01.mkv` for TV.
-
-## See Also
-
-- [[Dankserver Stack]]
-- [[ARR-Stack]] — automated download into Jellyfin libraries
-- [[Copyparty]] — alternative file viewer for existing collections
-- Plex — existing media server on this host

docs/Dankserver/RomM-Scraping.md

@@ -1,296 +0,0 @@
-# RomM Scraping & Asset Management
-
-This guide covers metadata providers, cover art, ROM naming conventions, emulator integration, and the scraping workflow for RomM on Dankserver.
-
-**Reference:** https://docs.romm.app
-
-## 1. Metadata Providers
-
-RomM v5.1.0+ supports 11+ metadata providers. The `scan.priority.metadata` list determines which provider wins for descriptive fields. The `scan.priority.artwork` list controls cover art sourcing separately.
-
-### Screenscraper (ScreenScraper.fr)
-
-- **Type:** Free, community-driven (French provider)
-- **Credentials:** `SCREENSCRAPER_USER` + `SCREENSCRAPER_PASSWORD` (account user/pass)
-- **Registration:** https://www.screenscraper.fr/membreinscription.php
-- **Rate limits:** No documented hard cap. Community-claimed ~5 req/sec burst.
-- **Capabilities:** Titles, descriptions, cover art, screenshots, manuals, 3D boxes, CD/cartridge covers, marquee logos, fanart, bezels, gameplay videos.
-- **Region support:** Excellent. Respects `scan.priority.region` and `scan.priority.region_mode`.
-- **Best for:** Richest media coverage (all art types), multi-platform, free. Use as secondary to IGDB for metadata, or swap priority order if Screenscraper art quality is preferred.
-
-### IGDB (Internet Game Database / Twitch)
-
-- **Type:** Freemium (free tier with limits; paid for higher volume)
-- **Credentials:** `IGDB_CLIENT_ID` + `IGDB_CLIENT_SECRET` (Twitch OAuth2)
-- **Registration:** Twitch Developer Portal. Create app with name `romm-<random hash>`, OAuth redirect `localhost`, category `Application Integration`, client type `Confidential`.
-- **Rate limits:** ~10 req/sec for free tier. Cached responses reduce effective calls.
-- **Capabilities:** Titles, descriptions, cover art, screenshots, related games, release dates, genres, platforms.
-- **Limitations:** No 3D boxes or physical media art. No manuals. No marquee/fanart.
-- **Default priority:** Highest (`igdb` listed first in both metadata and artwork priority).
-
-### MobyGames
-
-- **Type:** Paid (hobbyist API subscription; non-commercial use only)
-- **Credentials:** `MOBYGAMES_API_KEY`
-- **Rate limits:** Hobbyist tier ~0.2 req/sec (one request every 5 seconds).
-- **RomM recommendation:** "We recommend using ScreenScraper instead, as it is free to use."
-- **Best for:** Supplemental metadata depth for older PC/console titles.
-
-### RetroAchievements (RAHasher)
-
-- **Type:** Free, hash-based achievement tracking
-- **Credentials:** `RETROACHIEVEMENTS_API_KEY`
-- **How it works:** RomM bundles RAHasher (v1.8.3+). ROMs are hashed during scans, matched against RA online database. Achievement progress synced per-user via their RA username.
-- **Known issues:** Folder-based disc ROMs (`.cue` + `.bin`, `.gdi`) may fail hashing. Non-zip archives (`.7z`, `.rar`) had incorrect hash computation. Ensure `filesystem.skip_hash_calculation` is `false`.
-- **Important:** Achievements cannot be earned via RomM's built-in EmulatorJS player. Must use an RA-capable standalone emulator.
-
-### Additional Built-in Providers
-
-| Provider | Slug | Enabled Via | Type |
-|----------|------|-------------|------|
-| Hasheous | `hasheous` | `HASHEOUS_API_ENABLED=true` | Free, hash-based, proxies IGDB data |
-| Playmatch | `playmatch` | `PLAYMATCH_API_ENABLED=true` | Multi-provider hash matching |
-| LaunchBox | `launchbox` | `LAUNCHBOX_API_ENABLED=true` + XML | Local XML database |
-| Flashpoint | `flashpoint` | `FLASHPOINT_API_ENABLED=true` | 180,000+ browser games |
-| HowLongToBeat | `hltb` | `HLTB_API_ENABLED=true` | Game completion times |
-| SteamGridDB | `steamgriddb` | `STEAMGRIDDB_API_KEY` | Custom cover art |
-| ES-DE gamelist | `gamelist` | Place `gamelist.xml` in platform folder | Local XML |
-
-## 2. Recommended Provider Combinations
-
-| Combo | Providers | Best For | Cost |
-|-------|-----------|----------|------|
-| Quick Starter | Hasheous only | No API keys needed | Free |
-| French Connection | Screenscraper + RetroAchievements | Rich media, free | Free |
-| Chef's Choice (recommended) | Hasheous + IGDB + SteamGridDB + RetroAchievements | Best overall quality | Free (IGDB free tier) |
-| Twitch Fanboy | IGDB + Playmatch | Single-provider solution | Free |
-
-**For Dankserver,** the recommended setup:
-- `IGDB` (primary metadata) + `SCREENSCRAPER` (artwork backup + rich media) + `RETROACHIEVEMENTS` (achievements) + `HASHEOUS_API_ENABLED=true` (fast hash-based fallback)
-
-## 3. ROM File Naming Conventions
-
-### No-Intro Naming Scheme (Recommended)
-
-The de facto standard for cartridge and handheld ROMs. Only `Title` and `Region` are required.
-
-```
-Super Metroid (Japan, USA) (En,Ja)
-Sonic the Hedgehog (USA)
-Metal Gear Solid (USA) (Disc 1)
-[BIOS] PS1 System BIOS (USA)
-Tetris [!].gba
-```
-
-**Field meanings:**
-- `(Region)`: `USA`, `EUR`, `JPN`, `UE` (USA+Europe), `JUE` (Japan+USA+Europe)
-- `(Languages)`: `En`, `Ja`, `De`, `Fr`, `Es`
-- `[Status]`: `[!]` = verified good dump, `[b]` = bad dump, `[c]` = completed
-- `(Devstatus)`: `(Proto)`, `(Demo)`, `(Release Candidate)`
-- `[BIOS]` prefix: Identifies BIOS files
-
-### RomM's Region/Language/Revision Parsing
-
-RomM parses inline tags from filenames:
-
-```
-tetris (En) [1.0001] [HACK] [!].gba
-```
-
-- Region/language: ISO-like codes or full names
-- Revision: `rev v1` or `rev-1`
-- Provider ID tags: `(igdb-xxxx)`, `(ssfr-xxxx)`, `(moby-xxxx)`, `(ra-xxxx)` — forces specific provider match
-
-### Best Practices for Dankserver
-
-1. Use No-Intro naming as the standard.
-2. Use **Igir** to rename and organize ROMs into RomM's required folder structure:
-   ```
-   igir copy zip test clean \
-     --dat "No-Intro*.zip" \
-     --input "ROMs" \
-     --output "/opt/romm/roms/{romm}"
-   ```
-3. Keep `[BIOS]` prefix for BIOS files in the `bios/` directory.
-4. One ROM file per game folder where possible.
-5. Use archive formats (`.zip`, `.7z`) — RomM supports them natively.
-
-## 4. Cover Art & Media Assets
-
-### Media Types
-
-| Media Type | Description | Best Provider |
-|-----------|-------------|---------------|
-| `box2d` | Standard 2D front cover art | IGDB, Screenscraper |
-| `box3d` | 3D box art | Screenscraper |
-| `miximage` | Composite (box + screenshot + logo) | Screenscraper |
-| `screenshot` | In-game screenshot | IGDB, Screenscraper |
-| `marquee` | Transparent logo/header | Screenscraper |
-| `bezel` | EmulatorJS-compatible bezel | Screenscraper |
-| `manual` | PDF instruction manual | Screenscraper |
-| `video` | Gameplay video/trailer | Screenscraper |
-
-### Custom Artwork Directories
-
-RomM supports manual artwork placement via subdirectories within each game folder:
-
-```
-/opt/romm/library/roms/{platform}/{game}/
-    game.rom
-    covers/
-    screenshots/
-    3dboxes/
-    manuals/
-    fanart/
-    videos/
-    marquees/
-    bezels/
-```
-
-For ES-DE gamelist imports, media maps as:
-
-| gamelist.xml Tag | Folder Name |
-|-----------------|-------------|
-| `<cover>` | `covers/` |
-| `<backcover>` | `backcovers/` |
-| `<box3d>` | `3dboxes/` |
-| `<marquee>` | `marquees/` |
-| `<screenshot>` | `screenshots/` |
-| `<fanart>` | `fanart/` |
-| `<manual>` | `manuals/` |
-
-### MAME Artwork
-
-MAME uses cabinet pics, flyers, marquees, control panels, backdrops, overlays. Stored in `{platform}/mame_artwork/{game}/` subdirectories.
-
-Sources: Mr. Do's Arcade (https://www.mameworld.info/mrdo/), ProgettoSnaps (http://www.progettosnaps.net/artworks/).
-
-## 5. Emulator Integration
-
-### EmulatorJS (Built-in)
-
-Web-based player using RetroArch cores compiled to WebAssembly. Runs in-browser with no local emulator needed. Configured via `emulatorjs.*` settings in `config.yml`. Supports netplay via STUN/TURN servers.
-
-### Emulator Streaming
-
-Launch native emulator containers (PCSX2, Dolphin, Xemu) and stream picture/sound/input back to the browser. Configured via `streaming.*` in `config.yml`.
-
-### BIOS File Requirements
-
-BIOS files placed in `/opt/romm/library/bios/{platform}/`. Platform subdirectories required. File names should match emulator expectations:
-
-| Platform | BIOS File |
-|----------|-----------|
-| PlayStation | `scph1001.bin`, `scph5501.bin`, `scph5502.bin` |
-| GB/GBC | `gb_bios.bin`, `gbc_bios.bin` |
-| SNES | `sfc_bios.bin` |
-
-BIOS matching done by hash against platform requirements.
-
-### Folder Structure Options
-
-**Structure A (recommended):**
-```
-/opt/romm/library/
-    roms/
-        └── {platform}/
-            └── {game}.rom
-    bios/
-        └── {platform}/
-            └── {bios_file}.bin
-```
-
-**Structure B (fallback):**
-```
-/opt/romm/library/
-    └── {platform}/
-        ├── roms/
-        │   └── {game}.rom
-        └── bios/
-            └── {bios_file}.bin
-```
-
-Dankserver mounts `/opt/romm/library:/romm/library` — RomM auto-detects structure.
-
-## 6. Scraping Workflow
-
-### Recommended Pipeline
-
-1. Organize ROMs → No-Intro naming + RomM folder structure.
-2. Create `config.yml` → Set provider credentials, priority lists, region prefs.
-3. First scan → Scan ALL → verify everything is matched.
-4. Review unmatched → Manually match or add provider tags to filenames.
-5. Fetch artwork → Re-scan with `scan.media` extended to desired types.
-6. Sync RetroAchievements → Set user RA username → trigger sync.
-7. Export gamelist → Enable `scan.gamelist.export` for ES-DE/Batocera.
-8. Maintain → Enable scheduled rescans for new DAT updates.
-
-### Scan Types
-
-| Scan Type | What It Does |
-|-----------|-------------|
-| ALL | Full scan — hash, match metadata, fetch artwork |
-| UNMATCHED | Only matches unmatched ROMs (faster) |
-| METADATA | Updates metadata only |
-| ARTWORK | Re-fetches artwork only |
-| RETROACHIEVEMENTS | Syncs achievement data |
-
-### Region Handling
-
-```yaml
-scan:
-  priority:
-    region:
-      - "us"
-      - "eu"
-      - "jp"
-    region_mode: prefer_rom_tags   # or prefer_config
-```
-
-- `prefer_rom_tags` (default): ROM filename region tags take precedence.
-- `prefer_config`: Config regions come first.
-- **NTSC-U** (USA): 60 Hz, typically fastest, most translations.
-- **PAL** (Europe): 50 Hz, sometimes slower, may have bonus content.
-- **NTSC-J** (Japan): 60 Hz, Japanese only unless region-patched.
-
-## 7. Collection Management Tools
-
-| Tool | Primary Role | Platform |
-|------|-------------|----------|
-| RomVault 3.7.6 | DAT-file-based verification and reorganization | Windows |
-| clrMAMEPro | ROM set validation, deduplication, 1G1R | Windows |
-| DATBios | BIOS file organization using DAT files | Windows |
-| Batocera/Lakka integration | ES-DE gamelist import from RomM | Linux/ARM |
-
-## 8. Environment Variables (Dankserver)
-
-```bash
-# Auth
-ROMM_USERNAME=admin
-ROMM_PASSWORD=<strong password>
-ROMM_AUTH_SECRET_KEY=$(openssl rand -hex 32)
-
-# Metadata Providers
-IGDB_CLIENT_ID=<your-client-id>
-IGDB_CLIENT_SECRET=<your-client-secret>
-SCREENSCRAPER_USER=<screenscraper-username>
-SCREENSCRAPER_PASSWORD=<screenscraper-password>
-MOBYGAMES_API_KEY=<mobygames-key>
-RETROACHIEVEMENTS_API_KEY=<retroachievements-api-key>
-STEAMGRIDDB_API_KEY=<steamgriddb-key>
-
-# Feature Flags
-HASHEOUS_API_ENABLED=true
-PLAYMATCH_API_ENABLED=true
-
-# Scanning
-ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true
-ENABLE_SCHEDULED_RESCAN=true
-REFRESH_RETROACHIEVEMENTS_CACHE_DAYS=7
-```
-
-## See Also
-
-- [[RomM]] — main service documentation
-- [[Dankserver Stack]]
-- [[ARR-Stack]] — Readarr for ebook auto-downloads alongside ROM management

docs/Dankserver/RomM.md

@@ -1,194 +0,0 @@
-# RomM
-
-RomM (ROM Manager) is a self-hosted ROM library manager and player. It scans a ROM collection, enriches it with metadata and cover art from multiple providers, and serves it through a clean web interface. RomM also supports browser-based emulation via EmulatorJS and native emulator streaming.
-
-**Host:** Dankserver — **Hostname:** `romm.dankserver.local` — **Docs:** https://docs.romm.app
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Latest version | 5.2.0 (released 2026-07-29) |
-| Image | `ghcr.io/rommapp/romm:latest` |
-| Port | 8085 (host) → 8080 (container) |
-| Database | MariaDB (default); PostgreSQL optional |
-| Cache / Queue | Redis or Valkey (required for task queue) |
-| Frontend | React SPA |
-| Backend | FastAPI / Node |
-| Metadata providers | IGDB, Screenscraper, MobyGames, Hasheous, Playmatch, LaunchBox, Flashpoint, HowLongToBeat, SteamGridDB, RetroAchievements, TGDB, ES-DE gamelist |
-| Emulator player | EmulatorJS (WebAssembly RetroArch cores) |
-| Emulator streaming | Native containers (PCSX2, Dolphin, Xemu) → browser |
-
-## Architecture
-
-RomM 5.2.0 requires three supporting services. The compose stack includes `romm`, `mariadb`, and `redis` (or `valkey`).
-
-```mermaid
-graph LR
-    U[User] --> C[Caddy :80/:443]
-    C --> R[RomM :8085]
-    R --> DB[(MariaDB)]
-    R --> Cache[(Redis)]
-    R --> Lib[/romm/library]
-    R --> Res[/romm/resources]
-```
-
-## Configuration
-
-### Environment Variables
-
-| Variable | Required | Description |
-|----------|----------|-------------|
-| `ROMM_USERNAME` | Yes | Initial admin username |
-| `ROMM_PASSWORD` | Yes | Initial admin password (set on first boot) |
-| `ROMM_AUTH_SECRET_KEY` | Yes | JWT signing key. Generate with `openssl rand -hex 32`. Replaces the old `ROMM_SECRET_KEY`. |
-| `DB_HOST` | Yes | MariaDB hostname (e.g. `mariadb`) |
-| `DB_NAME` | Yes | Database name (e.g. `romm`) |
-| `DB_USER` | Yes | Database user |
-| `DB_PASSWD` | Yes | Database password |
-| `REDIS_HOST` | Yes | Redis hostname (e.g. `redis`) |
-| `DISABLE_SETUP_WIZARD` | No | Set to `true` to skip the first-run wizard |
-| `OIDC_ENABLED` | No | Enable OpenID Connect authentication |
-| `OIDC_CLIENT_ID` | No | OIDC client ID |
-| `OIDC_CLIENT_SECRET` | No | OIDC client secret |
-| `OIDC_REDIRECT_URI` | No | OIDC redirect URI |
-| `STREAMING_BROKER_SECRET` | No | Secret for native emulator streaming |
-| `STREAMING_SAVE_TIMEOUT` | No | Timeout for save states during streaming |
-| `KIOSK_MODE` | No | Enable kiosk mode |
-| `ROMM_SESSION_SECURE_COOKIE` | No | Secure cookie flag for sessions |
-
-### Metadata Provider Credentials
-
-These are set via environment variables or in `config.yml`:
-
-| Variable | Provider | Notes |
-|----------|----------|-------|
-| `IGDB_CLIENT_ID` / `IGDB_CLIENT_SECRET` | IGDB (Twitch OAuth2) | Create app at Twitch Developer Portal. Free tier: ~10 req/sec. |
-| `SCREENSCRAPER_USER` / `SCREENSCRAPER_PASSWORD` | Screenscraper | Account credentials (not API key). Free, community-driven. |
-| `MOBYGAMES_API_KEY` | MobyGames | Paid hobbyist API. Note: RomM recommends Screenscraper as free alternative. |
-| `RETROACHIEVEMENTS_API_KEY` | RetroAchievements | Hash-based achievements via bundled RAHasher binary. |
-| `STEAMGRIDDB_API_KEY` | SteamGridDB | Custom cover art via the search cover button. |
-| `HASHEOUS_API_ENABLED` | Hasheous | Set to `true`. Free, hash-based, proxies IGDB data. Fast scans. |
-| `PLAYMATCH_API_ENABLED` | Playmatch | Set to `true`. Multi-provider hash matching (IGDB, SS, SteamGridDB, RA). |
-
-### Config File
-
-RomM uses `config.yml` for scan settings, available through the admin UI (Scan Settings). Place the file at `/opt/romm/config/config.yml` on the host.
-
-Key scan config options:
-
-```yaml
-scan:
-  priority:
-    metadata:
-      - igdb
-      - ss        # Screenscraper
-    artwork:
-      - igdb
-      - ss
-    region:
-      - us
-      - eu
-      - jp
-    region_mode: prefer_rom_tags   # or prefer_config
-  media:
-    - box2d
-    - screenshot
-    - manual
-  gamelist:
-    export: true    # Export ES-DE compatible gamelist.xml
-```
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/romm/library` | `/opt/romm/library` |
-| `/romm/resources` | `/opt/romm/resources` |
-| `/romm/config` | `/opt/romm/config` |
-
-The library volume contains ROM files and subdirectories per platform. The resources volume stores cached artwork, metadata, and the SQLite/SQLite-backed asset cache. The config volume holds `config.yml` and other configuration.
-
-## Docker Compose
-
-```yaml
-services:
-  romm:
-    image: ghcr.io/rommapp/romm:latest
-    container_name: romm
-    restart: unless-stopped
-    environment:
-      ROMM_USERNAME: "${ROMM_USERNAME}"
-      ROMM_PASSWORD: "${ROMM_PASSWORD}"
-      ROMM_AUTH_SECRET_KEY: "${ROMM_AUTH_SECRET_KEY}"
-      DB_HOST: mariadb
-      DB_NAME: romm
-      DB_USER: "${ROMM_DB_USER}"
-      DB_PASSWD: "${ROMM_DB_PASSWD}"
-      REDIS_HOST: redis
-    ports:
-      - "8085:8080"
-    volumes:
-      - /opt/romm/library:/romm/library
-      - /opt/romm/resources:/romm/resources
-      - /opt/romm/config:/romm/config
-    depends_on:
-      mariadb:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-
-  mariadb:
-    image: docker.io/mariadb:11
-    container_name: romm-mariadb
-    restart: unless-stopped
-    environment:
-      MARIADB_ROOT_PASSWORD: "${MARIADB_ROOT_PASSWORD}"
-      MARIADB_DATABASE: romm
-      MARIADB_USER: "${ROMM_DB_USER}"
-      MARIADB_PASSWORD: "${ROMM_DB_PASSWD}"
-    volumes:
-      - /opt/romm/mariadb:/var/lib/mysql
-    healthcheck:
-      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-
-  redis:
-    image: docker.io/redis:7-alpine
-    container_name: romm-redis
-    restart: unless-stopped
-    volumes:
-      - /opt/romm/redis:/data
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-```
-
-## Healthcheck
-
-```yaml
-healthcheck:
-  test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
-  interval: 30s
-  timeout: 10s
-  retries: 3
-```
-
-## Gotchas
-
-- **First login overwrites env credentials.** The `ROMM_PASSWORD` env var sets the initial password. After first login, the admin password is controlled in the UI. The env var only applies on first boot.
-- **MariaDB and Redis are required.** RomM 5.2.0 no longer supports SQLite-only deployments. Both MariaDB and Redis must be running.
-- **Reverse proxy cache stale index.html.** Purge Caddy proxy cache after RomM version bumps to avoid serving a stale frontend after an upgrade.
-- **RAHasher build time.** Enabling RetroAchievements hashing increases the first image build. Skip unless you need achievement data.
-- **MobyGames is paid.** The hobbyist API has strict rate limits. Screenscraper is the recommended free alternative.
-- **OIDC requires additional services.** Authentik, Keycloak, or another OIDC provider must be running before enabling `OIDC_ENABLED`.
-
-## See Also
-
-- [[Dankserver Stack]]
-- [[RomM-Scraping]] — metadata providers, cover art, naming conventions, emulator integration
-- [[Calibre-Web]] — sibling library web UI

docs/Dankserver/lidarr.md

@@ -1,53 +0,0 @@
-# Lidarr
-
-Lidarr manages music albums in the *arr stack on Dankserver. It automatically searches, downloads, and organizes albums from indexers into the Jellyfin music library.
-
-**Host:** Dankserver — **Hostname:** `lidarr.dankserver.local` — **Docs:** https://wiki.servarr.com/lidarr
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `lscr.io/linuxserver/lidarr:latest` |
-| Port | 8686 (host) → 8686 (container) |
-| Min RAM | 256 MB |
-| Metadata | MusicBrainz |
-
-## Docker Compose
-
-```yaml
-lidarr:
-  image: lscr.io/linuxserver/lidarr:latest
-  container_name: lidarr
-  restart: unless-stopped
-  environment:
-    - PUID=1000
-    - PGID=1000
-    - TZ=America/New_York
-    - UMASK_SET=022
-  volumes:
-    - /opt/lidarr/config:/config
-    - /mnt/media/music:/music
-    - /mnt/media/downloads:/downloads
-  ports:
-    - "8686:8686"
-  networks: [appnet]
-```
-
-## Configuration
-
-1. Add Prowlarr as the indexer source (prefer torrent-based indexers for FLAC).
-2. Add qBittorrent as the download client with the `music` category.
-3. Set quality profile (Lossless → High Bitrate → Standard).
-4. Set root folder to `/music`.
-5. Add artists and verify downloads.
-
-## Gotchas
-
-- Album variants: Lidarr supports multiple releases of the same album (vinyl vs CD). Use "Preferred" and "Cutoff" quality profiles.
-- Music videos can be optionally downloaded separately from albums.
-
-## See Also
-
-- [[ARR-Stack]] — full *arr stack documentation
-- [[Jellyfin]] — media server that consumes Lidarr downloads

docs/Dankserver/mariadb.md

@@ -1,47 +0,0 @@
-# MariaDB (RomM Backend)
-
-MariaDB 11 provides the database backend for RomM on Dankserver. RomM 5.2.0 requires MariaDB (or PostgreSQL) for its data store.
-
-**Host:** Dankserver — **Docs:** https://mariadb.com/kb/en/docker-mariadb-information/
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/mariadb:11` |
-| Port | 3306 (internal, within `appnet`) |
-| Database name | `romm` |
-| Data volume | `/opt/romm/mariadb` |
-
-## Docker Compose
-
-```yaml
-mariadb:
-  image: docker.io/mariadb:11
-  container_name: romm-mariadb
-  restart: unless-stopped
-  environment:
-    MARIADB_ROOT_PASSWORD: "${MARIADB_ROOT_PASSWORD}"
-    MARIADB_DATABASE: romm
-    MARIADB_USER: "${ROMM_DB_USER}"
-    MARIADB_PASSWORD: "${ROMM_DB_PASSWD}"
-  volumes:
-    - /opt/romm/mariadb:/var/lib/mysql
-  healthcheck:
-    test: ["CMD", "mariadb-admin", "ping", "-h", "localhost"]
-    interval: 10s
-    timeout: 5s
-    retries: 3
-```
-
-## Backup
-
-Back up the volume directory periodically:
-
-```bash
-sudo rsync -a /opt/romm/mariadb /mnt/backup/romm-mariadb/
-```
-
-## See Also
-
-- [[RomM]] — service that uses this database

docs/Dankserver/prowlarr.md

@@ -1,55 +0,0 @@
-# Prowlarr
-
-Prowlarr is the indexer management layer for the *arr stack. It aggregates indexers (torrent and Usenet) and exposes a unified API that Sonarr, Radarr, Lidarr, and Readarr consume.
-
-**Host:** Dankserver — **Hostname:** `prowlarr.dankserver.local` — **Docs:** https://wiki.servarr.com/prowlarr
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `lscr.io/linuxserver/prowlarr:latest` |
-| Port | 9696 (host) → 9696 (container) |
-| Min RAM | 256 MB |
-| Auth | API key (Settings → General) |
-
-## Configuration
-
-1. Add indexers: Settings → Indexers → Add.
-2. Test each indexer with the **Test** button.
-3. Assign tags (e.g., `movies`, `tv`, `music`) to indexers for routing.
-4. Add download client: Settings → Download Clients → Add qBittorrent.
-
-## Docker Compose
-
-```yaml
-prowlarr:
-  image: lscr.io/linuxserver/prowlarr:latest
-  container_name: prowlarr
-  restart: unless-stopped
-  environment:
-    - PUID=1000
-    - PGID=1000
-    - TZ=America/New_York
-    - UMASK_SET=022
-  volumes:
-    - /opt/prowlarr/config:/config
-    - /mnt/media/downloads:/downloads:ro
-  ports:
-    - "9696:9696"
-  networks: [appnet]
-```
-
-## Gotchas
-
-- Prowlarr database grows with indexer data. Set up periodic vacuum/backup.
-- Indexer sync has a 15–60 minute delay. Search results may lag.
-- Rotate API key: update it in every arr app.
-
-## See Also
-
-- [[ARR-Stack]] — full *arr stack documentation
-- [[radarr]]
-- [[sonarr]]
-- [[lidarr]]
-- [[readarr]]

docs/Dankserver/qbittorrent.md

@@ -1,52 +0,0 @@
-# qBittorrent
-
-qBittorrent is the torrent download client for the *arr stack on Dankserver. It receives download requests from Sonarr, Radarr, Lidarr, and Readarr via category tagging.
-
-**Host:** Dankserver — **Hostname:** `qbittorrent.dankserver.local` — **Docs:** https://www.qbittorrent.org
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `lscr.io/linuxserver/qbittorrent:latest` |
-| Port | 8080 (host) → 8080 (container) |
-| Min RAM | 512 MB |
-| Auth | Web UI username/password; LAN mode available |
-
-## Configuration
-
-1. Change the default Web UI password (Settings → Web UI).
-2. Enable "No auth mode for LAN" for arr app connectivity (Settings → Web UI → uncheck "HTTP authentication required for clients").
-3. Create download categories: `movies`, `tv`, `music`, `books`.
-4. Set max active downloads: Settings → Downloads (recommended: 5 global, 3 per category).
-
-## Docker Compose
-
-```yaml
-qbittorrent:
-  image: lscr.io/linuxserver/qbittorrent:latest
-  container_name: qbittorrent
-  restart: unless-stopped
-  environment:
-    - PUID=1000
-    - PGID=1000
-    - TZ=America/New_York
-    - UMASK_SET=022
-  volumes:
-    - /opt/qbittorrent/config:/config
-    - /mnt/media/downloads:/downloads
-    - /mnt/media/downloads/incomplete:/downloads/incomplete
-  ports:
-    - "8080:8080"
-  networks: [appnet]
-```
-
-## Gotchas
-
-- qBittorrent Web UI password changes on first run (check `config/qBittorrent.conf`).
-- Ensure umask is set to `022` so downloaded files are readable by arr apps.
-- Monitor bandwidth limits to avoid saturating the network.
-
-## See Also
-
-- [[ARR-Stack]] — full *arr stack documentation

docs/Dankserver/radarr.md

@@ -1,54 +0,0 @@
-# Radarr
-
-Radarr manages movies in the *arr stack on Dankserver. It automatically searches, downloads, and organizes movies from indexers into the Jellyfin media library.
-
-**Host:** Dankserver — **Hostname:** `radarr.dankserver.local` — **Docs:** https://wiki.servarr.com/radarr
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `lscr.io/linuxserver/radarr:latest` (v6 stable) |
-| Port | 7878 (host) → 7878 (container) |
-| Min RAM | 512 MB |
-| Metadata | TMDB / TheMovieDB API |
-
-## Docker Compose
-
-```yaml
-radarr:
-  image: lscr.io/linuxserver/radarr:latest
-  container_name: radarr
-  restart: unless-stopped
-  environment:
-    - PUID=1000
-    - PGID=1000
-    - TZ=America/New_York
-    - UMASK_SET=022
-  volumes:
-    - /opt/radarr/config:/config
-    - /mnt/media/movies:/movies
-    - /mnt/media/downloads:/downloads
-  ports:
-    - "7878:7878"
-  networks: [appnet]
-```
-
-## Configuration
-
-1. Add Prowlarr as the indexer source.
-2. Add qBittorrent as the download client with the `movies` category.
-3. Set quality profile and cutoff quality.
-4. Set root folder to `/movies`.
-5. Add movies and verify downloads.
-
-## Gotchas
-
-- Radarr v6 is the current stable (v6.3.x as of 2026-08). `:latest` points to the current release.
-- Movie renaming produces `Title (Year)` format. Jellyfin scanner handles this.
-- File permissions: qBittorrent downloads as PUID; arr apps must read/move files.
-
-## See Also
-
-- [[ARR-Stack]] — full *arr stack documentation
-- [[Jellyfin]] — media server that consumes Radarr downloads

docs/Dankserver/readarr.md

@@ -1,57 +0,0 @@
-# Readarr
-
-> **RETIRED (May 2026).** The Servarr Team retired Readarr: its metadata became unusable and the community migration to Open Library stalled. There is no upstream development. Alternatives: the Bookshelf community fork, or the Calibre content server. See [Readarr (Retired)](https://wiki.servarr.com/readarr).
-
-Readarr manages ebooks and audiobooks in the *arr stack on Dankserver. It automatically searches, downloads, and organizes books into the Calibre library.
-
-**Host:** Dankserver — **Hostname:** `readarr.dankserver.local` — **Docs:** https://wiki.servarr.com/readarr
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `lscr.io/linuxserver/readarr:latest` (v0.2.x development) |
-| Port | 8787 (host) → 8787 (container) |
-| Min RAM | 256 MB |
-| Metadata | Goodreads / StoryGraph |
-
-## Docker Compose
-
-```yaml
-readarr:
-  image: lscr.io/linuxserver/readarr:latest
-  container_name: readarr
-  restart: unless-stopped
-  environment:
-    - PUID=1000
-    - PGID=1000
-    - TZ=America/New_York
-    - UMASK_SET=022
-  volumes:
-    - /opt/readarr/config:/config
-    - /mnt/media/books:/books
-    - /mnt/media/downloads:/downloads
-  ports:
-    - "8787:8787"
-  networks: [appnet]
-```
-
-## Configuration
-
-1. Add Prowlarr as the indexer source (torrent indexers with book content).
-2. Add qBittorrent as the download client with the `books` category.
-3. Set author profile (format preference: epub, mobi, etc.).
-4. Set quality profile for audiobooks (320 kbps MP3, 128 kbps, lossless).
-5. Set root folder to `/books`.
-6. Add authors and verify downloads.
-
-## Gotchas
-
-- Readarr is in development (~v0.2.x). Some features may be incomplete.
-- Calibre integration is folder-based, not API-based. Readarr drops files into `/books` which Calibre-Web serves.
-- Not all indexers have all formats (epub, mobi, pdf). Goodreads/StoryGraph metadata helps match editions.
-
-## See Also
-
-- [[ARR-Stack]] — full *arr stack documentation
-- [[Calibre-Web]] — ebook web frontend that serves Readarr downloads

docs/Dankserver/redis.md

@@ -1,40 +0,0 @@
-# Redis (RomM Cache / Queue)
-
-Redis 7 provides the caching and task queue backend for RomM on Dankserver. RomM 5.2.0 requires Redis (or Valkey) for its task queue.
-
-**Host:** Dankserver — **Docs:** https://redis.io/docs/
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/redis:7-alpine` |
-| Port | 6379 (internal, within `appnet`) |
-| Data volume | `/opt/romm/redis` |
-| Auth | None (local-only, no password needed) |
-
-## Docker Compose
-
-```yaml
-redis:
-  image: docker.io/redis:7-alpine
-  container_name: romm-redis
-  restart: unless-stopped
-  volumes:
-    - /opt/romm/redis:/data
-  healthcheck:
-    test: ["CMD", "redis-cli", "ping"]
-    interval: 10s
-    timeout: 5s
-    retries: 3
-```
-
-## Backup
-
-```bash
-sudo rsync -a /opt/romm/redis /mnt/backup/romm-redis/
-```
-
-## See Also
-
-- [[RomM]] — service that uses this cache

docs/Dankserver/sonarr.md

@@ -1,55 +0,0 @@
-# Sonarr
-
-Sonarr manages TV shows in the *arr stack on Dankserver. It automatically searches, downloads, and organizes episodes from indexers into the Jellyfin media library.
-
-**Host:** Dankserver — **Hostname:** `sonarr.dankserver.local` — **Docs:** https://wiki.servarr.com/sonarr
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `lscr.io/linuxserver/sonarr:latest` (v4 stable) |
-| Port | 8989 (host) → 8989 (container) |
-| Min RAM | 512 MB |
-| Metadata | TMDB / TVDB |
-
-## Docker Compose
-
-```yaml
-sonarr:
-  image: lscr.io/linuxserver/sonarr:latest
-  container_name: sonarr
-  restart: unless-stopped
-  environment:
-    - PUID=1000
-    - PGID=1000
-    - TZ=America/New_York
-    - UMASK_SET=022
-  volumes:
-    - /opt/sonarr/config:/config
-    - /mnt/media/tv:/tv
-    - /mnt/media/downloads:/downloads
-  ports:
-    - "8989:8989"
-  networks: [appnet]
-```
-
-## Configuration
-
-1. Add Prowlarr as the indexer source.
-2. Add qBittorrent as the download client with the `tv` category.
-3. Set quality profile, cutoff, and minimum/maximum episode sizes.
-4. Set root folder to `/tv`.
-5. Add series and verify downloads.
-
-## Gotchas
-
-- Sonarr v4 is the current stable. The v3 branch uses the `:v3` tag.
-- Anime has different quality profiles and naming conventions than Standard series.
-- Missing episodes: set "Unmonitor Missing Episodes" to auto-unmonitor old episodes.
-- Timezone-sensitive scheduler: ensure all containers use the same TZ.
-
-## See Also
-
-- [[ARR-Stack]] — full *arr stack documentation
-- [[Jellyfin]] — media server that consumes Sonarr downloads

docs/Hybrid CPU GPU Processing Dankserver.md

@@ -2,13 +2,13 @@
 
 ## Overview
 
-Dankserver uses NVIDIA GPU with system RAM for hybrid processing. vLLM splits model weights onto GPU and moves KV cache between GPU VRAM and system RAM. This allows larger models to run on GPUs with limited VRAM.
+Dankserver uses NVIDIA GPU with system RAM for hybrid processing. VLLM splits model weights onto GPU and moves KV cache between GPU VRAM and system RAM. This allows larger models to run on GPUs with limited VRAM.
 
 ## Architecture
 
 ### Model Weights
 
-Model weights stay on GPU. vLLM loads the entire model into GPU memory. You cannot offload model weights to CPU. The GPU must hold all model parameters.
+Model weights stay on GPU. VLLM loads the entire model into GPU memory. You cannot offload model weights to CPU. The GPU must hold all model parameters.
 
 ### KV Cache
 
@@ -41,9 +41,9 @@ Set `--gpu-memory-utilization` based on your GPU:
 
 Lower values leave more GPU memory for KV cache. This reduces swap frequency.
 
-### Swap Space / KV Offload
+### Swap Space
 
-Recent vLLM releases removed the `--swap-space` engine argument. Use `--kv-offloading-size` to move KV cache to CPU RAM. In Podman, mount the host cache read-write for KV offload writes. The table below shows historical `--swap-space` guidance as a sizing reference.
+Set `--swap-space` based on system RAM:
 
 | System RAM | Recommended Swap |
 |------------|------------------|
@@ -70,7 +70,7 @@ vllm serve meta-llama/Llama-3.1-70B-Instruct \
 Use FP8 KV cache for Ampere and newer GPUs:
 
 ```
-vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
+vllm serve Qwen/Qwen3-30B-A3B-FP8 \
     --kv-cache-dtype fp8 \
     --gpu-memory-utilization 0.9 \
     --swap-space 16
@@ -177,7 +177,7 @@ htop
 ### Single GPU, Medium Model
 
 ```
-vllm serve Qwen/Qwen3.6-35B-A3B \
+vllm serve Qwen/Qwen3-30B-A3B \
     --gpu-memory-utilization 0.85 \
     --swap-space 16 \
     --max-num-batched-tokens 4096 \
@@ -187,7 +187,7 @@ vllm serve Qwen/Qwen3.6-35B-A3B \
 ### Single GPU, FP8 Model
 
 ```
-vllm serve Qwen/Qwen3.6-35B-A3B-FP8 \
+vllm serve Qwen/Qwen3-30B-A3B-FP8 \
     --kv-cache-dtype fp8 \
     --gpu-memory-utilization 0.9 \
     --swap-space 16 \

docs/Infrastructure/Alloy.md

@@ -1,47 +0,0 @@
-# Alloy
-
-Alloy is Grafana's log and telemetry shipper. It replaces Promtail (EOL 2026-03-02). One Alloy instance runs per host and forwards container and journal logs to Loki, and optionally metrics to Prometheus.
-
-**Hosts:** Dankclaw + Dankserver (one per host) — **Docs:** https://grafana.com/docs/alloy/latest
-
-## Facts (from research)
-
-| Item | Value |
-|------|-------|
-| Image | `franksdank/alloy:latest` (local build with a static busybox healthcheck) |
-| Port | none published (push-only) |
-| Config | River syntax, `config.alloy` |
-| Successor to | Promtail (EOL 2026-03-02) |
-| Inputs | journald socket, container log files |
-| Output | Loki push endpoint |
-
-## Why Alloy and not Promtail
-
-Promtail reached end-of-life on 2026-03-02. Grafana Labs directs new installs to Alloy. This stack uses Alloy from day one.
-
-## Configuration
-
-River config at `shared/alloy/config.alloy`:
-
-- Read journald logs from the host's `/run/systemd/journal/socket`.
-- Read container logs from `/var/lib/containers/...` (podman) or the docker path.
-- Forward to `http://loki:3100/loki/api/v1/push` with `host` and `job` labels.
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/etc/alloy/config.alloy` | `shared/alloy/config.alloy` (bind mount) |
-| `/run/systemd/journal/socket` | host journald socket (read-only) |
-| `/var/log` | host logs (read-only) |
-
-## Gotchas
-
-- The journald socket mount is required for journal scraping.
-- Container log paths differ between docker (`/var/lib/docker/containers`) and podman (`/var/lib/containers/storage/...`); point the config at the right one per host.
-- One Alloy per host; do not run a single instance that tries to reach both hosts' logs.
-
-## See Also
-
-- [[Loki]] — the log backend
-- [[Observability]] — the full stack

docs/Infrastructure/Authentik.md

@@ -1,152 +0,0 @@
-# Authentik — the homelab's SSO / OIDC provider
-
-Authentik is the single sign-on (SSO) and OpenID Connect (OIDC) provider for the
-homelab. It fronts the Caddy vhosts that do not carry their own
-application-level authentication. Apps that have their own login (Grafana,
-Gitea, OpenWebUI, Jellyfin, the *arr stack, Pi-hole) are not wrapped in
-forward_auth — they handle their own session.
-
-- **Version:** `2026.8.0` (pinned in `dankclaw/compose.yaml`)
-- **Host:** dankclaw (`192.168.0.143`)
-- **Name:** `auth.franksdank.cloud`
-- **Stack:** 3 compose services — `authentik-postgres`, `authentik-server`,
-  `authentik-worker` (see [Pi-hole](Pi-hole.md) for the LAN-DNS pattern)
-
-## Why Authentik
-
-The comparison that chose it (against Authelia, Keycloak, Zitadel, Ory) is in
-`.docaudit/research/auth.md`. In short: full OIDC with single-logout, a web GUI
-for all configuration, a native Caddy `forward_auth` outpost, and a footprint
-that fits on a 3-user homelab (~1–1.5 GB idle). Redis was removed in 2025.10,
-so the stack is 3 services, not 4.
-
-## Architecture
-
-```mermaid
-graph LR
-    subgraph Browser
-        U[User]
-    end
-    subgraph Caddy [Caddy - auth gate]
-        FA[forward_auth]
-    end
-    subgraph AK [Authentik]
-        SRV[server :9000]
-        WKR[worker]
-        PG[(postgres)]
-        OUT[outpost /auth/caddy]
-    end
-    subgraph App [Target app]
-        A1[SearXNG / Hermes / Prometheus<br/>RomM / Copyparty / homepage]
-    end
-
-    U -->|1. GET /| FA
-    FA -->|2. GET /outpost.../auth/caddy| OUT
-    OUT -->|3. no session: 302 login| U
-    U -->|4. login at auth.franksdank.cloud| SRV
-    SRV --> PG
-    WKR --> PG
-    SRV -->|5. 2xx: session valid| FA
-    FA -->|6. proxy with headers| A1
-```
-
-1. The browser hits a protected vhost (e.g. `search.franksdank.cloud`).
-2. Caddy `forward_auth` clones the request to the Authentik outpost
-   (`/outpost.goauthentik.io/auth/caddy`).
-3. With no valid session, the outpost returns a `302` to the Authentik login
-   page (`auth.franksdank.cloud`).
-4. The user logs in at Authentik (email + password, TOTP/WebAuthn MFA, or an
-   upstream provider).
-5. On the next request the outpost returns `2xx`; Caddy copies the
-   `X-Authentik-*` headers and proxies to the app.
-
-## The three compose services
-
-| Service | Role | Image |
-|---|---|---|
-| `authentik-postgres` | PostgreSQL 16 (users, flows, sessions, events) | `docker.io/library/postgres:16-alpine` |
-| `authentik-server` | Core + admin API + the embedded Caddy outpost (port 9000) | `ghcr.io/goauthentik/server:2026.8.0` |
-| `authentik-worker` | Background tasks (event processing, provider sync, flows) | `ghcr.io/goauthentik/server:2026.8.0` |
-
-- **Postgres** is the only stateful store. Sessions, caching, and WebSockets
-  moved to PostgreSQL in 2025.10 (Redis removed).
-- **Server** and **worker** run the same image; the `command` selects the role
-  (`server` / `worker`).
-- The worker runs as `0:0` (rootless: container root maps to the host user that
-  owns the data dir).
-- Data persists under `/home/dank/authentik/` (bind mounts, repo policy — no
-  named volumes).
-
-## Caddy integration
-
-The Caddyfile wraps the no-auth vhosts in a `route` block. The pattern (one
-example; the others are identical):
-
-```caddyfile
-search.franksdank.cloud {
-	route {
-		# 1. The outpost path is proxied to Authentik (login UI + session).
-		reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-		# 2. forward_auth checks the session before serving the app.
-		forward_auth authentik-server:9000 {
-			uri /outpost.goauthentik.io/auth/caddy
-			copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-			trusted_proxies private_ranges
-		}
-		# 3. The app.
-		reverse_proxy 192.168.0.143:8081
-	}
-}
-```
-
-Vhosts protected by forward_auth: the homepage (`franksdank.cloud`,
-`www.franksdank.cloud`), SearXNG, Hermes, Prometheus, RomM, Copyparty. The
-`auth.franksdank.cloud` vhost proxies to the server directly (no forward_auth —
-it *is* the login).
-
-## Configuration (secrets)
-
-Two secrets live in the host `.env` (git-ignored; placeholders in
-`.env.example`):
-
-- `AUTHENTIK_PG_PASS` — the PostgreSQL password. Generate: `openssl rand -base64 32`.
-- `AUTHENTIK_SECRET_KEY` — the server's signing key. Generate: `openssl rand -base64 48`.
-
-## First-time setup
-
-1. `./scripts/deploy.sh dankclaw` — pulls, starts the 3 services (postgres
-   health-gated), reloads Caddy.
-2. Open `https://auth.franksdank.cloud` — the initial setup wizard runs. It
-   creates the superuser (email + password) and generates the DB schema.
-3. In the admin interface: **Directory → Applications** → add a **Proxy**
-   application (or an **OAuth2/OpenID Connect** provider) for each vhost you
-   protect. The Caddy outpost reads the application's policy.
-4. For apps with native OIDC (Grafana, Gitea, OpenWebUI): add an **OAuth2/OpenID
-   Connect** provider, create the app with the correct redirect URI, and paste
-   the `client_id` / `client_secret` into the app's SSO settings.
-
-## Operations
-
-- **Health:** `https://auth.franksdank.cloud/-/health/` (the server healthcheck
-  probes this).
-- **Logs:** `podman logs authentik-server` / `podman logs authentik-worker`.
-- **Backups:** `pg_dump` the `authentik-postgres` database (the single source of
-  truth). The `/home/dank/authentik/` dir holds media + static files.
-- **Upgrades:** bump the image tag in `dankclaw/compose.yaml`, run
-  `./scripts/deploy.sh dankclaw`. Authentik migrates the schema on startup.
-
-## Failure modes
-
-| Symptom | Cause | Fix |
-|---|---|---|
-| Every vhost 502s at the outpost | `authentik-server` down | `podman logs authentik-server`; check postgres health |
-| Login loops (302 → login → 302) | `trusted_proxies` / `X-Forwarded-*` mismatch | Confirm Caddy sets `X-Forwarded-For`; the outpost trusts `private_ranges` |
-| "Invalid signature" on the outpost | `AUTHENTIK_SECRET_KEY` differs between server and worker | Both services read the same `.env` var; re-deploy |
-| Postgres "password authentication failed" | `AUTHENTIK_PG_PASS` changed after first init | Reset via `pg_ctl` or re-init the data dir |
-
-## Cross-references
-
-- [Caddy](Caddy.md) — the reverse proxy that carries the forward_auth directives
-- [Pi-hole](Pi-hole.md) — the LAN DNS that resolves `auth.franksdank.cloud`
-- [Network Architecture](../Architecture/Network%20Architecture.md) — the flat naming scheme
-- Research: `.docaudit/research/auth.md` — the full provider comparison

docs/Infrastructure/Caddy.md

@@ -1,112 +0,0 @@
-# Caddy
-
-Caddy is the single reverse proxy for both hosts. It runs as a boot-persistent quadlet container (`caddy.service`) on Dankclaw (192.168.0.143), terminates TLS with real Let's Encrypt certificates, routes hostnames to containers on both hosts, and protects the homepage and sensitive services with HTTP basic auth.
-
-**Docs:** https://caddyserver.com/docs
-
-## Facts (live inventory)
-
-| Item | Value |
-|------|-------|
-| Runtime | Quadlet unit `caddy.service` (`dankclaw/caddy/quadlet/caddy.container`) — boot-persistent, self-healing |
-| Image | `franksdank/caddy-porkbun:latest` (local build, `dankclaw/caddy/Containerfile`) |
-| Config | `/home/dank/Containers/caddy/Caddyfile` (bind mount, rendered by deploy) |
-| Data | `/home/dank/Containers/caddy/data` (certs) and `.../config` (ACME state) |
-| Ports | 80, 443 on 192.168.0.143 |
-| TLS | Let's Encrypt, DNS-01 via the Porkbun module (one cert per vhost) |
-| Runs on | Dankclaw only — Dankserver has no Caddy |
-
-The Caddyfile in git (`dankclaw/Caddyfile`) carries a `REPLACE_WITH_HASH` placeholder. `scripts/deploy.sh` renders the real basicauth hash from the host `.env` (`CADDY_BASICAUTH_HASH`) into the mounted path before start. See [[Deployment Flow]].
-
-## Quadlet (boot persistence)
-
-Caddy left `compose.yaml` on 2026-08-26. A reboot exposed that the bare compose container never comes back: rootless Podman has no boot-time auto-start for bare containers, and `restart: unless-stopped` does not survive the explicit stop that shutdown performs. The quadlet unit is now the only way Caddy starts (see AGENTS.md "Boot Persistence & Self-Healing").
-
-- **Install path on Dankclaw:** `~/.config/containers/systemd/caddy.container` (unit `caddy.service`)
-- **Network:** `dankclaw_default` (the compose bridge — the Caddyfile's container-name upstreams must keep resolving; if no compose service has created the network yet at boot, quadlet creates it first and compose reuses it by name later)
-- **Published ports:** 80/tcp, 443/tcp
-- **Env file:** `/home/dank/Containers/caddy/env` (`PORKBUN_API_KEY`, `PORKBUN_API_SECRET_KEY`) — the deploy writes it from the checkout's `.env`; secrets never enter the repo (placeholders in `dankclaw/caddy/env.example`)
-- **Restart:** `Restart=always`, `RestartSec=5`; the health check probes the bare-IP catch-all (302) inside the container
-- **Image policy:** `Pull=never` — the image is built on-host and exists nowhere else; a boot must not stall on a registry
-- **Enable state:** `systemctl --user is-enabled caddy` reports `generated` — that is the healthy state for a quadlet. The podman generator creates the `default.target.wants` symlink from the unit's `[Install]` section at every user-manager start (linger covers boot); an explicit `systemctl enable` fails on generated units, so the deploy does not run it
-- **Deploy:** `./scripts/deploy.sh dankclaw --service caddy` — renders the Caddyfile, rebuilds the image if the Containerfile changed, retires the legacy split proxy, writes the env file, removes the old bare container (all state is in host volumes), then installs and starts the unit
-
-## The porkbun image
-
-Stock Caddy cannot solve the ACME challenge because the router exposes no inbound port 80/443. The local image adds the `github.com/caddy-dns/porkbun` module, which answers the DNS-01 challenge by creating and deleting a `_acme-challenge` TXT record through the Porkbun API. No inbound port is needed. Rebuild only when Caddy or the module needs an update:
-
-```
-podman build -t franksdank/caddy-porkbun:latest -f dankclaw/caddy/Containerfile .
-```
-
-The Caddyfile references the credentials through the environment:
-
-```
-acme_dns porkbun {
-    api_key {env.PORKBUN_API_KEY}
-    api_secret_key {env.PORKBUN_API_SECRET_KEY}
-}
-```
-
-The deploy writes both variables from the host `.env` into `/home/dank/Containers/caddy/env`, which the quadlet reads via `EnvironmentFile`. Keep the real keys in `.env` only; `.env.example` (and `dankclaw/caddy/env.example`) hold placeholders. The Porkbun account holds only `franksdank.cloud`.
-
-## Vhost map
-
-Every vhost uses the flat `franksdank.cloud` convention. The full list lives in `dankclaw/Caddyfile`. Routing by host:
-
-| Upstream | Reach |
-|----------|-------|
-| Dankclaw services (compose network) | by container name, e.g. `comfyui:8188` |
-| Dankclaw quadlet services | by published LAN port, e.g. `192.168.0.143:3000` (Gitea) |
-| Dankserver services | by published LAN port, e.g. `192.168.0.235:8085` (RomM) |
-
-The bare-IP catch-all `:80` 302-redirects to `https://dankclaw.franksdank.cloud/`. The legacy name `dankserver.franksdank.cloud` 302-redirects there too. `plex.franksdank.casa` is the only plain-HTTP vhost; it cannot get a DNS-01 cert because `franksdank.casa` is not in the Porkbun account. The canonical Plex name is `plex.franksdank.cloud`. Both Plex vhosts dial PMS **over TLS upstream** (`transport http { tls_insecure_skip_verify }` — in this Caddy build the flag is a bare token inside the transport block, not a `tls {}` sub-block): PMS 1.43.x silently closes plain-HTTP connections from non-loopback sources (diagnosed 2026-08-26 — it accepts, reads the request, and FINs without responding), which surfaced here as an empty-body 502 (blank page in browsers). `insecure_skip_verify` applies only to this pinned-LAN hop (PMS serves a self-signed cert); client-facing Let's Encrypt TLS is unchanged. The Plex web UI is at `/web/`; `/` returns the XML `MediaContainer` index.
-
-## Basic auth
-
-The Caddyfile in git carries a **bare** placeholder; deploy.sh substitutes the **full** hash from the host `.env` (`CADDY_BASICAUTH_HASH`):
-
-```
-chat.franksdank.cloud {
-    basicauth {
-        dank REPLACE_WITH_HASH
-    }
-    reverse_proxy 192.168.0.143:3005
-}
-```
-
-After rendering, the line reads `dank $2a$14$xxxx…` (one complete hash). Do not put a `$2a$14$` prefix in the template: it would render a double-prefixed, invalid hash (2026-08-22 incident — every basicauth vhost returned 401 for every password). `tests/deployment_flow.bats` pins the render contract.
-
-The homepage (and SearXNG, Hermes, Prometheus, RomM, Copyparty) use `forward_auth` (Authentik SSO) rather than basicauth; see [[Homepage]] and [[Authentik]]. The basicauth hash above is still rendered for the vhosts that keep it (Grafana, Pi-hole).
-
-## Operations
-
-Validate before a reload:
-
-```
-caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
-```
-
-The admin endpoint stays on (loopback-only, unpublished port), so a Caddyfile change hot-reloads within seconds via the file watcher. deploy.sh also issues an explicit reload after rendering. Manual reload from the host:
-
-```
-podman exec caddy caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
-```
-
-Certificates and ACME state live in the caddy data volume. Back it up with the stack. Each vhost owns one certificate; renewals are automatic and also use DNS-01.
-
-## Gotchas
-
-- A bare compose container is **not** boot-persistent: after the 2026-08-26 reboot Caddy stayed down for hours because `restart: unless-stopped` does not survive shutdown's explicit stop. It now runs only from its quadlet unit — do not re-add it to `compose.yaml`.
-- `admin off` disables the config file watcher: rendered changes then sit on disk while Caddy serves its stale in-memory config until the container is recreated, and nothing logs an error (2026-08-22: a broken basicauth hash survived several deploys this way). `tests/caddyfile.bats` forbids `admin off`.
-- The basicauth template must stay a bare `REPLACE_WITH_HASH` token (no `$2a$14$` prefix) — the rendered value is the complete hash. A double prefix silently rejects every password with 401; no Caddy error is logged.
-- A new vhost needs three things: a Pi-hole `addn-hosts` line, a Porkbun A record, and the Caddyfile block. See [[LAN-DNS]].
-- The G36 has no NAT loopback, so LAN clients must resolve the name to the LAN IP. That is the Pi-hole role in [[LAN-DNS]].
-- Caddy serves HTTPS on 443 and HTTP on 80 (redirects). The router needs no 80/443 port forward for LAN use.
-
-## See Also
-
-- [[LAN-DNS]] — how names resolve on the LAN and WAN
-- [[Network Architecture]] — topology, access policy
-- [[Homepage]]
-- [[Dankclaw Stack]]

docs/Infrastructure/Grafana.md

@@ -1,46 +0,0 @@
-# Grafana
-
-Grafana visualizes the metrics that Prometheus collects from every service. It runs on Dankclaw and is reachable at `grafana.franksdank.cloud` (via Caddy, with basicauth).
-
-**Host:** Dankclaw — **Hostname:** `grafana.franksdank.cloud` — **Port:** 3002 (internal) — **Docs:** https://grafana.com/docs
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/grafana/grafana:latest` |
-| Port | 3002 (host) → 3000 (container) |
-| Data volume | dashboards, users, datasource config |
-| Datasource | Prometheus (on the same host) |
-| Provisioning | dashboards in `/etc/grafana/provisioning/dashboards` |
-
-## Configuration
-
-```yaml
-environment:
-  GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER}"
-  GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
-  GF_SERVER_ROOT_URL: "https://grafana.franksdank.cloud"
-volumes:
-  - /opt/grafana/data:/var/lib/grafana
-  - /opt/grafana/provisioning:/etc/grafana/provisioning:ro
-```
-
-## Datasource
-
-Point Grafana at the Prometheus service by name (`http://prometheus:9090`) on the shared podman network.
-
-## Healthcheck
-
-Grafana responds on `/api/health` with 200 and `"database":"ok"`.
-
-## Gotchas
-
-- Change the default admin credentials at first boot (env or UI).
-- Dashboards provisioned as YAML survive container recreation; UI-created ones live in the data volume — back it up.
-
-## See Also
-
-- [[Dankclaw Stack]]
-- [[Prometheus]] — the datasource
-- [[Observability]] — the whole stack

docs/Infrastructure/Homepage Design Grilling.md

@@ -1,96 +0,0 @@
-# Homepage Design Grilling
-
-The decision log for the homepage design. All questions are settled (2026-08-23). This page records the final decisions and the places where the operator's answer changed the initial recommendation.
-
-## Final decisions
-
-| # | Decision | Value |
-|---|----------|-------|
-| Q1 | How the page learns up/down | Server-side checker writes `status.json`; the page reads only that file. (Documented future intent: client-side probing.) |
-| Q2 | How the checker runs | systemd timer, rewrites `status.json` every 3 minutes. |
-| Q3 | Page model | Single site on Dankclaw. Root = user services (no host labels). `/admin/` = admin services, second operator-only password. |
-| Q5 | Service split | User: SearXNG, ComfyUI, OpenWebUI, Gitea, Calibre-Web, RomM, Copyparty, Plex, Jellyfin. Admin: Hermes, Grafana, Prometheus, Loki, Caddy, dnsmasq, Gluetun, LLM fleet, low-level backends (status-only). The *arr stack is excluded. |
-| Q6 | Tourist `/how/` page | **Defer** — it is extra. |
-| Q7 | `status.json` | One file; each service carries a `category` field; each page filters. |
-| Q8 | `services.json` | One file with a `category` field. |
-| Q9 | Checker language | python3. |
-| Q10 | Probe method | **Per service** — `tcp` or `http`, set per entry. |
-| Q11 | Probe timeout | **5 s** per service. |
-| Q12 | LLM loading | Three states; a listening port is Up (health 503 = Up). |
-| Q13 | Checker location | `scripts/homepage-status.py` + `.service` + `.timer`, tracked in git. |
-| Q14 | Reach backends | `probe` = full `host:port` (`127.0.0.1:PORT` or `192.168.0.235:PORT`). |
-| Q15 | WIP | static `"wip": true`; the checker skips the probe. |
-| Q16 | Staleness | `generated_at` in `status.json`; the page warns when it is older than 10 min. |
-| Q17 | Writes | atomic (write a temp file, then rename). |
-| Q18 | Admin password | `CADDY_ADMIN_HASH` in the host `.env` (git-ignored), operator-only. |
-| Q19 | Per-path basicauth | `@admin path /admin/*` with a second `basicauth` block. |
-| Q20 | Descriptions | hand-written one-liners in `services.json`. |
-| Q21 | Open link | the Caddy site URL; status-only backends set `showLink: false`. |
-| Q22 | Docs link | **a per-service `/docs/` page. Do NOT link the existing reference docs (they may contain secrets).** |
-| Q23 | Layout | **TUI-style cards/panels** (a grid of panels), not a flat list. |
-| Q24 | Summary line | yes — a placeholder for now; refine later. |
-| Q25 | Terminus | bundle the font (woff2) + a monospace fallback. |
-| Q26 | Palette | as proposed (green `#33ff66`, red `#ff5555`, amber `#ffcc33`, cyan `#33ccff`, bg `#050805`). |
-| Q27 | Styling | **toggleable: default = full theme; a toggle strips to bare unstyled HTML (motherfuckingwebsite.com style).** |
-| Q28 | Cursor | **do not alter the user's cursor.** |
-| Q29 | Docs content | how a regular user uses the service; **no maintenance details.** |
-| Q30 | Docs path | **`/docs/<svc>/`** (not `/usage/`). |
-| Q31 | Docs auth | **the same auth as the homepage section** (user docs = shared password, admin docs = admin password). |
-| Q32 | Docs workstream | a separate effort; built as each service goes active. |
-| Q33 | Deploy | **git-driven; master always reflects the deployed state.** |
-| Q34 | `/srv/homepage/` | `deploy.sh` creates it and copies the assets. |
-| Q35 | Timer | enabled at boot. |
-| Q36 | Checker tests | **full TDD.** |
-| Q37 | Page/config tests | **full TDD.** |
-| Q38 | Security test | **full TDD.** |
-| Q39 | Backend move | a config edit (`probe` host:port). |
-| Q40 | LAN timeout | **5 s.** |
-| Q41 | Auto-refresh | **no** — the page reads `status.json` once on load. |
-| Q42 | Mobile | responsive. |
-| Q43 | Footer timestamp | **last master deployment datetime** (not last-check). |
-
-Styling (settled): terminal TUI, Terminus (monospace fallback), green-on-black, 1995 phosphor palette. Default = full theme; a toggle strips to bare unstyled HTML. Do not alter the user's cursor.
-
-## Where the operator changed the recommendation
-
-These are the deltas from the initial recommendations. Everything else was accepted as recommended.
-
-- **Q10** — the probe method is **per service** (`tcp` or `http`), not TCP-only.
-- **Q11 / Q40** — the probe timeout is **5 s** (not 3 s).
-- **Q22** — the docs link points to a **per-service `/docs/` page**, never the existing reference docs (they may contain secrets).
-- **Q23** — the layout is **TUI-style cards/panels**, not a flat list.
-- **Q27** — styling is **toggleable**: the full theme by default, with a toggle to bare unstyled HTML.
-- **Q28** — **do not alter the user's cursor** (no fake blinking cursor).
-- **Q29** — the docs are **user-facing only**; no maintenance details.
-- **Q30** — the docs path is **`/docs/`** (not `/usage/`).
-- **Q31** — the docs use **the same auth as the homepage section**.
-- **Q33** — **git-driven; master always reflects the deployed state.**
-- **Q36–Q38** — **full TDD.**
-- **Q41** — **no client-side auto-refresh.**
-- **Q43** — the footer shows the **last master deployment datetime** (not last-check).
-
-## Build status — complete (2026-08-23)
-
-The build is done, slice by slice, full TDD (each slice red → green, committed
-and pushed). 116 tests pass. The render logic is verified with a Node DOM shim.
-
-```mermaid
-flowchart LR
-    A[write the checker tests] --> B[write the checker]
-    B --> C[write services.json]
-    C --> D[write the pages + theme + toggle]
-    D --> E[extend deploy.sh + Caddyfile]
-    E --> F[tests green]
-    F --> G[deploy to Dankclaw]
-    G --> H[verify in browser]
-```
-
-Slices: 1 the checker core (tcp/wip/atomic/generated_at), 2 the http probe, 3
-the real `services.json` + schema, 4 the pages + theme + toggle, 5 the Caddyfile
-auth split, 6 the git-driven deploy (both hashes, `deployed.js`, the timer).
-
-**Remaining to go live:** the operator adds `CADDY_ADMIN_HASH` (the
-operator-only password, `caddy hash-password`) to the target `.env`, then runs
-`./scripts/deploy.sh dankclaw`. Until then the admin hash stays a placeholder.
-
-The per-service docs pages (Q22/Q29/Q30/Q31) are a separate workstream, built as each service goes active.

docs/Infrastructure/Homepage.md

@@ -1,203 +0,0 @@
-# Homepage
-
-A password-protected web site on Dankclaw lists the services that are up, with a short description of each and a link to both the service and its usage documentation.
-
-**Served by:** Caddy, `file_server` + `forward_auth` (Authentik SSO) — **Docs:** this page
-
-## Status — 2026-08-23: deployed to production
-
-The homepage is live on Dankclaw. `./scripts/deploy.sh dankclaw` copies the static assets to `/srv/homepage/`, installs the status checker (systemd timer, every 3 min, writes `status.json`), writes `deployed.js` (the last master deployment datetime), renders the Caddyfile (the shared basicauth hash from the host `.env`), reloads Caddy, and checks health. 146 tests pass. The Caddyfile auth uses a `route` block with `forward_auth` (see [Auth](#auth)); the caddy container mounts the host's `/srv/homepage` (not the repo source) so the generated files serve.
-
-**Moved to the naked domain (2026-08-24):** the homepage now serves `franksdank.cloud` and `www.franksdank.cloud`. The old names `dankclaw.franksdank.cloud` and `dankserver.franksdank.cloud` redirect to it. Authentication moved from per-page basicauth to Authentik `forward_auth` (see [Authentik](Authentik.md)).
-
-The per-service `/docs/` pages, the *arr stack, and the optional `/how/` page remain separate threads.
-
-## Design decision — 2026-08-22
-
-The live up/down check runs server-side. A checker on Dankclaw probes each backend and writes a same-origin `status.json`. The page fetches only that file and renders the badges. The browser never calls a backend directly. The checker runs as a systemd timer every 3 minutes.
-
-The site has two pages. The root lists user-facing services only, with no host labels. An `/admin/` subpage lists the admin and infrastructure services behind a separate password for the operator. Gitea is a user service. Hermes is an admin service. A "how it is built" page for visitors is deferred (Q6).
-
-The client-side probe design is blocked in this topology. The page origin is `https://franksdank.cloud`. The service URLs are `https://<name>.franksdank.cloud`. That is cross-origin and auth-gated. Both fail a browser `fetch`.
-
-Intended direction: move to client-side probing. Serve every service as a same-origin `https` path under `franksdank.cloud`. Add a health endpoint and CORS headers to each backend. Then the page probes each service directly. Keep this note until that shift is complete.
-
-## Structure
-
-The site has two pages. The root page lists the user services. The `/admin/` page lists the admin and infrastructure services. The optional `/how/` page is deferred (Q6).
-
-```mermaid
-graph TD
-    subgraph site["dankclaw.franksdank.cloud (Caddy)"]
-        ROOT["/  user services<br/>(shared password)"]
-        ADMIN["/admin/  admin services<br/>(operator password)"]
-        HOW["/how/  how it is built<br/>(deferred)"]
-        STATUS["/status.json"]
-    end
-    TIMER["systemd timer<br/>every 3 minutes"]
-    CHECKER["checker script<br/>probes each backend"]
-    BACKENDS["backends on both hosts"]
-    TIMER --> CHECKER
-    CHECKER -->|writes| STATUS
-    STATUS -->|polled| ROOT
-    STATUS -->|polled| ADMIN
-    CHECKER -->|probes| BACKENDS
-```
-
-## Status model
-
-Each service shows one of three states.
-
-```mermaid
-stateDiagram-v2
-    [*] --> Up
-    [*] --> Down
-    [*] --> WIP
-    Up --> Down: probe fails
-    Down --> Up: probe succeeds
-    WIP --> Up: work completes
-```
-
-- **Up** — the probe reached the backend.
-- **Down** — the probe failed (connection refused or timeout).
-- **WIP** — a static override. The service is intentionally not finished. The checker does not probe it.
-
-## Checker
-
-The checker runs on Dankclaw as a systemd timer. Every run probes each backend directly, then writes `status.json`. It probes the backend `host:port`, not the Caddy site. A Caddy `502` or `401` cannot tell up from down. The probe method is per service (`tcp` or `http`). The timeout is 5 s. The page reads `status.json` once on load. It does not auto-refresh.
-
-```mermaid
-flowchart TD
-    A[timer fires] --> B[read service config]
-    B --> C{for each service}
-    C -->|wip| D[mark WIP]
-    C -->|probe| E[probe: tcp or http]
-    E -->|connected| F[mark Up]
-    E -->|refused / timeout| G[mark Down]
-    D --> H[write status.json]
-    F --> H
-    G --> H
-```
-
-## Service classification
-
-**User services** (root page): SearXNG, ComfyUI, OpenWebUI, Gitea, Calibre-Web, RomM, Copyparty, Plex, Jellyfin.
-
-**Admin services** (`/admin/`): Hermes, Grafana, Prometheus, Loki, Caddy, Pi-hole, Gluetun, the LLM fleet, and the low-level backends (mariadb, redis, Postgres, node-exporter, alloy). The backends show status only. They get a link only where a UI is meaningful.
-
-**Excluded for now:** the *arr stack (Radarr, Sonarr, Lidarr, Readarr, Prowlarr, qBittorrent). It is not started. It is a separate effort, possibly on a different host.
-
-## Layout
-
-```
-shared/homepage/
-├── index.html        # root page (user services)
-├── admin.html        # admin + infrastructure services
-├── how.html          # deferred (Q6)
-├── style.css         # full theme; a toggle strips to bare HTML
-├── app.js            # loads status.json once, renders the panels
-├── services.json     # service config (user + admin)
-└── fonts/            # Terminus (woff2) for @font-face
-```
-
-`status.json` is generated at runtime in `/srv/homepage/`. It is not committed.
-
-## Rendering
-
-The page renders a **grid of TUI-style panels** — one panel per service. Each panel shows the state badge, the name, the description, and the `Open` and `Docs` links. A summary line at the top counts the states. The footer shows the last master deployment datetime (Q43).
-
-The page reads `status.json` **once on load**. It does not auto-refresh (Q41).
-
-Styling is **toggleable** (Q27). The default is the full terminal theme. A toggle strips the page to bare, unstyled HTML (motherfuckingwebsite.com style). The page does not alter the user's cursor (Q28).
-
-## Auth
-
-Caddy wraps the site with `forward_auth` (Authentik SSO). A single Authentik
-session covers the whole site: the user page, the `/admin/` page, and the
-shared data + assets. The browser's Authentik cookie is reused for the data
-fetches, so no per-page credential is needed. See [Authentik](Authentik.md) for
-the provider, the outpost, and the first-time setup.
-
-The Caddyfile pattern (one of the forward_auth vhosts):
-
-```
-franksdank.cloud www.franksdank.cloud {
-    root * /srv/homepage
-
-    route {
-        # 1. The outpost path is proxied to Authentik (login UI + session).
-        reverse_proxy /outpost.goauthentik.io/* authentik-server:9000
-
-        # 2. forward_auth checks the session before serving the page.
-        forward_auth authentik-server:9000 {
-            uri /outpost.goauthentik.io/auth/caddy
-            copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Email X-Authentik-Name
-            trusted_proxies private_ranges
-        }
-
-        # 3. The pages, in file order. 'handle' preserves order; a top-level
-        #    'rewrite' would be sorted first and break the path matchers.
-        handle /admin* {
-            try_files /admin.html
-            file_server
-        }
-
-        @data path /services.json /status.json /app.js /style.css /deployed.js
-        handle @data {
-            file_server
-        }
-
-        handle {
-            try_files /index.html
-            file_server
-        }
-    }
-}
-```
-
-A `handle` inline matcher is a single token; the multi-path `@data` match is a
-named matcher. The caddy container mounts the host's `/srv/homepage` (where the
-deploy writes the static files and the checker writes `status.json`), not the
-repo's `shared/homepage`. A compose change needs a container recreate (`podman
-compose up -d caddy`), not a reload. The Caddyfile syntax is documented in the
-agent knowledge base (`~/.pi/agent/docs/caddy/caddyfile.md`).
-
-## Service entry format
-
-```json
-{
-  "name": "SearXNG",
-  "category": "user",
-  "description": "Privacy-respecting metasearch engine.",
-  "url": "http://search.franksdank.cloud",
-  "docs": "/docs/searxng/",
-  "probe": "127.0.0.1:8081",
-  "method": "http",
-  "wip": false,
-  "showLink": true
-}
-```
-
-- `category` — `user` or `admin`. Selects the page.
-- `url` — the "Open" link (the Caddy site).
-- `docs` — the per-service docs page (`/docs/<svc>/`).
-- `probe` — the backend `host:port` the checker probes.
-- `method` — `tcp` or `http`. The probe type for this service (Q10).
-- `wip` — when true, the badge shows WIP and the checker skips the probe.
-- `showLink` — when false, the panel shows status only (no "Open" link).
-
-## Deployment and testing
-
-The deploy is **git-driven**. Master always reflects the currently deployed state (Q33). `deploy.sh` copies the assets to `/srv/homepage/`, installs the checker timer, renders the Caddyfile (both hashes), and reloads Caddy. It stamps the deployment datetime the footer shows (Q43).
-
-The build is **full TDD** (Q36–Q38). Tests exist before the code: the checker (a fake up port, a down port, a WIP service, and the JSON schema), the pages and config (the `services.json` schema, `caddy validate`, the `status.json` reference), and the auth check (the forward_auth block and the renderable basicauth placeholder).
-
-## Per-service docs (separate workstream)
-
-Each service gets a docs page at `/docs/<svc>/` (Q30). The content is a **how-to for a regular user** — no maintenance details (Q29). The pages are password-protected by the same auth as the homepage section they belong to (Q31). They are built as each service goes active (Q32). The `docs` link on a service points to its page. The existing reference docs in `docs/` are not linked from the homepage. They may contain secrets (Q22).
-
-## See Also
-
-- [[Homepage Design Grilling]] — the decision log
-- [[Caddy]] — the proxy that serves it
-- [[Dankclaw Stack]], [[Dankserver Stack]] — the services it lists

docs/Infrastructure/LAN-DNS.md

@@ -1,145 +0,0 @@
-# LAN DNS
-
-The homelab gives every service a name under `franksdank.cloud`. Users type a name; they never type a port number. This page covers how names reach users: the Pi-hole server, the G36 router hand-off, and the split between LAN and WAN resolution.
-
-See [[Caddy]] for the reverse proxy that serves the names, and [[Network Architecture]] for the full topology.
-
-## Components
-
-| Component | Where | Role |
-|-----------|-------|------|
-| Pi-hole | Dankclaw quadlet (`dankclaw/pihole/`) | Answers the flat service names for LAN clients |
-| SURFboard G36 | 192.168.0.1 | Hands dnsmasq to clients as the DHCP DNS server |
-| Porkbun | Registrar for `franksdank.cloud` | Answers the same names for the WAN |
-| Caddy | Dankclaw (192.168.0.143:443) | Serves every name over HTTPS |
-
-## How a name reaches a client
-
-```mermaid
-sequenceDiagram
-    participant C as LAN client (laptop, phone)
-    participant G as G36 router (192.168.0.1)
-    participant D as Pi-hole (192.168.0.143:53)
-    participant P as Porkbun (WAN)
-    participant X as Caddy (192.168.0.143:443)
-
-    C->>G: DHCP request
-    G-->>C: lease + DNS = 192.168.0.143
-    C->>D: dig romm.franksdank.cloud
-    D-->>C: 192.168.0.143
-    Note over D: addn-hosts answer<br/>(dankclaw/pihole/addn-hosts)
-    C->>X: https://romm.franksdank.cloud
-    X-->>C: RomM (192.168.0.235:8085 upstream)
-    C->>P: dig franksdank.cloud (unknown name)
-    P-->>C: 71.227.227.172
-```
-
-Pi-hole answers only the names in `addn-hosts`. Every other query goes to the upstream resolvers `1.1.1.1` and `1.0.0.1`. The zone apex and unknown subdomains fall through to Porkbun. This avoids a wildcard override.
-
-## Why split DNS
-
-The G36 has no NAT loopback (hairpin). A LAN client that resolves a name to the WAN IP and then connects to it would time out. Split DNS removes the hairpin: LAN clients get the LAN IP, WAN clients get the public IP. Both use the same name and the same Let's Encrypt certificate. See [[Caddy]] for the TLS side.
-
-## Pi-hole quadlet
-
-Pi-hole FTL is a dnsmasq fork. It serves the same `addn-hosts` file the retired dnsmasq quadlet used, mounted into `/etc/pihole/hosts/` read-only. It retired the dnsmasq quadlet on 2026-08-24.
-
-- **Location:** `dankclaw/pihole/` — `quadlet/pihole.container`, `env.example`, `addn-hosts`
-- **Ports:** 53/udp + 53/tcp (DNS) and 8083/tcp (web UI) on 192.168.0.143
-- **Host prerequisite 1:** `sysctl net.ipv4.ip_unprivileged_port_start=53` (persisted in `/etc/sysctl.d/99-dnsmasq.conf`)
-- **Host prerequisite 2:** `DNSStubListener=no` in `/etc/systemd/resolved.conf` (the resolved stub would otherwise hold port 53 inside the container netns)
-- **Env file:** `/home/dank/Containers/pihole/env` (never committed) — `ServerIP`, `TZ`, `DNS1` + `DNS2` (pinned upstreams `1.1.1.1` + `1.0.0.1`), `WEBPASSWORD` (the shared family password)
-- **Web UI:** `pihole.franksdank.cloud` (Caddy vhost → 8083), family login
-- **Deploy + restart after a change:** `./scripts/deploy-pihole-quadlet.sh dankclaw`
-
-Add a new service name in two places:
-
-1. `dankclaw/pihole/addn-hosts` — `192.168.0.143 <name>.franksdank.cloud`
-2. Porkbun — A record `<name> 71.227.227.172` (so WAN clients and the DNS-01 challenge work)
-
-Then add the vhost to [[Caddy]] and run `./scripts/deploy.sh dankclaw`. See [[Pi-hole]] for the full configuration.
-
-## G36 router DNS hand-off
-
-The router must tell DHCP clients to use Pi-hole. Until that step is done, new clients get the router's own DNS relay and the flat names do not resolve for them. Existing clients that already use Pi-hole are unaffected.
-
-**Manual step (web GUI at `https://192.168.0.1`):**
-
-1. Open **Gateway → Connection → Local IP Network → IPv4**.
-2. Uncheck **DNS Proxy** (disable the router DNS relay).
-3. Set **DNS** to **Statically configured**.
-4. Set **Primary DNS** to `192.168.0.143`.
-5. Leave the secondary field empty or set `0.0.0.0`.
-6. Apply, then re-lease DHCP on each client (`sudo dhclient -r && sudo dhclient`).
-
-The router admin interface has no programmatic API in this firmware. The web GUI is the only control path. Keep the credentials out of this repo; the repo tracks no router secrets.
-
-Verify after the change:
-
-```
-# on a LAN client
-resolv.conf          # nameserver 192.168.0.143
-dig romm.franksdank.cloud +short
-# expected: 192.168.0.143
-```
-
-## Dankclaw host DNS
-
-The Dankclaw host itself resolves the flat names through Pi-hole:
-
-- `/etc/systemd/resolved.conf`: `DNS=192.168.0.143 192.168.0.1` (Pi-hole first, router as fallback)
-
-Without this, the host would resolve the names through the router to the WAN IP and time out (no hairpin).
-
-## Dankware host DNS
-
-`dankware` is a LAN client, not a Pi-hole server. Its flat names resolve through Pi-hole (`192.168.0.143`), but its resolver setup differs from Dankclaw's because it carries two `+DefaultRoute` links.
-
-- **Two default links:** `enp47s0` (ethernet, metric 100) and `wlan0` (wifi, metric 600), each getting the same DHCP DNS list `[192.168.0.143, 1.1.1.1, …]`.
-- **The flap:** `systemd-resolved` picks a different "current" resolver per link — `enp47s0` settled on the Comcast v6 upstream (→ WAN IP, times out on the LAN), `wlan0` on Pi-hole (→ LAN IP). Flat-name resolution flapped between the two on every query.
-- **The fix:** pin `192.168.0.143` as the static IPv4 resolver on both NetworkManager connections. `resolved.conf`'s own `DNS=` is a per-link fallback, so the link itself must be pinned:
-
-```bash
-for c in "Wired connection 1" FRANKFI-5G; do
-  nmcli connection modify "$c" ipv4.method auto \
-    ipv4.dns 192.168.0.143 \
-    ipv4.ignore-auto-dns yes \
-    ipv4.ignore-auto-routes no
-done
-nmcli connection up "Wired connection 1"
-nmcli connection up FRANKFI-5G
-systemctl restart systemd-resolved
-```
-
-`method:auto` keeps DHCP for the address/gateway/route (dual-default traffic path preserved); `ignore-auto-dns=yes` uses only the static resolver; `ignore-auto-routes=no` keeps the DHCP routes. Verify with a cache-clear loop — `romm.franksdank.cloud` should return `192.168.0.143` on every run:
-
-```bash
-for i in $(seq 1 10); do resolvectl flush-caches; dig romm.franksdank.cloud +short; done
-```
-
-The apex and unknown subdomains still fall through to Porkbun (→ `71.227.227.172`), so WAN clients and a bare `dig franksdank.cloud` keep working.
-
-## WAN records (Porkbun)
-
-The Porkbun zone `franksdank.cloud` carries an A record for every vhost plus the apex, all pointing at `71.227.227.172`. TTL is 300. The zone has no wildcard record: unknown subdomains return NXDOMAIN. Caddy's DNS-01 challenges add and remove `_acme-challenge.*` TXT records through the same API during issuance. See [[Caddy]] for the TLS flow.
-
-`franksdank.casa` is not in this Porkbun account. The `plex.franksdank.casa` vhost stays plain HTTP for that reason. The canonical Plex name is `plex.franksdank.cloud` (HTTPS).
-
-## Failure modes
-
-| Symptom | First check |
-|---------|-------------|
-| Name does not resolve on a new client | `resolv.conf` shows 192.168.0.1, not .143 — the G36 hand-off step is missing |
-| Name resolves to 71.227.x and times out on the LAN | Client used the router DNS — same root cause |
-| Flat name flaps between 192.168.0.143 and 71.227.x on one client | Two `+DefaultRoute` links; resolved picks a different "current" resolver per link. Pin 192.168.0.143 as static DNS on both NM connections — see [[#Dankware host DNS]] |
-| `dig @192.168.0.143` fails entirely | Pi-hole quadlet down: `systemctl --user status pihole` |
-| Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`). FTL serves it. Use the FQDN, or an `/etc/hosts` entry on the client |
-| HTTPS works from the host but not the WAN | No port forward on the G36 for that service (manual router step) |
-| Cert renewal fails in Caddy logs | Porkbun API credentials in the Dankclaw `.env`; test with the Porkbun API `/ping` endpoint |
-
-## See Also
-
-- [[Pi-hole]] — the LAN DNS server (quadlet, records, web UI)
-- [[Caddy]] — TLS, vhost list, DNS-01
-- [[Network Architecture]] — topology, access policy
-- [[Dankclaw Stack]] — the host that carries Pi-hole and Caddy

docs/Infrastructure/Loki.md

@@ -1,51 +0,0 @@
-# Loki
-
-Loki is the log aggregation backend of the observability stack. It stores and queries container and system logs, and it feeds the log panels in Grafana. It replaces the need for a full ELK stack.
-
-**Host:** Dankclaw — **Docs:** https://grafana.com/docs/loki/latest
-
-## Facts (from research)
-
-| Item | Value |
-|------|-------|
-| Image | `franksdank/loki:latest` (local build with a static busybox healthcheck) |
-| Port | 3100 (host) → 3100 (container) |
-| Mode | single-binary (all components in one process) |
-| Storage | filesystem (bind mount) |
-| Auth | `auth_enabled: false` (single-tenant homelab) |
-| Schema | v13 / TSDB |
-
-## Configuration
-
-Config lives at `shared/loki/loki-config.yaml`. Key settings:
-
-- `schema_config` v13 with `store: tsdb` and filesystem object store.
-- `compactor` with `retention_enabled: true` so logs do not grow without bound.
-- `replication_factor: 1` for a single instance.
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/loki` | `/opt/loki/data` |
-| `/etc/loki/loki-config.yaml` | `shared/loki/loki-config.yaml` (bind mount) |
-
-## Healthcheck
-
-Loki responds on `/ready` with 200.
-
-## Log Shipping
-
-The [[Alloy]] agent on each host ships container and journal logs to Loki. Grafana queries Loki through the datasource in [[Observability]].
-
-## Gotchas
-
-- Filesystem storage grows; enable compactor retention.
-- Loki and Promtail/Alloy must agree on the push URL: `http://loki:3100/loki/api/v1/push`.
-- For multi-host shipping, Loki's HTTP listener binds `0.0.0.0`.
-
-## See Also
-
-- [[Observability]]
-- [[Alloy]] — the log shipper
-- [[Grafana]] — the query frontend

docs/Infrastructure/Node-Exporter.md

@@ -1,55 +0,0 @@
-# Node-Exporter
-
-Node-Exporter exposes host metrics (CPU, RAM, disk, network) to Prometheus. One instance runs per host inside its own container.
-
-**Hosts:** Dankserver + Dankclaw — **Docs:** https://prometheus.io/docs/guides/node-exporter
-
-## Facts (from research)
-
-| Item | Value |
-|------|-------|
-| Image | `prom/node-exporter:latest` |
-| Port | 9100 (host) |
-| Network | `host` mode |
-| Config | `--path.procfs=/host/proc`, `--path.sysfs=/host/sys`, `--path.rootfs=/rootfs` |
-
-## Configuration
-
-```yaml
-services:
-  node-exporter:
-    image: prom/node-exporter:latest
-    restart: always
-    network_mode: host
-    pid: host
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    command:
-      - --path.procfs=/host/proc
-      - --path.sysfs=/host/sys
-      - --path.rootfs=/rootfs
-```
-
-## Prometheus Targets
-
-Prometheus scrapes each host's node-exporter directly:
-
-| Host | Target |
-|------|--------|
-| Dankserver | `192.168.0.235:9100` |
-| Dankclaw | `192.168.0.143:9100` |
-
-These scrape jobs live in the Prometheus config; see [[Prometheus]] and [[Observability]].
-
-## Gotchas
-
-- Requires `network_mode: host` (or a published port) so Prometheus can reach 9100.
-- The `/proc`, `/sys`, and root mounts must be `read-only` (`:ro`).
-- Do not mount the same paths on both hosts' containers; each node-exporter reflects only its own host.
-
-## See Also
-
-- [[Prometheus]] — the scraper
-- [[Observability]] — the full stack

docs/Infrastructure/Observability.md

@@ -1,53 +0,0 @@
-# Observability
-
-The observability stack runs on Dankclaw: Grafana for dashboards, Prometheus for metrics, and log aggregation for container logs. This is a small homelab — no Kubernetes.
-
-## Stack
-
-```mermaid
-graph LR
-    S[Services] -->|/metrics| P[Prometheus]
-    S -->|logs| T[Alloy]
-    T --> L[Loki]
-    P --> G[Grafana]
-    L --> G
-```
-
-| Component | Role | Image | Port |
-|-----------|------|-------|------|
-| [[Prometheus]] | metrics collection | `prom/prometheus` | 9090 |
-| [[Grafana]] | dashboards | `grafana/grafana` | 3002 |
-| Loki | log storage & query | `grafana/loki` | 3100 |
-| Alloy | log shipping (per host) | `grafana/alloy` | — |
-
-## Log Aggregation Choice
-
-Loki + Alloy is the lightweight Grafana-native option for a two-server homelab:
-
-- Alloy reads container logs from the host (journald socket + container log files) and pushes them to Loki. It replaces Promtail, which reached end-of-life on 2026-03-02.
-- Loki stores compressed logs with labels; queries run in Grafana's Explore.
-- No Elasticsearch memory cost, no Kubernetes required.
-
-Alternatives considered and rejected: ELK (too heavy), Promtail (EOL), VictoriaLogs (newer, less documentation), plain `journald` (no centralized query).
-
-## Scrape Surface
-
-| Target | Source |
-|--------|--------|
-| Prometheus self | `localhost:9090/metrics` |
-| ComfyUI | `/system_stats` (limited) or blackbox probe |
-| Gitea | `/metrics` (built-in exporter) |
-| Grafana | `/metrics` (built-in) |
-| Host metrics | node-exporter `:9100` |
-
-For services without native exporters, use the Prometheus **blackbox exporter** in probe mode to assert up/down and HTTP latency. The homepage status also probes the same endpoints.
-
-## Healthchecks
-
-Every compose service gets a healthcheck so podman reports container health and Promtail/Grafana can alert on it. See [[Deployment Flow]] for the verify step.
-
-## See Also
-
-- [[Grafana]], [[Prometheus]]
-- [[Dankclaw Stack]]
-- [[Homepage]] — the user-facing status surface

docs/Infrastructure/Pi-hole.md

@@ -1,73 +0,0 @@
-# Pi-hole
-
-Pi-hole is the LAN DNS server for the homelab. It runs as a quadlet container on Dankclaw (192.168.0.143). It answers the flat `franksdank.cloud` / `franksdank.casa` service names and filters ad traffic on the LAN. It retired the dnsmasq quadlet on 2026-08-24. See [[LAN-DNS]] for the name-resolution flow and the router hand-off.
-
-## Components
-
-| Component | Where | Role |
-|-----------|-------|------|
-| `pihole` quadlet | `dankclaw/pihole/quadlet/pihole.container` | Boot-persistent container unit |
-| FTL | `docker.io/pihole/pihole:latest` | DNS engine (a dnsmasq fork) + ad blocking |
-| `addn-hosts` | `dankclaw/pihole/addn-hosts` | One A record per Caddy vhost |
-| sysctl drop-in | `dankclaw/pihole/sysctl/99-dnsmasq.conf` (repo) → `/etc/sysctl.d/` | Rootless port-53 threshold, applied at boot |
-| env file | `/home/dank/Containers/pihole/env` (host) | `ServerIP`, `TZ`, `DNS1`, `DNS2`, `WEBPASSWORD` |
-| Web UI | `pihole.franksdank.cloud` (Caddy → 8083) | Dashboard + admin, family login |
-
-## Quadlet
-
-- **Install path on Dankclaw:** `~/.config/containers/systemd/pihole.container` (unit `pihole.service`)
-- **Network:** `slirp4netns` (the same pattern as open-webui and gitea)
-- **Published ports:** 53/udp + 53/tcp (DNS), 8083/tcp (web UI)
-- **Mounts:**
-  - `/home/dank/Containers/pihole/etc-pihole` → `/etc/pihole` (config + state, preserved across restarts)
-  - `/home/dank/Containers/pihole/etc-dockers` → `/etc/dockers` (blocklists)
-  - `/home/dank/Containers/pihole/addn-hosts` → `/etc/pihole/hosts/addn-hosts` (read-only; the record file from this repo)
-- **Restart:** `Restart=always`, `RestartSec=5`
-- **Deploy:** `./scripts/deploy-pihole-quadlet.sh dankclaw` (idempotent; preserves the data dirs)
-
-## Records
-
-FTL does not read `/etc/dnsmasq.d/` — its generated `dnsmasq.conf` has no `conf-dir`. The only file-based local-record mechanism is `hostsdir=/etc/pihole/hosts`: FTL parses every file in that directory as an `/etc/hosts` file. The quadlet therefore mounts the repo record file there, read-only:
-
-```
-/home/dank/Containers/pihole/addn-hosts -> /etc/pihole/hosts/addn-hosts
-```
-
-The file format is hosts-style (`<IP> <name> [<name> ...]`), one A record per Caddy vhost, all pointing at `192.168.0.143`. The file stays in sync with the Caddyfile (`tests/pihole.bats` cross-checks the two). Add a name there and in Porkbun, then deploy. See [[LAN-DNS]] for the two-place rule.
-
-**Pitfall (2026-08-24):** the first quadlet mounted the file into `/etc/dnsmasq.d/`. FTL never read it; names only resolved from the query cache. A fresh name returned NXDOMAIN. Mount into `/etc/pihole/hosts/`, not `/etc/dnsmasq.d/`.
-
-## Upstream
-
-The slirp netns has no useful `resolv.conf`. The env file pins the upstreams: `DNS1=1.1.1.1`, `DNS2=1.0.0.1` (mirrors the retired `dnsmasq.conf` `server=` lines). Every query that is not a local record goes to those resolvers. The zone apex and unknown subdomains fall through to Porkbun.
-
-## Web UI and API
-
-- **URL:** `https://pihole.franksdank.cloud/` (Caddy vhost, family basicauth → 8083)
-- **Password:** `WEBPASSWORD` in the env file (the shared family password, the same credential as `CADDY_BASICAUTH_HASH`)
-- **API:** `https://pihole.franksdank.cloud/api/...` — authenticate with the session cookie, then query the blocklist, the local DNS records, and the query log
-- **Port on the LAN:** `http://192.168.0.143:8083/` (direct, no Caddy)
-
-## Host prerequisites
-
-1. `net.ipv4.ip_unprivileged_port_start=53`, persisted in `/etc/sysctl.d/99-dnsmasq.conf` — rootless Podman may then publish port 53. The repo file `dankclaw/pihole/sysctl/99-dnsmasq.conf` is the single source of truth: `deploy_pihole` installs it, stages any drop-in that sets a value above 53 in the home trash, comments out a stale active line in `/etc/sysctl.conf`, and applies everything with `sysctl --system` (the same code path boot uses via systemd-sysctl). **Incident (2026-08-26):** the value was reset to 80 by *two* sources — a later-sorted drop-in (`99-rootless-ports.conf`) and a stale line in `/etc/sysctl.conf`, which systemd-sysctl processes **after** all of `sysctl.d/` and therefore overrides every drop-in. The runtime override from an earlier manual `sysctl -w` died with a reboot, and `pihole.service` crash-looped on `bind: permission denied` for port 53 (restart counter 1645) until both conflicts were removed.
-2. `DNSStubListener=no` in `/etc/systemd/resolved.conf` — the resolved stub would otherwise hold port 53 inside the container netns.
-
-Both prerequisites survive the dnsmasq → Pi-hole swap. Do not remove them.
-
-## Failure modes
-
-| Symptom | First check |
-|---------|-------------|
-| `dig @192.168.0.143` fails entirely | Quadlet down: `systemctl --user status pihole` |
-| A name does not resolve for a new client | The client's `resolv.conf` does not show `192.168.0.143` — the G36 hand-off is missing. See [[LAN-DNS]] |
-| The web UI asks for a password loop | `WEBPASSWORD` in the env file is the placeholder `changeme` — fill the real value, restart the unit |
-| Upstream queries fail (local names still answer) | `DNS1`/`DNS2` in the env file are missing or wrong |
-| Port 53 `bind: permission denied` crash loop after a reboot | sysctl conflict: the value must be 53 at boot, and remember `/etc/sysctl.conf` is applied **after** all of `sysctl.d/`. Redeploy pihole — `deploy_pihole` reinstalls the drop-in, neutralizes conflicting drop-ins, and comments out a stale `/etc/sysctl.conf` line |
-| Single-label name (for example `dankclaw`) answers `127.0.1.1` | The rootless container `/etc/hosts` carries the host entry (`127.0.1.1 <host>`), and FTL serves it. Use the FQDN, or add an `/etc/hosts` entry on the client |
-
-## See Also
-
-- [[LAN-DNS]] — name resolution flow, router hand-off, split DNS
-- [[Caddy]] — the `pihole.franksdank.cloud` vhost
-- [[Dankclaw Stack]] — the host that carries Pi-hole

docs/Infrastructure/Prometheus.md

@@ -1,57 +0,0 @@
-# Prometheus
-
-Prometheus collects metrics from every service on Dankclaw by scraping their `/metrics` endpoints. It feeds the Grafana dashboards. The `node-exporter` containers may add podman/cgroup metrics for full host visibility.
-
-**Host:** Dankclaw — **Hostname:** `prometheus.dankclaw.local` — **Port:** 9090 — **Docs:** https://prometheus.io/docs
-
-## Facts
-
-| Item | Value |
-|------|-------|
-| Image | `docker.io/prom/prometheus:latest` |
-| Port | 9090 (host) → 9090 (container) |
-| Config | `/etc/prometheus/prometheus.yml` (bind mount) |
-| Data volume | TSDB |
-| Retention | `--storage.tsdb.retention.time=30d` |
-
-## prometheus.yml Targets
-
-```yaml
-scrape_configs:
-  - job_name: "prometheus"
-    static_configs:
-      - targets: ["localhost:9090"]
-
-  - job_name: "comfyui"
-    static_configs:
-      - targets: ["comfyui:8188"]
-
-  - job_name: "node"
-    static_configs:
-      - targets: ["node-exporter:9100"]
-```
-
-Allow the `/metrics` endpoint on each service where the exporter exists. For services without a native exporter, use the Prometheus `blackbox` exporter (probe mode) to check up/down and latency.
-
-## Volumes
-
-| Container path | Host path (convention) |
-|----------------|------------------------|
-| `/etc/prometheus` | `/opt/prometheus/config` |
-| `/prometheus` | `/opt/prometheus/data` |
-
-## Healthcheck
-
-Prometheus responds on `/-/healthy` with 200.
-
-## Gotchas
-
-- Keep scrapes modest for a small homelab; too many targets bloat the TSDB.
-- Use recording rules for heavy queries (see [[Grafana]]).
-- Set a retention policy or the data volume grows without bound.
-
-## See Also
-
-- [[Dankclaw Stack]]
-- [[Grafana]] — the dashboard frontend
-- [[Observability]] — the whole stack

docs/Infrastructure/authentik-postgres.md

@@ -1,14 +0,0 @@
-# authentik-postgres
-
-The PostgreSQL 16 database for [Authentik](Authentik.md) — the homelab's SSO /
-OIDC provider. It is the only stateful store in the stack (users, flows,
-sessions, events). Redis was removed in Authentik 2025.10; sessions and caching
-now live here.
-
-- **Compose service:** `authentik-postgres` (in `dankclaw/compose.yaml`)
-- **Image:** `docker.io/library/postgres:16-alpine`
-- **Data:** `/home/dank/authentik/postgres` (bind mount; no named volumes)
-- **Health:** `pg_isready -d authentik -U authentik`
-
-See [Authentik.md](Authentik.md) for the full architecture, setup, backups
-(`pg_dump`), and failure modes.

docs/Infrastructure/authentik-server.md

@@ -1,15 +0,0 @@
-# authentik-server
-
-The Authentik server — the core, the admin API, and the embedded Caddy outpost.
-This is the service the Caddy `forward_auth` vhosts proxy to
-(`authentik-server:9000`). It also serves the login page at
-`https://auth.franksdank.cloud`.
-
-- **Compose service:** `authentik-server` (in `dankclaw/compose.yaml`)
-- **Image:** `ghcr.io/goauthentik/server:2026.8.0` (command: `server`)
-- **Port:** `9000` (published; the outpost + admin + login all live here)
-- **Data:** `/home/dank/authentik/data` (bind mount, shared with the worker)
-- **Health:** `http://localhost:9000/-/health/`
-
-See [Authentik.md](Authentik.md) for the full architecture, the Caddy
-forward_auth pattern, first-time setup, and failure modes.

docs/Infrastructure/authentik-worker.md

@@ -1,14 +0,0 @@
-# authentik-worker
-
-The Authentik worker — background tasks: event processing, provider
-synchronization, scheduled flows, and certificate renewal. It runs the same
-image as the server; the `command: worker` selects the role.
-
-- **Compose service:** `authentik-worker` (in `dankclaw/compose.yaml`)
-- **Image:** `ghcr.io/goauthentik/server:2026.8.0` (command: `worker`)
-- **Runs as:** `0:0` (rootless: container root maps to the host user that owns the data dir)
-- **Data:** `/home/dank/authentik/data` (bind mount, shared with the server)
-- **No port** — the worker has no HTTP endpoint.
-
-See [Authentik.md](Authentik.md) for the full architecture, setup, and failure
-modes.

docs/LLM/DFlash DSpark Speculative Decoding.md

@@ -1,218 +0,0 @@
-# DFlash and DSpark Speculative Decoding
-
-## Overview
-
-DFlash and DSpark are the two newest speculative decoding methods. They draft many tokens in one parallel pass instead of one token at a time. This reduces decode latency on memory-bound workloads.
-
-DFlash comes from Z Lab at UC Berkeley. DSpark comes from DeepSeek. Both work with vLLM and llama.cpp. Both have draft models for the Qwen3.6 models that Dankserver and Dankclaw run.
-
-## How DFlash Works
-
-DFlash is a lightweight block diffusion model. It predicts a block of tokens in a single forward pass. Key design points:
-
-- The draft model reuses the embedding and LM head from the target model.
-- Only the intermediate layers are trained.
-- It fuses target-model hidden features into the KV projections of every draft layer.
-- A 16-token DFlash drafter has lower latency than a 1-layer EAGLE-3 drafter producing 8 tokens.
-
-Results from the DFlash paper (arXiv:2602.06036):
-
-- Over 6x lossless speedup on Qwen3-8B.
-- Up to 2.5x higher speedup than EAGLE-3.
-- Diffusion drafting cost is flat, so block size does not increase cost.
-
-## How DSpark Works
-
-DSpark is a semi-autoregressive extension of DFlash from DeepSeek (arXiv:2607.05147). It keeps the DFlash parallel diffusion backbone and adds a lightweight sequential Markov head. This models inter-token dependencies inside each block. It also uses confidence-scheduled verification, which stops verifying tokens that have a high rejection risk.
-
-Results from the DSpark paper:
-
-- 60-85% faster per-user generation than MTP-1 on DeepSeek-V4 production traffic.
-- Useful when parallel drafts decay in acceptance rate.
-
-Both DFlash and DSpark are lossless under rejection sampling. Use the strict rejection sampler.
-
-## Runtime Support
-
-| Runtime | DFlash | DSpark | MTP |
-|---------|--------|--------|-----|
-| vLLM | yes (v0.20.1+) | yes | yes |
-| llama.cpp | yes | yes | yes |
-| SGLang | yes | no | yes |
-
-### vLLM
-
-vLLM v0.20.1+ has core DFlash support. Use the `--speculative-config` JSON object.
-
-DFlash on vLLM:
-
-```
-vllm serve Qwen/Qwen3.6-35B-A3B \
-    --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 15}' \
-    --attention-backend flash_attn \
-    --max-num-batched-tokens 32768
-```
-
-Legacy flags also work:
-
-```
-vllm serve Qwen/Qwen3.6-35B-A3B \
-    --spec-method dflash \
-    --spec-model z-lab/Qwen3.6-35B-A3B-DFlash \
-    --spec-tokens 8
-```
-
-DFlash requires a non-causal attention backend. Set `"attention_backend": "flash_attn"` in the speculative config if needed. On ROCm, check that the flash_attn backend is available before using DFlash.
-
-DSpark on vLLM:
-
-```
-vllm serve <target-model> \
-    --speculative-config '{"method": "dspark", "model": "<dspark-draft>", "num_speculative_tokens": 15}'
-```
-
-### llama.cpp
-
-llama.cpp supports DFlash, DSpark, and MTP through `--spec-type`. When you pass `--spec-draft-hf`, llama.cpp auto-discovers the sidecar files and infers the type. The priority order is DSpark, DFlash, EAGLE-3, MTP.
-
-`--spec-type` accepts a **comma-separated list** of speculative implementations. `common_speculative_init` (in `common/speculative.cpp`) builds a vector of implementations and chains drafting through all of them (`common_speculative_draft_params.drafting` is "used to chain the drafts through all the available implementations"). This means you can **layer ngram-mod on top of a DFlash draft model** in one server process:
-
-```
-llama-server --hf-repo <target-GGUF> \
-    --spec-draft-hf <dflash-draft-GGUF> \
-    --spec-type draft-dflash,ngram-mod
-```
-
-The DFlash drafter proposes, then ngram-mod runs as an additional implementation in the chain. Verified against upstream `common/speculative.cpp` (b10133+).
-
-DFlash on llama.cpp:
-
-```
-llama-server --hf-repo <target-GGUF> \
-    --spec-draft-hf <draft-GGUF> \
-    --spec-type draft-dflash \
-    --spec-draft-n-max 16
-```
-
-DSpark on llama.cpp:
-
-```
-llama-server --hf-repo <target-GGUF> \
-    --spec-draft-hf <draft-GGUF> \
-    --spec-type draft-dspark
-```
-
-MTP on llama.cpp:
-
-```
-llama-server --hf-repo <target-with-MTP> \
-    --spec-type draft-mtp \
-    --spec-draft-n-max 2
-```
-
-Note: the flag was renamed from `--spec-type mtp` to `--spec-type draft-mtp` in May 2026 (PR #22673, build b9180+). An old flag silently disables speculative decoding.
-
-### MTP drafter availability for Qwen3.6-27B
-
-MTP is not a separate drafter model. It is a "nextn" head that must live in the target GGUF (or be loaded as a draft alongside it). llama.cpp's `draft-mtp` implementation is fully wired for the `qwen35` and `qwen35moe` architectures (PR #22673).
-
-Local check on Dankclaw (2026-08-13), tensor listing via GGUF header parse:
-
-| File | Architecture | Tensors | MTP tensors |
-|------|--------------|---------|-------------|
-| `Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf` | `qwen35` | 851 | 0 |
-| `Qwen3.6-27B-DFlash-Q8_0.gguf` (draft) | `dflash` | 58 | 0 |
-
-The HauhauCS uncensored GGUFs ship without the MTP head. To use MTP with them, load an MTP-only draft GGUF from the base Qwen3.6-27B model as a sidecar:
-
-```
-llama-server --model <HauhauCS-Q8_K_P.gguf> \
-    --spec-draft-model <MTP-only-draft.gguf> \
-    --spec-type draft-mtp \
-    --spec-draft-n-max 2
-```
-
-Compatible MTP-only draft GGUFs on Hugging Face:
-
-| Repo | Notes |
-|------|-------|
-| `a4lg/Qwen3.6-27B-MTP-ONLY-GGUF` | MTP-only subset of the base model; pairs with derivative/abliterated targets via `--spec-draft-model` |
-| `lym00/Qwen3.6-27B-MTP-ONLY-GGUF` | Same purpose; built from `convert_hf_to_gguf.py --mtp` |
-| `ggml-org/Qwen3.6-27B-MTP-GGUF` | Full model with MTP head (replaces the target, not a sidecar) |
-| `hyperhuzaifa/Qwen3.6-27B-Uncensored-MTP-GGUF` | HauhauCS Q4_K_P with the MTP head grafted in (replaces the target) |
-
-Speculative decoding is lossless under rejection sampling: the target verifies every draft token. A draft head from the base model works with a derivative target; wrong proposals just lower the acceptance rate. The accepted MTP draft lengths are 2-3 tokens; the "MTP Defaults Are a Trap" writeup (thefrontierlab.ai) documents the tuning pitfalls.
-
-On Dankclaw, `draft-dflash` remains the benchmarked winner (20.2 t/s at 4k vs 16.9 for DFlash+ngram and 8.7 for ngram-only). MTP is a viable alternative to test, not a bug fix: the parallel-stream releases investigated on 2026-08-13 were client-side aborts and a Dankserver crash, not speculative decoding behavior (see [[deployment]]).
-
-## Draft Models
-
-DFlash draft model sizes for the models on these servers:
-
-| Target model | DFlash draft | Draft size |
-|--------------|--------------|------------|
-| Qwen3.6-35B-A3B | z-lab/Qwen3.6-35B-A3B-DFlash | 0.4B |
-| Qwen3.6-27B | z-lab/Qwen3.6-27B-DFlash | 2B |
-| Qwen3.5-35B-A3B | z-lab/Qwen3.5-35B-A3B-DFlash | 0.4B |
-| Qwen3.5-27B | z-lab/Qwen3.5-27B-DFlash | 2B |
-
-The DFlash draft is not a standalone model. It must pair with its target model. The model cards recommend block size 8 for higher concurrency and 16 for single-stream throughput.
-
-## Deployment on Dankserver (NVIDIA RTX 3090)
-
-The Dankserver llama.cpp deployment (`frank-default-small`, Qwen3.6-35B-A3B) is **active** on Dankserver (port 7770) as of the 2026-08-17 redo (the earlier 2026-08-16/17 retirement of this service was reversed when the PP split was retired). The vLLM material below documents the archived AWQ/DFlash path on the same hardware.
-
-> **DFlash retired from active service.** `DFlash` (draft-dflash) was tested initially but did not deliver the desired speed gains under the `frank-default-small` workload (see ngram-mod benchmarks below). The `Alittlehammmer/Qwen3.6-35B-A3B-DFlash` draft GGUF remains in `/home/dank/models/` as reference material only.
-
-The vLLM material below documents the archived AWQ/DFlash path on the same hardware. The RTX 3090 is Ampere (compute capability 8.6). Use these conditions:
-
-- FP8 KV cache is valid on CUDA 12.
-- FP8 weights compute through a fallback path; no native FP8 tensor cores.
-- The DFlash draft model (0.4B) fits easily in 24 GB VRAM alongside the target.
-
-Working DFlash flags for this server:
-
-```
---spec-method dflash
---spec-model z-lab/Qwen3.6-35B-A3B-DFlash
---spec-tokens 8
---kv-cache-dtype fp8
-```
-
-Example DFlash + AWQ on the RTX 3090:
-
-```
-docker run --rm --name vllm-qwen36-dflash \
-    --gpus all --ipc=host \
-    -p 8000:8000 \
-    -v ~/.cache/huggingface:/root/.cache/huggingface \
-    docker.io/vllm/vllm-openai:latest \
-    cyankiwi/Qwen3.6-35B-A3B-AWQ-4bit \
-    --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3.6-35B-A3B-DFlash", "num_speculative_tokens": 8}' \
-    --attention-backend flash_attn \
-    --gpu-memory-utilization 0.95
-```
-
-## Deployment on Dankclaw (AMD Strix Halo gfx1151)
-
-`frank-default-large` on Dankclaw runs the llama.cpp pipeline-parallel head (Qwen3.8-27B-Uncensored Q6_K) on port 7770, RPC-split with the Dankserver worker. Spec is the standalone MTP draft over the ngram-mod fallback, 1 slot at 262144 ctx, q8_0/f16 KV, head GPU side ROCm0. See [[qwen3.8-spec-matrix]] and [[Qwen3.8 PP Deployment]] for the current state and the 2026-08-15 Vulkan0/ROCm0 high-context bench.
-
-History: before `--spec-default`, the box ran DSpark (`draft-dspark`, retired 2026-08-15 after a ROCm illegal-memory-access crash under two concurrent 16k prompts) and before that bundled MTP (`draft-mtp`, retired same day). DFlash was retired on 2026-08-14 (premature-EOS under parallel load). The old DFlash draft GGUF (`Qwen3.6-27B-DFlash-Q8_0.gguf`), the MTP-only drafter (`Qwen3.6-27B-MTP-ONLY-Q8_0.gguf`), and the DSpark draft (`Qwen3.8-27B-DSpark-Q8_0.gguf`) remain in `/home/dank/models/` as reference only.
-
-## Notes
-
-- Speculative decoding helps low-QPS, memory-bound, latency-focused workloads. It helps less at high QPS.
-- DFlash needs a non-causal attention backend; the ROCm flash_attn backend must be validated before production use.
-- The old `--swap-space` flag no longer exists in current vLLM. Use `--kv-offloading-size`.
-- `--speculative-config` is the preferred CLI. Legacy `--spec-method`, `--spec-model`, and `--spec-tokens` flags still exist.
-- Use `rejection_sample_method` default `strict` for lossless output.
-
-## See Also
-
-- [[Quantization by Hardware Architecture]]
-- [[Running Qwen3.6 35B A3B FP8 on Dankserver]]
-- [[Running Qwen3.6 27B FP8 on Dankclaw]]
-- [[Dankclaw System Specs]]
-- [[Dankserver System Specs]]
-- [[VLLM Overview]]
-- [[deployment]]

docs/LLM/Dankclaw Improvements.md

@@ -1,113 +0,0 @@
-# Dankclaw — Improvements Research
-
-This page collects concrete, sourced recommendations to improve Dankclaw
-(the AMD Strix Halo inference box). It covers self-hosted agentic tools,
-inference runtime tuning, and server/OS/hardware settings. Each item lists a
-priority and the exact action.
-
-Dates: researched 2026-08. Hardware is verified on the live machine unless
-marked "verify first". Re-verified 2026-08-15/16; corrections marked below.
-
-## Corrections (re-verified 2026-08-15/16)
-
-- **Backend**: production is **Vulkan0**, not ROCm. Measured on this box
-  (b10433): Vulkan 25.13 t/s vs ROCm 24.33 t/s decode at 16k; at 262k ctx
-  Vulkan decodes 48% faster (5.68 vs 3.83 t/s). ROCm wins prefill (410 vs
-  302 t/s @64k). The "HIP out-performs Vulkan for decode" claim did not
-  hold on the 8060S. See [[frank-default-large-benchmark]].
-- **Power profile**: the `/sys/class/drm/card0/device/power_profile` file
-  does not exist on this machine (checked 2026-08-15). GPU DPM is already
-  `performance`, forced level `auto`. This item is N/A.
-- **KV cache**: production runs `q8_0` K **and** `q8_0` V (both quantized).
-- **Spec**: DFlash/DSpark are retired. Production is `--spec-default`
-  (ngram-mod) with draft-mtp first. See [[qwen3.8-spec-matrix]].
-- **TTM**: raised to 114 GiB on 2026-08-25, set live (no reboot) and
-  persisted in `/etc/modprobe.d/ttm.conf`. The effective VRAM ceiling that
-  once held the service at ~15 GiB is gone.
-
-## Verified Memory Picture (measured 2026-08-11)
-
-| Figure | Value | Source |
-| ------ | ----- | ------ |
-| OS physical RAM | 30.5 GiB | `/proc/meminfo` |
-| TTM pages_limit | 114 GiB (29,883,536 x 4KiB) — raised 2026-08-25 | `/sys/module/ttm/parameters/pages_limit` |
-| ROCm VRAM total | 96.0 GiB | `rocm-smi --showmeminfo vram` |
-| ROCm VRAM used | 76.2 GiB | `rocm-smi` while serving 3x200700 |
-
-The OS accounts only ~30.5 GiB physical RAM. The TTM GPU-addressable pool
-was capped at ~15.24 GiB (the real pre-2026-08-25 ceiling); that cap is now
-lifted to 114 GiB. ROCm/SMI still reports a 96 GiB VRAM carve-out (VGM) — a
-different, driver-reported figure that has always been larger than the kernel
-cap.
-This split is the Strix Halo GTT/VGM design. It drives the batch/ubatch memory
-limits documented in [[frank-default-large-benchmark]].
-
-## 1. Self-Hosted Agentic Tools
-
-All of these connect to the local llama.cpp OpenAI-compatible endpoint
-(`http://localhost:7770/v1`). None need NVIDIA.
-
-| Tool | Use | Endpoint config | Priority |
-| ---- | --- | --------------- | -------- |
-| Open WebUI | Chat + agent harness, plugins, RAG, tool calling | Add OpenAI-compatible provider -> `http://localhost:7770/v1` | High |
-| Cline (VS Code) | IDE coding agent with MCP | Provider "OpenAI Compatible", `API_URL=http://localhost:7770/v1` | High |
-| Aider (CLI) | Terminal pair-programming agent | `OPENAI_API_BASE=http://localhost:7770/v1` | Medium |
-| Continue.dev (VS Code) | Open-source coding agent | `config.json` -> `{type:"openai", apiBase:"http://localhost:7770"}` | Medium |
-| OpenHands (Agent Canvas) | Self-hosted multi-agent coding platform | LiteLLM custom model + Base URL to the endpoint | Medium |
-| AnythingLLM | RAG / knowledge chat | Provider "OpenAI (Generic)" -> endpoint | Medium |
-| Browser Use | Browser automation agent (Playwright) | `ChatOpenAI(base_url="http://localhost:7770/v1")` | Low |
-| Qdrant | Vector store for RAG | Standalone, CPU-native | Low |
-
-The pragmatic first installs are Open WebUI, Cline, and Aider. They all work
-with the current llama.cpp server and need no extra GPU.
-
-## 2. Inference Runtime Tuning (llama.cpp on gfx1151)
-
-Verified against the running service and llama.cpp source.
-
-| Change | Recommendation | Priority |
-| ------ | -------------- | -------- |
-| Backend | **Vulkan0** (2026-08-15, corrected — see Corrections). ROCm only for prefill-heavy workloads. | Done |
-| KV cache | `q8_0` K / `q8_0` V in production. Try `q4_0` only if more context is needed. | Keep |
-| Flash attention | Leave `--flash-attn auto`. It helps prompt processing, not decode. | Keep |
-| Load mode | Start with mmap (default). Enable `--mlock` only if page-fault stutter appears. | Optional |
-| Threads | Use `--threads 12 --threads-batch 16 --threads-http 1` for 16 Zen5 cores + 3 slots. Frees cores for the ROCm driver. | Medium |
-| DFlash n_max | Current 8 (block 16) is high. Try `--spec-draft-n-max 5`; it may reduce rejection cost at the same throughput. | Verify |
-| ngram layering | Measured slower than DFlash alone at 4k (16.9 vs 20.2 t/s). Not enabled. | Keep off |
-
-## 3. Server / OS / Hardware Settings
-
-All require root and a reboot for the modprobe change.
-
-| Setting | Action | Effect | Priority |
-| ------- | ------ | ------ | -------- |
-| TTM GPU pool | `/etc/modprobe.d/ttm.conf` → `options ttm pages_limit=29883536 page_pool_size=29883536` (applied 2026-08-25, set live then persisted) | Lifts the 15 GiB cap to 114 GiB; the full 262k KV + Q8 AGENT payload fit in one slot | Done |
-| Power profile | **N/A on this machine** (2026-08-15): no `power_profile` sysfs; GPU DPM already `performance` | none | ~~Medium~~ |
-| NVMe scheduler | Already `none` (`[none]` confirmed) | No change | Done |
-| BIOS VGM | Set Variable Graphics Memory / UMA to "High" in the EVO-X2 BIOS | Max dedicated VRAM for the GPU carve-out; verify in the live BIOS | Verify |
-| Firmware/EC | Check GMKtec for a newer EC/fan-curve firmware | Better sustained thermal behavior | Verify |
-
-## 4. Other Enhancements
-
-- Test `--spec-draft-n-max 5` and re-benchmark acceptance; if throughput rises,
-  change the env tunable.
-- Consider switching the disabled `vllm-thinkingcap.service` unit off or
-  removing it to avoid confusion (it is loaded in a failed state).
-- Profile generation under 3-slot concurrent load before touching boot
-  settings, so any tuning is measured against the real workload.
-
-## Sources
-
-- [[frank-default-large-benchmark]] — measured backend/spec numbers
-- [[deployment]] — deploy and update process
-- AMD ROCm RDNA3.5 system optimization guide (gfx1151 memory/BIOS)
-- llama.cpp source: `ggml/CMakeLists.txt`, `ggml/src/ggml-hip/CMakeLists.txt`,
-  `common/arg.cpp`, `common/common.h`
-- NotebookCheck Ryzen AI MAX+ 395 analysis
-
-## See Also
-
-- [[frank-default-large-benchmark]]
-- [[DFlash DSpark Speculative Decoding]]
-- [[deployment]]
-- [[index]]

docs/LLM/Dankclaw System Specs.md

@@ -1,106 +0,0 @@
-# Dankclaw System Specs
-
-## Server Overview
-
-Dankclaw is an AMD APU server for CIRU ROCmFPX llama.cpp inference. The host runs Ubuntu 26.04 with an AMD Ryzen AI MAX+ 395 Strix Halo APU. The APU uses unified memory — CPU and GPU share the same physical RAM pool.
-## Hardware Specifications
-
-### CPU
-
-| Property | Value |
-|----------|-------|
-| Model | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S |
-| Architecture | x86_64 |
-| Cores | 16 (1 socket, 1 thread per core) |
-| Max MHz | 5187.50 |
-| Min MHz | 625.00 |
-| NUMA Nodes | 1 (node 0, 31213 MB) |
-
-### GPU (Integrated)
-
-| Property | Value |
-|----------|-------|
-| Model | AMD Radeon 8060S (Strix Halo) |
-| Device ID | 0x1586 |
-| Architecture | gfx1151 (Strix Halo) |
-| GPU-Visible Memory | 64 GiB (rocm-smi; SVM-backed, see the memory model below) |
-| Physical RAM | 64 GiB (61 GiB visible; upgraded from 32 GB, corrected 2026-08-17) |
-
-### NPU
-
-| Property | Value |
-|----------|-------|
-| Model | AMD XDNA2 (Ryzen AI) |
-| Rating | 50 TOPS |
-| Device | /dev/accel/accel0 |
-| Driver | amdxdna 0.7 (in-tree, kernel 7.0.0-29-generic) |
-| Firmware | 1.1.2.65 |
-| Runtime | FastFlowLM 0.9.45 + XRT (installed 2026-08-16; **retired 2026-08-17**) |
-| Status | Not in service. Research only. See [[FastFlowLM NPU Inference]] |
-
-The NPU is retired; the 52625 port and the `frank-default-small` name now serve the CIRU ROCmFPX llama.cpp runtime (see [[ciru-activefpx-promptforge]]).
-
-### ROCm
-
-| Property | Value |
-|----------|-------|
-| ROCk Module | Loaded |
-| Runtime Version | 1.18 |
-| ROCm Version | 7.2.4 system (/opt/rocm) + TheRock 7.15 SDK in the CIRU venv (used for the runtime) |
-| Agent Name | AMD RYZEN AI MAX+ 395 w/ Radeon 8060S |
-| GPU Arch | gfx1151 (Strix Halo) |
-| GPU Global Memory | 64 GiB (rocm-smi; SVM-backed) |
-
-### Memory
-
-| Property | Value |
-|----------|-------|
-| System RAM | 64 GiB (shared CPU + GPU; 61 GiB visible) |
-| GPU-Visible (ROCm) | 64 GiB (rocm-smi; SVM-backed) |
-| Swap | 8.0 GiB |
-| /dev/shm | ~15 GiB (half of RAM) |
-
-### Storage
-
-| Filesystem | Size | Used | Avail | Use% |
-|------------|------|------|-------|------|
-| / (nvme0n1p2) | 1.9T | 1.3T | 545G | 70% |
-
-### Network
-
-| Interface | Type | Speed | IP | Notes |
-|-----------|------|-------|----|-------|
-| eno1 | onboard | 2500 Mb/s (2.5 GbE) | 192.168.0.143 (DHCP) | Carries the RPC path to Dankserver since 2026-08-17. Default route via 192.168.0.1. |
-| enx00e04cc2e1e5 | USB-C to Ethernet adapter | 1000 Mb/s | 192.168.0.116 | UNPLUGGED 2026-08-17. While present, the route to 192.168.0.235 went through this adapter (lower route metric), so all RPC traffic was capped at 1 GbE. See [[Pipeline Parallelism/Ethernet Bandwidth Math]] section 0. |
-| docker0 | bridge | - | 172.17.0.1 | Usually down. |
-
-The L2 gateway/switch at 192.168.0.1 (MAC f0:4b:8a:65:c5:40) is at
-least 2.5 GbE-capable. Whether it is 10G-capable is unverified — check
-the device itself before buying 10G USB-C adapters; otherwise a direct
-cable between claw and server is required for 10G.
-
-### OS and Software
-
-| Property | Value |
-|----------|-------|
-| OS | Ubuntu 26.04 LTS (Resolute Raccoon) |
-| Kernel | 7.0.0-29-generic |
-| Python | 3.14.4 (system), 3.12.13 (CIRU venv) |
-| CIRU runtime | ciru-ai/ROCmFPX v2.3 (llama-server, ROCm 7.15 SDK) |
-| uv | 0.12.2 |
-
-## CIRU Runtime Relevance
-
-- Unified memory eliminates separate VRAM limits. The GPU uses system RAM.
-- The GPU budget is the rocm-smi figure: 64 GiB VRAM reported by rocm-smi (SVM-backed). The older 96 GiB figure was the addressable SVM pool, not the budget. Trust rocm-smi and the runtime; do not use `free` or `dmidecode` for the GPU budget.
-- The active service is `frank-default-large` on port 7770 (moved from 52625 on 2026-08-22): CIRU ROCmFPX llama.cpp serving kingjones/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP (Q6 AGENT dense, 262144 ctx, standalone MTP Q4_0 head). See [[kingjones-strix-mtp]].
-- The runtime requires ROCm 7.15 (model card requirement). The system ROCm is 7.2.4; the retired ling3-ciru venv bundles a self-contained TheRock ROCm 7.15 SDK that the serve script puts first in `LD_LIBRARY_PATH`.
-- The retired vLLM venvs (`/home/dank/vllm-venv`, `/home/dank/vllm-vk-venv`) were moved to `/home/dank/.AGENTTRASH/` on 2026-08-18.
-- 1.1 TB free disk space is ample for model storage.
-
-## See Also
-
-- [[ciru-activefpx-promptforge]]
-- [[Shared Memory Architecture Dankclaw]]
-- [[frank-redo-2026-08-17]]
-- [[FastFlowLM NPU Inference]]

docs/LLM/Dankserver System Specs.md

@@ -1,77 +0,0 @@
-# Dankserver System Specs
-
-## Server Overview
-
-Dankserver is an NVIDIA GPU server for vLLM inference. The host runs Ubuntu 26.04 (Resolute Raccoon) with an RTX 3090 and 62 GB of system RAM.
-
-## Hardware Specifications
-
-### CPU
-
-| Property | Value |
-|----------|-------|
-| Architecture | x86_64 |
-| Vendor | AuthenticAMD |
-| Model | AMD Ryzen 7 7700X |
-| Cores | 8 (1 socket) |
-| NUMA Nodes | 1 (node 0, 63503 MB) |
-
-### GPU
-
-| Property | Value |
-|----------|-------|
-| Model | NVIDIA GeForce RTX 3090 |
-| VRAM | 24576 MiB (24 GB) |
-| Driver | 595.84 |
-| CUDA Version | 13.2 |
-| CUDA Toolkit | 12.0 (nvcc) |
-
-### Memory
-
-| Property | Value |
-|----------|-------|
-| System RAM | 62 GiB |
-| Swap | 127 GiB |
-| /dev/shm | 32 GiB |
-
-### Storage
-
-| Filesystem | Size | Used | Avail | Use% |
-|------------|------|------|-------|------|
-| / (nvme0n1p2) | 915G | 729G | 140G | 84% |
-| /mnt/entropy (dm-0) | 91T | 86T | 910G | 99% |
-
-### Network
-
-| Interface | Type | Speed | IP | Notes |
-|-----------|------|-------|----|-------|
-| eno1 | onboard | 2500 Mb/s (2.5 GbE) | 192.168.0.235 (DHCP) | RPC worker endpoint (port 50052). Default route via 192.168.0.1. |
-| enxe8ea6ab7ebca | USB-C class adapter | 2500 Mb/s | 192.168.0.252 | UNPLUGGED 2026-08-17. Was idle (unused by the RPC path). |
-| wlp8s0 | WiFi | - | - | Unused (carrier down). |
-| docker0 | bridge | - | 172.17.0.1 | Down; internal docker traffic only. |
-
-### OS and Software
-
-| Property | Value |
-|----------|-------|
-| OS | Ubuntu 26.04 LTS (Resolute Raccoon) |
-| Kernel | 7.0.0-30-generic |
-| Python | 3.14.4 |
-
-### Active Services
-
-The active service on Dankserver is `frank-default-small.service`: the llama.cpp Ornith-1.5-35B-A3B CRACK (Q6_K) on port 7770, with expert tensors selected by `--override-tensor`; every full-attention block plus blocks 35–40 remain on CUDA. The earlier PP worker (`frank-pp-worker.service`) was retired with the 2026-08-17 redo. See [[deployment]].
-
-## Hardware Relevance
-
-- 24 GB VRAM hosts the Q6_K non-expert tensors and the full-attention/final-layer expert placement. The context pool contains 491520 f16 cells across three 163840-token slots. The service uses ngram-mod only and `ubatch=1024`. Verify actual headroom after each model restart with `nvidia-smi` and the service journal.
-- 62 GB system RAM holds the remaining CPU-resident expert tensors with room to spare.
-- 127 GB swap is available for CPU KV cache paging if GPU KV offload is disabled.
-- Root disk has approximately 140 GB free. Store the GGUF models under `/home/dank/models` (about 32 GB total).
-
-## See Also
-
-- [[VLLM Overview]]
-- [[Hybrid CPU GPU Processing Dankserver]]
-- [[RAM CPU Offloading in VLLM]]
-- [[Running Qwen3.6 35B A3B FP8 on Dankserver]]

docs/LLM/FastFlowLM NPU Inference.md

@@ -1,324 +0,0 @@
-> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology.
-> See [frank-redo-2026-08-17.md](frank-redo-2026-08-17.md) and
-> [deployment.md](deployment.md) for the active configuration.
-> The `frank-default-small` name and port 52625 now belong to the vLLM
-> service on Dankclaw (nameistoken Quark W8A8); this page is archive.
-
----
-type: research
-title: FastFlowLM NPU Inference
-description: FastFlowLM (FLM), the AMD NPU-first LLM runtime for Ryzen AI XDNA2 chips. Fit analysis for the 50 TOPS NPU on Dankclaw, published benchmarks, verified install state, and a test runbook.
-tags:
-  - fastflowlm
-  - npu
-  - xdna2
-  - dankclaw
-  - research
-timestamp: 2026-08-16
----
-
-# FastFlowLM NPU Inference
-
-FastFlowLM (FLM) runs LLMs entirely on the AMD Ryzen AI NPU. It targets XDNA2 chips: Strix, Strix Halo, Kraken, and Gorgon Point. The project describes itself as "Ollama, but deeply optimized for NPUs." As of 2026-08-16, FLM is installed on Dankclaw, the NPU validates, and `frank-default-small` (port 52625) serves in production. See Promotion.
-
-## Verdict (2026-08-16)
-
-- The NPU is a complement, not a replacement. It fits dense models up to about 9B.
-- Qwen3.8-27B (dense) cannot run on the NPU. No FLM build exists and the memory budget is too small.
-- Qwen3.6-35B-A3B has an official FLM build (Q4_KS). AMD benchmarks it on a 96 GB machine. It probably does not fit the 30 GiB budget of Dankclaw.
-- Realistic first candidates: Qwen3.5-9B, Qwen3-8B, gpt-oss-20b (at the budget edge).
-- **Landed 2026-08-16:** gpt-oss-20b serves on the NPU at 19.3 t/s decode, concurrently with the PP head. See Measured Results.
-- The memlock limit was the setup blocker. Fixed 2026-08-16 via `/etc/security/limits.d/99-flm-memlock.conf`.
-
-## What FastFlowLM Is
-
-FLM is an NPU-first inference runtime. It started as a 2025 university project (University of Rhode Island, Ken Qing Yang). The team joined AMD on 2026-07-17. The repository moved to the ROCm organization on 2026-08-11 as v1.0.0.
-
-Key properties, per the project README and docs:
-
-- Runs fully on the NPU. No GPU or CPU load during inference.
-- CLI mode (`flm run <tag>`) and server mode (`flm serve <tag>`).
-- Server mode exposes a REST API and an OpenAI-compatible API. Default port is 52625.
-- Models are not GGUF. FLM downloads pre-compiled kernel packs (built with AMD IRON + AIE-MLIR) from Hugging Face.
-- Context lengths up to 256k tokens on listed models.
-- Vision, audio, embedding, and MoE model support.
-- Runtime is about 17 MB. Orchestration code is MIT-licensed. Kernels are free for commercial use.
-- Models store in `~/.config/flm/` (Linux). Override with `FLM_MODEL_PATH`.
-
-## NPU Hardware on Dankclaw
-
-Dankclaw runs an AMD Ryzen AI MAX+ 395 (Strix Halo). The chip integrates an XDNA2 NPU rated at 50 TOPS. It sits beside the Radeon 8060S iGPU and shares the same 30 GiB unified memory pool.
-
-Live verification on Dankclaw, 2026-08-16:
-
-```
-$ flm validate
-[FLM]  Fetching models from: /opt/fastflowlm/share/flm/model_list.json
-[Linux]  Kernel: 7.0.0-29-generic
-[Linux]  NPU: /dev/accel/accel0 with 8 columns
-[Linux]  NPU FW Version: 1.1.2.65
-[Linux]  amdxdna version: 0.7
-[ERROR]  Memlock limit is too low (8MB). Please raise the limit or set to infinity.
-```
-
-- Kernel 7.0.0-29-generic ships the `amdxdna` driver in-tree. The driver is loaded.
-- Firmware 1.1.2.65 meets the 1.1.0.0 minimum.
-- The XRT runtime packages (`libxrt2`, `libxrt-npu2`) are installed.
-- The only failure is the memlock ulimit (8 MB). FLM needs `unlimited`.
-- Ubuntu 26.04 is not on the FLM supported-distro list (24.04, 25.10, Arch, generic). Validation passes anyway. Treat 26.04 as "generic Linux, working".
-
-## Memory Model
-
-The NPU allocates from system RAM. AMD/Microsoft enforce an internal cap of about half of total system RAM for NPU use. The gpt-oss model card states a cap of about 15.6 GB on a 32 GB machine, which is half.
-
-Budget math for this fleet:
-
-| Machine | RAM | NPU budget (approx. half) |
-|---------|-----|---------------------------|
-| Dankclaw (Strix Halo) | 30 GiB | ~15 GiB |
-| Dankserver (Ryzen 7 7700X) | 62 GB | no NPU |
-| FLM bench box (Kraken Point) | 32 or 96 GB | ~16 or ~48 GB |
-
-Weight sizes (approximate, Q4-class):
-
-| Model | Quant | Weights | Fits ~15 GiB budget? |
-|-------|-------|---------|----------------------|
-| Qwen3.5-9B | Q4_1 | ~5.5 GB | yes |
-| Qwen3-8B | Q4_1 | ~5 GB | yes |
-| gpt-oss-20b (MoE, 3.6B active) | Q4_1 | ~15.1 GB | at the edge |
-| Qwen3.6-35B-A3B (MoE, 3B active) | Q4_KS | ~20 GB | no, unproven at 30 GiB |
-| Qwen3.8-27B (dense) | any Q4 | ~16 GB+ | no build exists |
-
-Do not load a model larger than the budget first. An oversized load can starve the unified pool and crash the box while production services still run.
-
-## Supported Models (Qwen family, plus gpt-oss)
-
-All Qwen cards use Q4_1 unless noted. Context lengths are the FLM-published maxima.
-
-| Tag | Base model | Context max | Notes |
-|-----|-----------|-------------|-------|
-| `qwen3:0.6b` | Qwen3-0.6B | 32k | thinking toggle |
-| `qwen3:1.7b` | Qwen3-1.7B | 32k | thinking toggle |
-| `qwen3:4b` | Qwen3-4B | 32k | tool calling |
-| `qwen3:8b` | Qwen3-8B | 32k | tool calling |
-| `qwen3-tk:4b` | Qwen3-4B-Thinking-2507 | 256k | thinking |
-| `qwen3-it:4b` | Qwen3-4B-Instruct-2507 | 256k | tool calling |
-| `qwen3vl-it:4b` | Qwen3-VL-4B-Instruct | 256k | vision |
-| `qwen2.5-it:3b` | Qwen2.5-3B-Instruct | 32k | |
-| `qwen2.5vl-it:3b` | Qwen2.5-VL-3B-Instruct | 256k | vision |
-| `qwen3.5:0.8b` | Qwen3.5-0.8B | 256k | vision, thinking toggle |
-| `qwen3.5:2b` | Qwen3.5-2B | 256k | vision, tool calling |
-| `qwen3.5:4b` | Qwen3.5-4B | 256k | vision, tool calling |
-| `qwen3.5:9b` | Qwen3.5-9B | 256k | vision, tool calling |
-| `qwen3.6-moe:35b-a3b` | Qwen3.6-35B-A3B | 256k | **Q4_KS**, MoE, vision, thinking, tool calling |
-| `gpt-oss:20b` | gpt-oss-20b | 128k | Q4_1, ~15.1 GB in FLM |
-
-Other families: LLaMA, DeepSeek, Gemma, MedGemma, TranslateGemma, LiquidAI/LFM, Phi, Nanbeige, Whisper, EmbeddingGemma, SmolVLA.
-
-## Published Benchmarks (FLM, vendor numbers)
-
-All vendor results come from AMD Ryzen AI 7 350 (Kraken Point, 40 TOPS NPU) boxes under the default NPU power mode (Performance). Our NPU is a 50 TOPS Strix Halo XDNA2. Treat these numbers as directional, not as our expected speed.
-
-Qwen3.5, FLM v0.9.38, Kraken Point 32 GB DRAM. Decode t/s by starting context length:
-
-| Model | 1k | 2k | 4k | 8k | 16k | 32k |
-|-------|-----:|-----:|-----:|-----:|-----:|-----:|
-| Qwen3.5-0.8B | 39.2 | 38.0 | 36.3 | 33.1 | 28.1 | 21.6 |
-| Qwen3.5-2B | 26.8 | 26.2 | 25.4 | 23.7 | 21.3 | 17.0 |
-| Qwen3.5-4B | 15.0 | 14.6 | 14.2 | 13.3 | 11.8 | 9.6 |
-| Qwen3.5-9B | 9.3 | 9.2 | 9.0 | 8.5 | 7.8 | 6.9 |
-
-Prefill t/s for the same run:
-
-| Model | 1k | 2k | 4k | 8k | 16k | 32k |
-|-------|-----:|-----:|-----:|-----:|-----:|-----:|
-| Qwen3.5-0.8B | 983 | 1257 | 1471 | 1584 | 1579 | 1447 |
-| Qwen3.5-2B | 803 | 1004 | 1142 | 1223 | 1225 | 1151 |
-| Qwen3.5-4B | 378 | 440 | 479 | 493 | 487 | 450 |
-| Qwen3.5-9B | 284 | 333 | 362 | 379 | 378 | 357 |
-
-Qwen3.6-35B-A3B (Q4_KS), FLM v0.9.45, Kraken Point **96 GB DRAM**:
-
-| Metric | 1k | 2k | 4k | 8k | 16k | 32k |
-|--------|-----:|-----:|-----:|-----:|-----:|-----:|
-| Decode t/s | 13.65 | 13.41 | 13.09 | 12.51 | 11.24 | 9.51 |
-| Prefill t/s | 78.98 | 118.04 | 156.43 | 197.93 | 218.84 | 221.96 |
-
-Context: the current PP split (head + 3090 worker) decodes at 14-18 t/s for the 27B dense model. A 35B-A3B at ~10-14 t/s on an NPU with 48 GB of RAM budget is not a speed win over the existing topology.
-
-The vendor also claims "over 10x more power-efficient" versus GPU. That is a marketing claim. We have not measured it.
-
-## Fit to the Frank Program
-
-| Our model | NPU verdict |
-|-----------|-------------|
-| Qwen3.8-27B UD-Q8_K_XL (Dankclaw, dense) | No. No FLM build. ~16 GB+ even at Q4. Exceeds the ~15 GiB budget. |
-| Qwen3.6-35B-A3B (Dankserver, MoE) | Unproven. Official build exists. Vendor bench ran on 96 GB RAM. A 30 GiB box has ~15 GiB for the NPU. Test last, with the service stopped. |
-| Qwen3.5-9B / Qwen3-8B | Yes, the realistic tier. ~5-6 GB. Leaves room for KV and the OS. |
-| gpt-oss-20b (MoE) | **Landed.** Weights are 13.8 GB on disk (q4nx). The model holds ~15 GiB resident on the NPU. Decode is 19.3 t/s. See Measured Results. |
-
-Proposed role: a small, always-on, low-power endpoint for triage, summarization, embeddings, or agent side-tasks on the Dankclaw NPU. It frees the iGPU and the 3090 for the 27B/35B workloads. It does not replace either.
-
-## Install State on Dankclaw (verified 2026-08-16)
-
-- `fastflowlm 1.0.1` installed (deb, upgraded from 0.9.45-12-ga6bd056 on 2026-08-17). `flm` at `/usr/bin/flm`. The 0.9.45 runtime refused the qwythos pack, which declares `flm_version` 1.0.0.
-- `libxrt2` / `libxrt-npu2` (Xilinx Runtime) installed.
-- `/dev/accel/accel0` present. `flm validate` passes fully.
-- Memlock fixed: `/etc/security/limits.d/99-flm-memlock.conf` sets soft and hard memlock to unlimited. A fresh sshd session picks it up via pam_limits. No reboot needed.
-- `gpt-oss:20b` pulled (4/4 files verified, 13.8 GB `model.q4nx` + tokenizer) into `~/.config/flm/models/GPT-OSS-20B-NPU2/`.
-- `qwythos:9b` (Huihui-Qwythos-9B-Claude-Mythos-5-1M-abliterated-NPU2) installed 2026-08-17 via `flm-add.py` (downloaded from the HF repo; stdlib-only, no pip needed). Tag registered in the user registry `~/.config/flm/model_list.json` (a copy of the v1.0.1 system registry + the custom entry); kernels symlinked `~/.config/flm/xclbins/Huihui-...-NPU2 -> /opt/fastflowlm/share/flm/xclbins/Qwen3.5-9B-NPU2`. The service env exports `FLM_CONFIG_PATH` and `FLM_XCLBIN_PATH` so the tag resolves (without them flm hits the unknown-tag fallback, see the incident below).
-- `flm serve gpt-oss:20b --host 0.0.0.0 -p 52625 --pmode performance` runs as a nohup process. It is not a systemd unit. It does not survive a reboot.
-- The `llama-server` (frank-default-large, the PP head) holds an fd on `/dev/accel/accel0` from its own startup (device enumeration, not compute). FLM opened the same device concurrently without conflict. Verified live 2026-08-16.
-
-## Measured Results (2026-08-16, gpt-oss-20b on the Dankclaw NPU)
-
-Conditions: `--pmode performance`, PP head (27B, port 7770) serving concurrently, 5.5 GiB system RAM headroom steady-state, NPU resident footprint ~15 GiB.
-
-| Run | Prompt tok | Compl tok | Wall s | Prefill t/s | Decode t/s | KV occ % |
-|-----|-----------:|----------:|-------:|------------:|-----------:|---------:|
-| 1 (smoke) | 74 | 63 | 7.2 | 19.5 | 19.3 | 1.7 |
-| 2 (160-tok prompt) | 161 | 256 | 17.4 | 40.8 | 19.3 | 5.1 |
-| 3 (95-tok prompt, 512 out) | 95 | 512 | 30.8 | 24.3 | 19.3 | n/a |
-
-Decode is flat at 19.3 t/s across all three runs. Output includes `reasoning_content` (gpt-oss is a reasoning model). At `max_tokens 256` one run spent its whole budget thinking and returned empty content. Give it 500+ tokens or strip reasoning in the client.
-
-Long-context prefill is not yet measured (runs 1-3 used under 200 prompt tokens). The 35B-A3B step remains untested.
-
-## Model Bakeoff (measured 2026-08-16, 4 models)
-
-All runs on the Dankclaw NPU (50 TOPS Strix Halo), single stream, `--pmode performance`, PP head (frank-default-large) serving concurrently on 7770. Decode t/s is the FLM-reported rate. Vision accuracy = ground-truth elements out of 4 (text SPECTRUM, red circle left, blue square right, three green triangles bottom).
-
-| Model | Tag | Weights | Decode t/s (short) | 8k prefill t/s | 8k decode t/s | Vision decode t/s | Vision accuracy | Max ctx (registry) |
-|-------|-----|---------|-------------------:|---------------:|--------------:|------------------:|----------------|--------------------|
-| Gemma4-E4B | `gemma4-it:e4b` | 6.8 GB + 912 MB audio | 12.2 | 582 | 9.9 | 11.7 | 4/4 | 32768 |
-| Qwen3.5-9B | `qwen3.5:9b` | 7.3 GB | 7.2 | 343 | 6.6 | 7.3 | 4/4 | 32768 |
-| GPT-OSS-20B | `gpt-oss:20b` | 13.8 GB | 19.3 | 378 | 16.9 | n/a (no vision) | 0/4 | **8192** |
-| Qwen3.6-35B-A3B | `qwen3.6-moe:35b-a3b` | 22.2 GB | 12.7 | 207 | 11.2 | 12.7 | 4/4 | 32768 |
-
-Notes:
-
-- gpt-oss:20b is the fastest decoder (MoE, 3.6B active) but its KV cache fills at ~8k tokens (98.85% occupancy at 7,926; requests past ~8k return 400). It has no vision encoder.
-- 35B-A3B loads 28-29 GiB resident. It works but leaves ~1.5 GiB headroom with the PP head running. Do not run it alongside the head at full load.
-- Vision prefill for the two vision models with ~720 token image input: gemma 141 t/s, qwen3.5 176 t/s, 35B-A3B 68 t/s.
-- All four comply with the same image at 4/4 elements except gpt-oss, which cannot see it.
-
-## Promotion (2026-08-16)
-
-`frank-default-small` now runs the NPU endpoint on Dankclaw (was the Dankserver llama.cpp service). Unit: `scripts/frank-default-small.service` (symlinked into `/etc/systemd/system`), wrapper `scripts/serve-flm-small.sh`, tunables `scripts/frank-default-small.env`. `LimitMEMLOCK=infinity` is set in the unit. Default model: `gemma4-it:e4b`. Swap `FLM_MODEL` in the env to switch.
-
-### Boot policy (2026-08-17)
-
-The `frank-default-small` unit is disabled for boot. Only the PP pair (`frank-default-large` + `frank-pp-worker`) starts at boot. The unit file was removed from `/etc/systemd/system`; the running FLM service keeps serving until stopped or rebooted. To restore the endpoint, run these commands on Dankclaw:
-
-```
-sudo ln -sf /home/dank/vllmScripts/scripts/frank-default-small.service /etc/systemd/system/frank-default-small.service
-sudo systemctl daemon-reload
-sudo systemctl enable frank-default-small
-```
-
-Live config (2026-08-16): `FLM_PMODE=turbo`, `FLM_ASR=1` (loads Whisper-V3-Turbo-NPU2; exposes `/v1/audio/transcriptions`), `FLM_CTX_LEN=131072` (verified max — Gemma4 natively supports 128K and the NPU budget fits it; KV occupancy measured at 0.11% for a 123-token exchange, decode 12.0 t/s, 13 GiB headroom).
-
-The PP split was renamed `frank-pp-head` → `frank-default-large` (2026-08-16, commit `48142f5`).
-
-Model swap (2026-08-16, commit `50c55ee`): `FLM_MODEL=qwen3.6-moe:35b-a3b`, `FLM_CTX_LEN=262144` (native max), `FLM_PMODE=turbo`, `FLM_ASR=1` kept. Decode measured 12.5-13.1 t/s. The Gemma4 live config above (131072 ctx) applied to `gemma4-it:e4b`. Swap `FLM_MODEL` and `FLM_CTX_LEN` back to restore it.
-
-Model swap (2026-08-17, commit `c85ff6c`): `FLM_MODEL=qwythos:9b` (Huihui-Qwythos-9B-Claude-Mythos-5-1M-abliterated-NPU2, Q4NX 7.76 GB). The pack installs through `flm-add` (user registry `~/.config/flm/model_list.json` + user xclbins `~/.config/flm/xclbins`, kernels symlinked from official `Qwen3.5-9B-NPU2`). Runtime upgraded to FLM 1.0.1 first (see Install State). `FLM_CTX_LEN=262144` native max (Qwen3.5-9B hybrid: 8 of 32 layers full attention, so KV stays small; occupancy 0.008% at 21 tokens). `FLM_PMODE=turbo`, `FLM_ASR=1` kept. The env file now exports `FLM_CONFIG_PATH` / `FLM_XCLBIN_PATH` — required, or flm treats the tag as unknown and silently serves llama3.2:1b. `qwen3.6-moe:35b-a3b` files stay installed for rollback. pi client entry (`~/.pi/agent/models.json` on dankware): id `qwythos:9b`, ctx 262144, input `[text]` (pack has no vision weights), compat `supportsDeveloperRole: false` kept.
-
-## Parallel Streams (measured 2026-08-16)
-
-The NPU serializes generations. It runs exactly one stream at a time. Extra streams queue in FIFO order. FLM server flags `--socket` (default 10) and `--q-len` (default 10) cap queue depth only. They do not add concurrency.
-
-Concurrent-request test, gpt-oss-20b, 128-token cap per stream:
-
-| Streams | Wall s | Aggregate t/s | Per-stream wall s |
-|--------:|-------:|--------------:|-------------------|
-| 1 | 10.6 | 12.1 | 10.6 |
-| 2 | 21.1 | 12.0 | 10.8 / 21.1 |
-| 4 | 41.1 | 11.6 | 10.9 / 21.6 / 32.4 / 41.1 |
-
-The 4-stream wall times form a near-perfect 10.7 s ladder. Each stream self-reports 19.3 t/s decode, counting only its own active slices. Trust per-stream wall time, not the self-reported speed, under concurrency.
-
-Consequence: the NPU endpoint is a single lane. It suits low-frequency background work (triage, summarization, agent side-tasks). It does not suit interactive chat for several simultaneous users. The 27B llama.cpp service, by contrast, runs PARALLEL=2 with true concurrent slots.
-
-## Test Runbook
-
-Do not run large model loads while the production service holds most of the 30 GiB pool. Check free RAM first or stop the service. The NPU shares the unified pool with everything else on the box.
-
-```mermaid
-graph TD
-    A[1. Raise memlock to unlimited] --> B[2. flm validate + xrt-smi examine]
-    B --> C{NPU visible to XRT?}
-    C -- no --> D[Fix driver or firmware path]
-    C -- yes --> E[3. flm run qwen3.5:4b smoke test, /verbose]
-    E --> F{Generates tokens?}
-    F -- no --> D
-    F -- yes --> G[4. flm serve on a scratch port, never 7770]
-    G --> H[5. Bench decode + prefill t/s]
-    H --> I{Speed acceptable?}
-    I -- no --> J[Record results, stop]
-    I -- yes --> K[6. Attempt gpt-oss:20b]
-    K --> L{Loads in ~15 GiB?}
-    L -- no --> J
-    L -- yes --> M[Bench, record]
-    M --> N[7. Last: qwen3.6-moe:35b-a3b, service stopped]
-    N --> O[Record, decision]
-```
-
-1. Raise memlock. Add `* soft memlock unlimited` and `* hard memlock unlimited` to `/etc/security/limits.conf`. Then reboot. For a systemd unit, set `LimitMEMLOCK=infinity` instead.
-2. Verify the NPU. Run `flm validate`. Run `xrt-smi examine`. Both must show the NPU. `flm validate` uses the kernel device. `flm run` uses XRT. A passing validate does not guarantee XRT sees the device.
-3. Smoke test. Run `flm run qwen3.5:4b`. Toggle `/verbose` for performance reporting. Type `/bye` to exit.
-4. Server mode. Run `flm serve qwen3.5:4b`. The default port is 52625. Use a scratch port if 52625 is taken. Never 7770.
-5. Bench. Measure decode and prefill t/s at 1k/4k/16k starting context. Record to this page.
-6. Step up in size: 9B, then gpt-oss-20b.
-7. Attempt `qwen3.6-moe:35b-a3b` only after the smaller runs succeed, and only with the production service stopped. Watch `free -h` during the model download and load.
-
-## Incident: qwen3.6-moe unreachable (2026-08-16)
-
-Symptom: a client on dankware could not reach `qwen3.6-moe:35b-a3b` on port 52625. The journal shows three separate causes in the 14:19-15:03 window.
-
-1. **EXEC 203 crash loop (14:19-14:20).** The exec bit was lost on `serve-flm-small.sh` on the target. The unit failed `status=203/EXEC: Permission denied` four times, then started after the bit was restored. The git mode is `100755` (commit `3f42210`), and a `git pull` restores the bit. Verify with `git ls-files -s scripts/serve-flm-small.sh`.
-2. **Wrong model ID (14:29-14:52).** The client config still used the model ID `frank-default-small` (a service alias). FLM rejects that tag with `[ERROR] Model tag 'frank-default-small' is not supported.` Each such request also triggered the unknown-tag fallback below.
-3. **Swap churn (14:48-15:03).** The swap `gemma4-it:e4b` → `qwen3.6-moe:35b-a3b` restarted the service several times. Each load of the 23 GB model takes about 1.5-2.5 minutes at 262144 ctx. The endpoint does not serve during a load.
-
-Fix: the client model ID must be the FLM registry tag `qwen3.6-moe:35b-a3b`. The working pi agent entry in `~/.pi/agent/models.json` is `baseUrl http://192.168.0.143:52625/v1`, `id qwen3.6-moe:35b-a3b`, `contextWindow 262144`, `reasoning true`, `input [text, image]`. Verified end-to-end from dankware (192.168.0.226) at 15:32 on 2026-08-16.
-
-### Unknown-tag fallback (FLM behavior)
-
-When a request names a tag that is not in the FLM registry, FLM does not return a clean 404. It logs the ERROR, then falls back to the default model `llama3.2:1b`:
-
-- If the `llama3.2:1b` files are missing, FLM downloads them (1.2 GB on 2026-08-16) while the request hangs.
-- FLM then loads the 1B model in place of the requested model.
-- The files now exist in `~/.config/flm/models/Llama-3.2-1B-NPU2` (1.3 GB) on dankclaw. A future unknown-tag request loads the 1B model silently, with no download and no error.
-
-Keep client model IDs as FLM registry tags. Never send a service alias. Optional cleanup: move `~/.config/flm/models/Llama-3.2-1B-NPU2` (an incident artifact) to `.AGENTTRASH` so a stray tag re-downloads loudly instead of serving the wrong model.
-
-### Request compatibility: the `developer` role (found 2026-08-16)
-
-pi sessions failed against this endpoint with `Stream ended without finish_reason` while curl worked. Root cause, verified by field bisection:
-
-- The FLM Qwen chat template accepts only the roles `system`, `user`, and `assistant`. Any other role raises `Unexpected message role.` in the template.
-- FLM reports the error differently per mode. Non-streaming: HTTP 500 with the template traceback. Streaming: HTTP 200, and the error JSON is the first body. A streaming client sees a stream with no chunks and no `finish_reason`.
-- pi sends `role: "developer"` when `model.reasoning && compat.supportsDeveloperRole` (pi source, `openai-completions.js`). The default `supportsDeveloperRole` is true for standard OpenAI-compatible providers, so a reasoning model without a `compat` block always triggers the bug.
-- Fix: `"compat": { "supportsDeveloperRole": false }` on the model entry in `~/.pi/agent/models.json`. pi then sends `role: "system"`. Verified with a headless pi session: full round trip, `finish_reason: stop`.
-- FLM ignores unknown body fields. `reasoning_effort`, `store`, `max_completion_tokens`, and `tools: []` all arrive in pi requests and pass through without error.
-
-## Risks and Open Questions
-
-- **Ubuntu 26.04 is unofficial.** The distro list stops at 25.10. Validation passes, but breakage would fall on the "generic Linux" path.
-- **The 30 GiB budget is untested at the very top.** gpt-oss-20b (15 GiB resident) works. 35B-A3B (~20 GB weights) is still an unknown. Vendor data covers 32 GB and 96 GB boxes only.
-- **The FLM serve process is not under systemd.** A reboot of Dankclaw kills it. The PP head service is enabled and returns. The FLM process must be restarted by hand. A unit file is a natural next step.
-- **gpt-oss reasoning budget.** The model can spend its full token budget on `reasoning_content` and return empty content. Clients must handle or cap this.
-- **No llama.cpp GGUF path.** FLM uses its own pre-compiled kernels. Our UD-Q8_K_XL and HauhauCS builds are not importable. NPU work means FLM-specific model tags.
-- **NPU production role (2026-08-16).** `frank-default-small` serves `qwen3.6-moe:35b-a3b` on port 52625 (swap churn and the unknown-tag incident are recorded above). The remaining NPU content (benchmarks, runbook steps) is still research until a step lands and is recorded.
-
-## Citations
-
-1. ROCm/FastFlowLM README (chip support, Linux support date, ROCm move, quick start, model storage paths): https://github.com/ROCm/FastFlowLM , accessed 2026-08-16.
-2. FastFlowLM docs overview: https://fastflowlm.com/docs/ , accessed 2026-08-16.
-3. FastFlowLM Qwen model cards (tags, quantization, context lengths): https://fastflowlm.com/docs/models/qwen/ , accessed 2026-08-16.
-4. FastFlowLM gpt-oss model card (15.1 GB weight size, ~15.6 GB NPU cap, 32 GB inconsistency note): https://fastflowlm.com/docs/models/gpt-oss/ , accessed 2026-08-16.
-5. FastFlowLM Qwen3.5 benchmarks (v0.9.38, Kraken Point 32 GB): https://fastflowlm.com/docs/benchmarks/qwen3.5_results/ , accessed 2026-08-16.
-6. FastFlowLM Qwen3.6 benchmarks (v0.9.45, Kraken Point 96 GB): https://fastflowlm.com/docs/benchmarks/qwen3.6_results/ , accessed 2026-08-16.
-7. AMD blog, "FastFlowLM Joins AMD to Advance AI Inference" (2026-07-17): https://www.amd.com/en/blogs/2026/fastflowlm-joins-amd-to-advance-ai-inference.html
-8. Wikipedia, "Zen 5" article, Strix Halo section (Ryzen AI MAX+ 395: XDNA2 NPU, 50 TOPS, 256-bit LPDDR5X) via local ZIM `wikipedia_en_all_maxi_2026-02`.
-9. Live system checks on Dankclaw, 2026-08-16: `flm validate` output, `dpkg -l` package list, kernel 7.0.0-29-generic, `/proc/3052096/fd` device fd inspection.

docs/LLM/Pipeline Parallelism/Ethernet Bandwidth Math.md

@@ -1,530 +0,0 @@
----
-type: reference
-title: Ethernet Bandwidth Math
-description: Bytes-per-token math for pipeline vs tensor parallelism, feasibility tables for 1/2.5/10 GbE, and real multi-node benchmark numbers.
-tags:
-  - pipeline-parallelism
-  - research
-timestamp: 2026-08-17
----
-
-# Bandwidth Analysis: Multi-Node LLM Inference over Ethernet
-
-## 0. Measured on the Qwen3.8-27B PP deployment (2026-08-17)
-
-Everything in this section was measured on the live deployment
-([[Qwen3.8 PP Deployment]]) while sampling `/proc/net/dev` at 50 Hz on
-both ends. The theoretical tables below use H=4096/L=32; the real model
-is larger (see constants), which raises the per-token bytes but not the
-conclusions.
-
-### 0.1 Verified constants (from the GGUF + env, not guessed)
-
-| Constant | Value | Source |
-| -------- | ----- | ------ |
-| `hidden_size` (embedding_length) | 5120 | GGUF metadata (`qwen35.embedding_length`) |
-| Blocks | 65 (45 on worker, 20 on head) | `--tensor-split 0.68,0.32` |
-| FA layers | 16 total, interval 4 (11 worker / 5 head) | GGUF `full_attention_interval` |
-| KV per FA layer per token | 3136 B (K q8_0 1088 + V f16 2048) | env file comment, confirmed |
-| ubatch | 256 | `UBATCH_SIZE` in `frank-default-large.env` |
-| MTP verify step positions | 5 (1 + n_max 4) | `SPEC_DRAFT_N_MAX=4` |
-| Link (old path) | 1 GbE via USB-C adapter on claw | `enx00e04cc2e1e5`, negotiated 1000 Mb/s |
-| Link (current path) | 2.5 GbE eno1 to eno1 | negotiated 2500 Mb/s both ends |
-| iperf3, 2.5 GbE path | 281 MB/s claw->server, 294 MB/s server->claw | measured |
-| ping RTT, 1 GbE USB path | min 0.239 / avg 0.742 / max 1.085 / mdev 0.334 ms | measured |
-| ping RTT, 2.5 GbE path | min 0.224 / avg 0.585 / max 0.883 / mdev 0.211 ms | measured |
-
-The USB-C adapter added ~15 us to RTT (0.239 vs 0.224 ms min) and more
-jitter (mdev 0.334 vs 0.211 ms).
-
-### 0.2 Measured wire bytes (the number that matters)
-
-Raw hidden state per position per direction = 5120 x 2 = **10.0 KB**.
-The *measured* bytes per position per direction are ~4x that:
-
-| Direction | Measured | Raw hidden |
-| --------- | -------- | ---------- |
-| head -> worker | 39.8 KB/position | 10.0 KB |
-| worker -> head | 38.4 KB/position | 10.0 KB |
-
-The extra ~30 KB/position is the serialized RPC graph (per-position
-nodes for all 45 worker blocks: attention, KV append, SSM ops)
-plus protocol framing. It scales with position count just like the
-payload, so it does not shrink away.
-
-Per ubatch (256 positions) the measured crossing is **20.5 MB**
-(10.4 MB head->worker + 10.1 MB worker->head).
-
-### 0.3 Measured throughput and the validated model
-
-Measured prefill (fresh 13,959-token prompt, 14k context, 2.5 GbE
-path): **457.9 t/s** (30,485 ms, journal task 11924). Decomposition:
-
-| Term | 2.5 GbE (measured anchor) |
-| ---- | ------------------------- |
-| wire per ubatch (20.5 MB) | 70 ms |
-| compute + overhead per ubatch (residual) | 489 ms |
-| ubatch total | 559 ms = 2.18 ms/token |
-
-Using that residual and only swapping the wire time per link speed:
-
-| Link | wire/ubatch | predicted prefill | avg link use | line-rate burst duty |
-| ---- | ----------- | ----------------- | ------------ | -------------------- |
-| 1 GbE (old USB path) | 174 ms | **386 t/s** | 15.8% | 26.2% |
-| 2.5 GbE (current) | 70 ms | 458 t/s (matches measured) | 6.4% | 12.5% |
-| 10 GbE | 17 ms | **505 t/s** | 1.6% | 3.4% |
-
-Delta: 1 -> 2.5 GbE is **+18.6% prefill**; 2.5 -> 10 GbE is **+9.4%**.
-
-### 0.4 The KV snapshot: where the link really saturates
-
-On task release, `llama-server` saves the conversation to the prompt
-cache. The KV for the 11 FA layers living on the worker GPU must be
-copied **worker -> head** for that snapshot, and the slot is BLOCKED
-while it runs (`PARALLEL=1` = one user queues behind it).
-
-Measured for a 98,100-token conversation: 3,454 MB copied (model check:
-98100 x 11 x 3136 B = 3.38 GB, matches), at line rate (254 MB/s avg,
-308 MB/s peak).
-
-| Link | snapshot duration (slot blocked) |
-| ---- | -------------------------------- |
-| 1 GbE | ~31 s |
-| 2.5 GbE | 13.6 s (measured; journal: "prompt cache update took 14418 ms") |
-| 10 GbE | ~5 s |
-
-This is the only phase that runs the link at ~100% utilization for a
-long time. Prefill runs at 12-26% of line rate in bursts (per ubatch)
-and ~6-16% on average; decode runs at ~2% or less.
-
-### 0.5 Decode (MTP verify step, 5 positions)
-
-| Link | net time per step | share of 164 ms step | decode t/s |
-| ---- | ----------------- | -------------------- | ---------- |
-| 1 GbE | 4.0 ms (RTT 0.74 + wire 3.4) | 2.4% | ~23.0 |
-| 2.5 GbE | 1.9 ms (RTT 0.59 + wire 1.4) | 1.1% | 23.47 (measured) |
-| 10 GbE | 0.6 ms (RTT ~0.25 + wire 0.3) | 0.4% | ~23.5 |
-
-Decode is latency-touched but bandwidth-trivial. Average wire rate
-~19 Mbit/s in each direction.
-
-### 0.6 Verdict for the 10 GbE USB-C adapter question
-
-- The 1 GbE -> 2.5 GbE upgrade (the big win) is already captured for
-  free: the claw's native 2.5 GbE `eno1` now carries the RPC path
-  (the 1 GbE USB adapter was unplugged 2026-08-17; routing moved the
-  path, the RPC TCP re-attached on the new interface).
-- 2.5 -> 10 GbE buys +9.4% prefill and 13.6 s -> 5 s snapshot blocking.
-  It requires the L2 device at 192.168.0.1 to be 10G-capable, or a
-  direct cable between the two machines (dedicated link, second IP
-  pair, or a lower-metric static route).
-- At current usage (PARALLEL=1, decode-heavy), 2.5 GbE is comfortable.
-
-## 1. Bandwidth Math: Bytes per Token per Layer Boundary
-
-### 1.1 Pipeline Parallelism (PP)
-
-During autoregressive generation, each transformer layer produces a hidden state of shape `(batch_size, hidden_size)`. At a pipeline stage boundary, this tensor must be communicated between nodes.
-
-**Forward pass only (inference):**
-```
-bytes_per_boundary = batch_size × hidden_size × 2   [fp16 = 2 bytes]
-```
-
-**Key detail:** During *generation* (one token at a time), seq_len = 1 at each boundary crossing. The KV cache stays local. The hidden state is `(batch, hidden_size)` — independent of context length.
-
-**For a 27B dense model (e.g., Qwen3.6-27B):**
-- Hidden size: H = 4096 (standard for Llama/Qwen-style architectures).
-  Note: the Qwen3.8-27B actually deployed on this fleet has H = 5120
-  (GGUF-verified); see section 0, which supersedes these theoretical
-  numbers with measurements.
-- Per-token, per-boundary: `4096 × 2 = 8,192 bytes = 8 KB`
-- With Nₚₚ stage boundaries: total = `Nₚₚ × 8 KB/tok`
-
-**For a 36B MoE model (e.g., Qwen3.6-35B-A3B):**
-- Hidden size: H = 6144 (common for this scale)
-- Layers: L = 48 (typical)
-- Per-token, per-boundary: `6144 × 2 = 12,288 bytes = 12 KB`
-- With Nₚₚ stage boundaries: total = `Nₚₚ × 12 KB/tok`
-
-**Full example — 27B model, 2 PP stages (1 boundary), batch=1:**
-```
-1 boundary × 4096 × 2 bytes = 8,192 bytes/tok
-```
-
-**Example — same model, batch=64:**
-```
-1 boundary × 4096 × 64 × 2 bytes = 524,288 bytes = 512 KB/tok
-```
-
-**Example — 36B model, 2 PP stages, batch=1:**
-```
-1 boundary × 6144 × 2 bytes = 12,288 bytes = 12 KB/tok
-```
-
-### 1.2 Tensor Parallelism (TP) — for comparison
-
-In tensor parallelism, each MLP and attention layer performs an all-reduce on the hidden state:
-
-```
-bytes_per_layer = batch_size × hidden_size × 2   [one all-reduce per layer]
-total_bytes_per_tok = L × batch_size × hidden_size × 2   [L all-reduces total]
-```
-
-**For 27B, H=4096, L=32, TP=2, batch=1:**
-```
-32 layers × 4096 × 1 × 2 bytes = 262,144 bytes = 256 KB/tok
-(16 KB per layer all-reduce × 32 layers)
-```
-
-**For 36B, H=6144, L=48, TP=2, batch=1:**
-```
-48 layers × 6144 × 1 × 2 bytes = 589,824 bytes = 576 KB/tok
-(12 KB per layer all-reduce × 48 layers)
-```
-
-### 1.3 Scaling with batch size
-
-| Model | H | L | tok/s | Batch | PP boundary bytes/tok | TP total bytes/tok |
-|-------|-----|-----|-------|-------|----------------------|--------------------|
-| 27B | 4096 | 32 | 10 | 1 | 8 KB | 256 KB |
-| 27B | 4096 | 32 | 50 | 1 | 8 KB | 256 KB |
-| 27B | 4096 | 32 | 100 | 1 | 8 KB | 256 KB |
-| 27B | 4096 | 32 | 10 | 16 | 128 KB | 4,096 KB |
-| 27B | 4096 | 32 | 50 | 16 | 128 KB | 4,096 KB |
-| 27B | 4096 | 32 | 100 | 16 | 128 KB | 4,096 KB |
-| 27B | 4096 | 32 | 10 | 64 | 512 KB | 16,384 KB |
-| 27B | 4096 | 32 | 50 | 64 | 512 KB | 16,384 KB |
-| 27B | 4096 | 32 | 100 | 64 | 512 KB | 16,384 KB |
-| 36B | 6144 | 48 | 10 | 1 | 12 KB | 576 KB |
-| 36B | 6144 | 48 | 50 | 1 | 12 KB | 576 KB |
-| 36B | 6144 | 48 | 100 | 1 | 12 KB | 576 KB |
-| 36B | 6144 | 48 | 10 | 16 | 192 KB | 9,216 KB |
-| 36B | 6144 | 48 | 50 | 16 | 192 KB | 9,216 KB |
-| 36B | 6144 | 48 | 100 | 16 | 192 KB | 9,216 KB |
-| 36B | 6144 | 48 | 10 | 64 | 768 KB | 36,864 KB |
-| 36B | 6144 | 48 | 50 | 64 | 768 KB | 36,864 KB |
-| 36B | 6144 | 48 | 100 | 64 | 768 KB | 36,864 KB |
-
----
-
-## 2. Required Bandwidth vs. Ethernet Capabilities
-
-### 2.1 Ethernet link capacities
-
-| Link | Raw Rate | Usable TCP (approx.) | MB/s usable |
-|------|----------|---------------------|-------------|
-| 1 GbE | 1,000 Mbit/s | ~940 Mbit/s | ~118 MB/s |
-| 2.5 GbE | 2,500 Mbit/s | ~2,350 Mbit/s | ~294 MB/s |
-| 10 GbE | 10,000 Mbit/s | ~9,400 Mbit/s | ~1,175 MB/s |
-
-### 2.2 Required bandwidth (Mbps) by model, batch, and throughput
-
-**27B model (H=4096), 1 PP boundary:**
-
-| tok/s | Batch=1 | Batch=16 | Batch=64 |
-|-------|---------|----------|----------|
-| 10 | 0.13 Mbit/s | 2.05 Mbit/s | 8.19 Mbit/s |
-| 50 | 0.65 Mbit/s | 10.24 Mbit/s | 40.96 Mbit/s |
-| 100 | 1.31 Mbit/s | 20.48 Mbit/s | 81.92 Mbit/s |
-
-**36B model (H=6144), 1 PP boundary:**
-
-| tok/s | Batch=1 | Batch=16 | Batch=64 |
-|-------|---------|----------|----------|
-| 10 | 0.20 Mbit/s | 3.07 Mbit/s | 12.29 Mbit/s |
-| 50 | 0.98 Mbit/s | 15.36 Mbit/s | 61.44 Mbit/s |
-| 100 | 1.97 Mbit/s | 30.72 Mbit/s | 122.88 Mbit/s |
-
-**27B model, TP (L=32 layers all-reduce):**
-
-| tok/s | Batch=1 | Batch=16 | Batch=64 |
-|-------|---------|----------|----------|
-| 10 | 2.05 Mbit/s | 32.77 Mbit/s | 131.07 Mbit/s |
-| 50 | 10.24 Mbit/s | 163.84 Mbit/s | 655.36 Mbit/s |
-| 100 | 20.48 Mbit/s | 327.68 Mbit/s | 1,310.72 Mbit/s |
-
-### 2.3 Feasibility matrix
-
-Green = feasible (< 50% of link capacity). Yellow = marginal (50–90%). Red = saturated (> 90%).
-
-**27B model — PP (1 boundary), batch=1:**
-
-| | 1 GbE (118 MB/s) | 2.5 GbE (294 MB/s) | 10 GbE (1175 MB/s) |
-|---|---|---|---|
-| 10 tok/s | ✅ 0.13 Mbit/s | ✅ 0.13 Mbit/s | ✅ 0.13 Mbit/s |
-| 50 tok/s | ✅ 0.65 Mbit/s | ✅ 0.65 Mbit/s | ✅ 0.65 Mbit/s |
-| 100 tok/s | ✅ 1.31 Mbit/s | ✅ 1.31 Mbit/s | ✅ 1.31 Mbit/s |
-
-**27B model — PP (1 boundary), batch=16:**
-
-| | 1 GbE | 2.5 GbE | 10 GbE |
-|---|---|---|---|
-| 10 tok/s | ✅ 2.05 Mbit/s | ✅ 2.05 Mbit/s | ✅ 2.05 Mbit/s |
-| 50 tok/s | ✅ 10.24 Mbit/s | ✅ 10.24 Mbit/s | ✅ 10.24 Mbit/s |
-| 100 tok/s | ✅ 20.48 Mbit/s | ✅ 20.48 Mbit/s | ✅ 20.48 Mbit/s |
-
-**27B model — PP (1 boundary), batch=64:**
-
-| | 1 GbE | 2.5 GbE | 10 GbE |
-|---|---|---|---|
-| 10 tok/s | ✅ 8.19 Mbit/s | ✅ 8.19 Mbit/s | ✅ 8.19 Mbit/s |
-| 50 tok/s | ✅ 40.96 Mbit/s | ✅ 40.96 Mbit/s | ✅ 40.96 Mbit/s |
-| 100 tok/s | ✅ 81.92 Mbit/s | ✅ 81.92 Mbit/s | ✅ 81.92 Mbit/s |
-
-**27B model — TP (32 layers), batch=1:**
-
-| | 1 GbE | 2.5 GbE | 10 GbE |
-|---|---|---|---|
-| 10 tok/s | ✅ 2.05 Mbit/s | ✅ 2.05 Mbit/s | ✅ 2.05 Mbit/s |
-| 50 tok/s | ✅ 10.24 Mbit/s | ✅ 10.24 Mbit/s | ✅ 10.24 Mbit/s |
-| 100 tok/s | ✅ 20.48 Mbit/s | ✅ 20.48 Mbit/s | ✅ 20.48 Mbit/s |
-
-**27B model — TP (32 layers), batch=16:**
-
-| | 1 GbE | 2.5 GbE | 10 GbE |
-|---|---|---|---|
-| 10 tok/s | ✅ 32.77 Mbit/s | ✅ 32.77 Mbit/s | ✅ 32.77 Mbit/s |
-| 50 tok/s | ✅ 163.84 Mbit/s | ✅ 163.84 Mbit/s | ✅ 163.84 Mbit/s |
-| 100 tok/s | ✅ 327.68 Mbit/s | ✅ 327.68 Mbit/s | ✅ 327.68 Mbit/s |
-
-**27B model — TP (32 layers), batch=64:**
-
-| | 1 GbE | 2.5 GbE | 10 GbE |
-|---|---|---|---|
-| 10 tok/s | ✅ 131.07 Mbit/s | ✅ 131.07 Mbit/s | ✅ 131.07 Mbit/s |
-| 50 tok/s | ✅ 655.36 Mbit/s | ✅ 655.36 Mbit/s | ✅ 655.36 Mbit/s |
-| 100 tok/s | ✅ 1310.72 Mbit/s | ✅ 1310.72 Mbit/s | ✅ 1310.72 Mbit/s |
-
-**27B TP batch=64 at 100 tok/s: 1310.72 Mbit/s > 1 GbE usable (1000 Mbit/s) — link-saturated on 1 GbE**
-
-**27B TP batch=64 at 100 tok/s: 1310.72 Mbit/s ≈ 2.5 GbE usable (2500 Mbit/s) — ~52% saturated**
-
-**Key finding:** PP is feasible across all ethernet classes for all batch sizes up to at least 100 tok/s. TP becomes bottlenecked on 1 GbE at batch=64 and 100 tok/s.
-
----
-
-## 3. Why PP is the Only Realistic Option over Ethernet
-
-### 3.1 Communication cost comparison
-
-| Scheme | Communication per tok per rank | 27B, H=4096, batch=1 | 36B, H=6144, batch=1 |
-|--------|------|---|---|
-| PP (Nₚₚ boundaries) | Nₚₚ × H × 2 | Nₚₚ × 8 KB | Nₚₚ × 12 KB |
-| TP (L layers all-reduce) | L × H × 2 | 32 × 8 KB = 256 KB | 48 × 12 KB = 576 KB |
-
-PP communication per token is **L/Nₚₚ times smaller** than TP communication per token (factor of ~16× for 2 PP stages).
-
-### 3.2 Latency-bound nature of autoregressive generation
-
-During autoregressive generation, each layer computes one token at a time. In TP, every layer requires an all-reduce (2× ring latency or 1× tree latency per layer). For L=32 layers, that's 32 synchronization points per token.
-
-In PP, activations cross the stage boundary only once per layer (or L/Nₚₚ times). The critical path is:
-
-```
-Stage 0 computes → send hidden state across boundary → Stage 1 computes → ...
-```
-
-This is a single bottleneck per token, with latency proportional to network RTT × number of stages, not L.
-
-### 3.3 Bandwidth utilization
-
-With PP, each boundary crossing transfers `H × 2` bytes. For 8 KB at 1 GbE, the transfer completes in ~68 µs. This is negligible compared to compute time (milliseconds per layer on a GPU). The link is underutilized.
-
-With TP, each layer transfers `H × 2` bytes but this happens L times. For 256 KB total at 1 GbE, that's ~2 ms of pure transfer time — potentially comparable to compute time on a fast GPU.
-
-### 3.4 Real-world evidence
-
-- Petals uses PP exclusively over the public internet and achieves usable inference at 1–2 tok/s even with 100 Mbit/s links [1].
-- llama.cpp RPC splits layers across nodes in pipeline order, not tensor sharding [2].
-- exo uses TP but achieves speedup only with RDMA over Thunderbolt (~50 Gbit/s) [3].
-- Petals paper Table 3: inference performance does **not depend much on bandwidth** (1 Gbit/s vs 100 Mbit/s gives nearly identical results) but **degrades significantly with higher latency** [1].
-
-**Conclusion:** For gigabit-class ethernet, PP is the only practical multi-node parallelism strategy for inference. TP can work on 10 GbE for small batches but saturates at 1 GbE with moderate batch sizes.
-
----
-
-## 4. Real-World Multi-Node Inference Benchmarks
-
-### 4.1 Summary table
-
-| System | Model | Hardware | # Nodes | Network | tok/s (batch=1) | Notes |
-|--------|-------|----------|---------|---------|-----------------|-------|
-| Petals (local) | BLOOM-176B | 3× A100 80GB | 3 | 1 Gbit/s, <5 ms RTT | 1.71 | PP, 22 stages, quantized [1] |
-| Petals (local) | BLOOM-176B | 3× A100 80GB | 3 | 100 Mbit/s, <5 ms RTT | 1.66 | ~3% slowdown vs 1 Gbit/s [1] |
-| Petals (local) | BLOOM-176B | 3× A100 80GB | 3 | 100 Mbit/s, 100 ms RTT | 1.23 | Latency matters more than BW [1] |
-| Petals (wan) | BLOOM-176B | 14 consumer GPUs | 14 | Real internet, 100–1000 Mbit/s | 0.83 | Europe+N. America [1] |
-| Petals | Llama 2 70B | consumer GPUs | various | Internet | up to 6 | Interactive chat [4] |
-| Petals | Falcon 180B | consumer GPUs | various | Internet | up to 4 | Interactive chat [4] |
-| llama.cpp RPC | Qwen3-235B | 4× M3 Ultra Mac Studio | 4 | Ethernet (2.5 GbE) | ~20 | Scales down with more nodes [3] |
-| llama.cpp RPC | DeepSeek v3.1 | 4× M3 Ultra Mac Studio | 4 | Thunderbolt 5 | ~25 | Scales down with more nodes [3] |
-| llama.cpp RPC | Kimi K2 (1T) | 4× M3 Ultra Mac Studio | 4 | Thunderbolt 5 | ~30 | A32B sparse, 32B active [3] |
-| Exo (RDMA) | Qwen3-235B | 4× M3 Ultra Mac Studio | 4 | TB5 RDMA (~50 GbE) | 32 | TP, scales UP with more nodes [3] |
-| Exo (RDMA) | DeepSeek v3.1 | 4× M3 Ultra Mac Studio | 4 | TB5 RDMA (~50 GbE) | ~30 | 671B dense TP [3] |
-| vLLM PP | Llama-3-70B | 2× A100 80GB | 2 | NVLink + Ethernet | ~15 | PP across nodes, TP within [5] |
-
-### 4.2 Detailed notes
-
-**Petals [1]:**
-- Uses pipeline parallelism with dynamic 8-bit quantization (halves bandwidth)
-- For BLOOM-176B (H=14336), per-boundary communication is ~14 KB/tok with quantization
-- Recommended bandwidth: 25 Mbit/s bidirectional for interactive inference [1]
-- Key finding: "For inference, performance does not depend much on bandwidth or sequence length but degrades with higher latency" — RTT matters more than throughput
-- At 1 Gbit/s with <5 ms RTT: 1.71 steps/s; at 100 Mbit/s with <5 ms: 1.66 steps/s (only 3% difference)
-- At 100 Mbit/s with 100 ms RTT: 1.23 steps/s (28% degradation)
-
-**llama.cpp RPC [2]:**
-- Model weights and KV cache distributed across nodes proportionally to memory
-- Default: pipeline-style distribution of layers (first layers on node A, last on node B)
-- Each node runs `ggml-rpc-server`, main host uses `--rpc` flag
-- RDMA transport supported (RoCEv2), negotiated automatically with libibverbs
-- Over ethernet, performance decreases with more nodes (each extra node adds a hop)
-- Over RDMA/TB5, performance scales up with more nodes (less latency per hop)
-
-**Exo [3]:**
-- Uses tensor parallelism with MLX distributed backend
-- With RDMA over Thunderbolt 5: up to 1.8× speedup on 2 devices, 3.2× on 4 devices
-- Topology-aware auto-parallelism considers latency/bandwidth between links
-- Thunderbolt 5 provides ~50–60 GbE real-world throughput (RDMA)
-
-**Key insight from all systems:** Ethernet (1 Gbit/s) is sufficient for PP inference of 20–40B models because per-token activation traffic is small (8–12 KB/boundary). The bottleneck is **latency** (RTT between nodes), not bandwidth.
-
----
-
-## 5. Overheads
-
-### 5.1 TCP/IP overhead
-
-| Overhead | Amount |
-|----------|--------|
-| Ethernet header + trailer | 18 bytes (14 + 4 FCS) |
-| IPv4 header (no options) | 20 bytes |
-| TCP header (no options) | 20 bytes |
-| Total per packet | ~58 bytes overhead |
-| MTU (jumbo frames 9000) | 9000 bytes payload → 0.6% overhead |
-| MTU (1500 standard) | 1500 bytes payload → 3.9% overhead |
-
-**Practical impact:** For 8 KB transfers (27B PP boundary), use jumbo frames where possible to minimize TCP overhead. For small transfers (< 1.5 KB), a single TCP packet is used and the overhead is 58/8192 = 0.7% per transfer — negligible.
-
-### 5.2 RPC/serialization overhead
-
-- llama.cpp RPC: Protobuf serialization adds ~10–20% overhead for small tensors
-- Petals: PyTorch tensor serialization over hivemind/gRPC, adds serialization time
-- vLLM PP: gRPC with protobuf, similar overhead
-- Serialization scales with tensor size: negligible for 8 KB+ tensors
-
-### 5.3 Pipeline bubble overhead
-
-With Nₚₚ pipeline stages and microbatching, the pipeline bubble fraction is:
-
-```
-bubble_fraction = (Nₚₚ - 1) / (Nₚₚ + microbatches - 1)
-```
-
-For 2 stages, 1 microbatch (single tok): bubble = 1/2 = 50% idle time.
-For 2 stages, 8 microbatches: bubble = 1/9 = 11%.
-
-In autoregressive generation with batch=1 and no microbatching, each stage processes one token sequentially:
-- Stage 0 finishes, sends hidden state, waits for Stage 1
-- Stage 1 processes, returns result, Stage 0 processes next token
-- Effective utilization ≈ 1/Nₚₚ if no pipelining overlap
-
-**Mitigation:** Process multiple tokens before waiting. Send next token to Stage 0 while Stage 1 is computing — pipelining reduces bubble.
-
-### 5.4 Per-hop LAN latency
-
-| Network | Typical RTT | One-way latency |
-|---------|------------|-----------------|
-| 1 GbE (Cat6) | 30–50 µs | 15–25 µs |
-| 2.5 GbE (Cat6) | 30–50 µs | 15–25 µs |
-| 10 GbE (Cat6a) | 20–30 µs | 10–15 µs |
-| RoCEv2 (IB/25GbE) | 1–5 µs | <5 µs |
-| RDMA (Thunderbolt 5) | 10–30 µs round-trip end-to-end | — |
-
-**Impact on PP inference:** With Nₚₚ = 2 stages and batch=1, one token takes:
-```
-t_compute + t_roundtrip + t_compute
-```
-
-At 30 µs RTT and ~2 ms layer compute time per GPU: the network adds ~30 µs, which is 1.5% of total. Negligible at LAN speeds.
-
-With Nₚₚ = 4 and 30 µs per-hop RTT: ~120 µs network, still <6% of compute time.
-
-### 5.5 Quantization overhead
-
-- Petals uses dynamic blockwise 8-bit quantization: halves hidden state bandwidth from 16-bit to 8-bit [1]
-- For 27B: 8 KB → 4 KB per boundary with 8-bit quantization
-- Accuracy impact: <1% on standard benchmarks [1]
-- Quantization/dequantization adds ~1–5 ms per stage on modern GPUs (negligible vs. compute)
-
----
-
-## 6. Conclusions and Recommendations
-
-### For the 27B / 36B models on gigabit-class ethernet:
-
-1. **1 GbE is sufficient for PP inference** up to at least 100 tok/s at batch=64 (81.92 Mbit/s required, well under 1000 Mbit/s capacity). Even with TCP/IP and RPC overheads, margin is >90%.
-
-2. **Pipeline parallelism is the correct choice** over ethernet — PP sends 8–12 KB per boundary vs. 256–576 KB for TP, a 32× advantage for 2 PP stages.
-
-3. **Latency matters more than bandwidth** — the Petals paper demonstrates that dropping from 1 Gbit/s to 100 Mbit/s only reduces throughput by ~3%, while increasing RTT from 5 ms to 100 ms reduces it by ~28%.
-
-4. **With 8-bit quantization**, bandwidth requirements halve, providing additional safety margin.
-
-5. **Jumbo frames recommended** to minimize TCP overhead on larger transfers.
-
-6. **For batch sizes ≥ 16**, 1 GbE remains viable for PP but consider 2.5 GbE for headroom and faster model loading.
-
----
-
-## References
-
-[1] Borzunov et al., "Petals: Collaborative Inference and Fine-tuning of Large Models," ACL 2023. arXiv:2209.01188. Available at: https://arxiv.org/pdf/2209.01188 — Primary benchmark source: Table 3 shows inference throughput at various bandwidths and latencies for BLOOM-176B. Key finding: inference insensitive to bandwidth, sensitive to latency. Recommended 25 Mbit/s bidirectional bandwidth.
-
-[2] ggml-org/llama.cpp RPC documentation. Available at: https://github.com/ggml-org/llama.cpp/blob/master/tools/rpc/README.md — Architecture for distributed LLM inference via RPC. Supports RDMA transport (RoCEv2), automatic negotiation, local tensor caching.
-
-[3] exo-explore/exo README. Available at: https://github.com/exo-explore/exo — Multi-node LLM inference on Apple Silicon with TP and RDMA over Thunderbolt 5. Benchmarks with Qwen3-235B, DeepSeek v3.1, Kimi K2 on 4× M3 Ultra Mac Studio.
-
-[4] Petals chatbot demo and health monitor. Available at: https://chat.petals.dev, https://health.petals.dev — Real-world performance of Llama 2 70B (up to 6 tok/s) and Falcon 180B (up to 4 tok/s) on public swarm.
-
-[5] vLLM pipeline parallel documentation. Available at: https://docs.vllm.ai/en/stable/dev/engine/llm_engine.html — vLLM supports PP across nodes using gRPC with CUDA-graph support for low-latency stage transitions.
-
----
-
-## Appendix: Full bandwidth formulae
-
-### Pipeline Parallelism (generation, batch=1)
-
-```
-bytes_per_boundary_per_tok = hidden_size × 2   [fp16]
-total_bytes_per_tok = num_boundaries × hidden_size × 2
-bandwidth_mbps = total_bytes_per_tok × tokens_per_second × 8 / 1,000,000
-```
-
-### Pipeline Parallelism (generation, batch=N)
-
-```
-bytes_per_boundary_per_tok = hidden_size × N × 2
-total_bytes_per_tok = num_boundaries × hidden_size × N × 2
-```
-
-### Tensor Parallelism (generation, batch=1)
-
-```
-bytes_per_layer_allreduce = hidden_size × 2
-total_bytes_per_tok = num_layers × hidden_size × 2
-bandwidth_mbps = total_bytes_per_tok × tokens_per_second × 8 / 1,000,000
-```
-
-### Tensor Parallelism (generation, batch=N)
-
-```
-total_bytes_per_tok = num_layers × hidden_size × N × 2
-```
-
-### With 8-bit quantization (like Petals)
-
-```
-# Simply divide all fp16 byte counts by 2
-bytes_per_boundary_per_tok = hidden_size × 1   [int8]
-```

docs/LLM/Pipeline Parallelism/Expected Performance.md

@@ -1,97 +0,0 @@
----
-type: reference
-title: Expected Performance
-description: Estimated tokens per second for a 25-40B model split across two nodes over 1/2.5/10 GbE, with real benchmark anchors.
-tags:
-  - pipeline-parallelism
-  - performance
-  - estimate
-timestamp: 2026-08-14
----
-
-# Expected Performance
-
-This page estimates the speed of a 25-40B model split across two machines
-over ethernet. It uses the bandwidth math in [[Ethernet Bandwidth Math]]
-and the measured numbers in [[Mixed-Vendor Experiments]].
-
-## The Bandwidth Ceiling Is Not the Limit
-
-A pipeline boundary moves only the hidden state. For a 27B model (hidden
-5120), that is 10240 bytes per token per boundary in fp16. Even at 64
-tokens per second with batch 64, the traffic is about 262 Mbit/s. 1 GbE can
-carry it. The real limits are latency, RPC overhead, and the slowest node.
-
-The Petals paper measured this directly. Dropping from 1 Gbit/s to
-100 Mbit/s changed throughput by 3 percent. Raising round-trip latency from
-5 ms to 100 ms dropped throughput by 28 percent. Latency matters more than
-bandwidth for pipeline inference.
-
-## Real Benchmark Anchors
-
-| Experiment | Model | Network | Decode (t/s) |
-| ---------- | ----- | ------- | ------------ |
-| Mac Studio + DGX Spark (Metal+CUDA) | Qwen2.5-72B Q4 | 10 GbE | 6 |
-| Mac Studio + DGX Spark (Metal+CUDA) | Qwen2.5-7B Q4 | 10 GbE | 53 |
-| RTX 4070 + MI50 + 3050 (CUDA+ROCm) | Qwen3-VL-30B Q5 | 1 GbE | 51.5 clean, 17.8 at 32k ctx |
-| Two Strix Halos (ROCm RPC) | MiniMax-M2.5-139B Q8 | USB4 | 15.35 |
-| AMD four-node Strix Halo (5 GbE) | Kimi K2.5 (1T) | 5 GbE | 8.30 |
-| Tesla V100 + R9700x2 (Vulkan, one box) | Qwen3.6-35B-A3B Q8 | PCIe | 62 |
-
-Pattern: RPC decode runs about 2-5 times slower than the best single node.
-Prefill benefits from more compute across nodes. The 30B-class model over
-1 GbE with warm cache reached 51.5 t/s on a short context and 17.8 t/s at
-32k context.
-
-## Estimate for Qwen3.6-27B Q8, Two Nodes
-
-Scenario: RTX 3090 (24 GB VRAM, NVIDIA desktop) plus Strix Halo NUC
-(Radeon 8060S, 30 GB physical RAM). The model is the dense 27B at Q8
-(29-32 GB). Each node holds part of the weights and KV cache.
-
-| Link | Decode (t/s) | Prefill (t/s) | Notes |
-| ---- | ------------ | ------------- | ----- |
-| 1 GbE | 8-15 | 200-500 | Latency-bound; per-token RPC round trip |
-| 2.5 GbE | 10-18 | 250-600 | Some headroom; latency still dominant |
-| 10 GbE | 12-20 | 300-800 | Closest to the published 10 GbE runs |
-
-The dense 27B decodes slower per token than the MoE 30B A3B class. Expect
-the lower half of the ranges at full context.
-
-### With Speculative Decoding
-
-Speculative decoding (MTP, DFlash, or ngram-mod) raises effective decode.
-Tightwad measured 1.86x on a slow RPC pool. Estimate the effective rate at
-1.4-1.8x the raw RPC decode:
-
-| Link | Raw decode (t/s) | With spec (t/s) |
-| ---- | ---------------- | --------------- |
-| 1 GbE | 8-15 | 12-24 |
-| 2.5 GbE | 10-18 | 15-28 |
-| 10 GbE | 12-20 | 18-32 |
-
-## Context That Fits
-
-The two real machines give about 51 GiB of usable memory combined
-(24 GiB VRAM plus about 27 GiB on the Strix Halo, which has 30 GiB physical
-RAM; the 96 GiB driver ceiling is not physical). A 29-32 GB model leaves
-about 20-22 GiB for KV cache. The Qwen3.6-27B KV cache costs about 85-100
-KiB per token with q8_0 K and f16 V (measured on the large service at
-602100 tokens = about 50 GB). That budget gives a total context pool of
-roughly 200k-260k tokens across both nodes.
-
-## Caveats
-
-- Every node must run the exact same llama.cpp build. Version skew hangs
-  the handshake.
-- The slowest node sets the pace for every token.
-- Model load crosses the network once per start. Expect minutes, not
-  seconds, for the split load.
-- These are estimates. The exact combination of an RTX 3090 desktop and a
-  Strix Halo NUC has no published benchmark yet.
-
-## See Also
-
-- [[Ethernet Bandwidth Math]]
-- [[Mixed-Vendor Experiments]]
-- [[Sample Script Set]]

docs/LLM/Pipeline Parallelism/Mixed-Vendor Experiments.md

@@ -1,228 +0,0 @@
----
-type: reference
-title: Mixed-Vendor Experiments
-description: Real-world NVIDIA + AMD Strix Halo multi-node experiments: what works (llama.cpp RPC, Vulkan split), what breaks (mixed CUDA+ROCm vLLM), measured tok/s.
-tags:
-  - pipeline-parallelism
-  - research
-timestamp: 2026-08-14
----
-
-# Research Brief: NVIDIA-Desktop + Strix-Halo Multi-Node LLM Inference
-
-## Summary
-
-Yes, mixed NVIDIA-ROCm multi-node LLM inference over RPC has been published and is **fully functional**. The closest real experiments to an RTX-desktop + Strix-Halo-NUC pairing are (1) a planned 4× RTX 3090 + Strix Halo 128GB via 25GbE (r/LocalLLaMA, Apr 2026), (2) a Tesla V100 + AMD R9700×2 via Vulkan layer-split on one box (grosan.co.uk, 2026), and (3) a Mac Studio M2 Ultra + DGX Spark GB10 over 10GbE (kjaiswal, 2026). Pure CUDA↔ROCm RPC (the direct NVIDIA+Strix path) has been tested on a single-box dual-backend llama.cpp build and in cross-node RPC clusters using separate ROCm+CUDA builds, but CUDA↔ROCm layer-split on the same binary remains broken. Vulkan is the universal backend that works across vendors.
-
-**Verdict:** The NVIDIA-desktop + Strix-Halo multi-node path via llama.cpp RPC is viable today. Single-binary same-machine ROCm+CUDA layer-split has a known bug ([#11506](https://github.com/ggml-org/llama.cpp/issues/11506)). Cross-node RPC works with separate ROCm and CUDA builds on each machine. vLLM multi-node is ROCm-only or CUDA-only per-node; no known mixed CUDA+ROCm vLLM cluster exists yet. Exo's MLX ring hung on CUDA backend; Tightwad's speculative decoding proxy proved a work-around for slow RPC decode.
-
----
-
-## Findings
-
-### 1. Published Experiments: NVIDIA + AMD Strix Halo Nodes
-
-**1a. Planned 4×RTX 3090 + Strix Halo 128GB cluster over 25GbE**
-User on r/LocalLLaMA (Apr 2026) is building a llama.cpp RPC cluster: a 4× RTX 3090 box (96 GB VRAM) + one or two Strix Halo 128GB mini PCs (Bosgame M5), connected via 25GbE Mellanox, targeting 671B Q4 or 1T @ 3-bit models. Post was seeking community validation before purchasing. [Source](https://www.reddit.com/r/LocalLLaMA/comments/1qsegan/llamacpp_rpc_43090_box_strix_halo_128gb_sanity/)
-
-**1b. Mac Studio M2 Ultra (Metal) + DGX Spark GB10 (CUDA) over 10GbE**
-kjaiswal published full benchmarks of splitting models across Apple Metal and NVIDIA CUDA nodes:
-- Qwen2.5-7B Q4_K_M (4.4 GB): Local Metal 92 tok/s vs RPC Metal+CUDA 53 tok/s decode, but **4.2× faster prefill** (318 vs 76 tok/s). The 7B fits both machines — RPC trades decode speed for prefill speed.
-- Qwen2.5-72B Q4_K_M (44.2 GB): Local Metal 11 tok/s vs RPC Metal+CUDA 6 tok/s decode (~47% penalty). Split: 31 GB Metal + 14 GB CUDA.
-- Projected for models needing both: MiniMax M2.5 (138 GB, ~5-8 tok/s), Qwen3-235B (132 GB, ~5-8 tok/s).
-- **Conclusion:** RPC's value is capacity (100 GB+ models), not speed. Decode is 2× slower over RPC due to per-token network round-trips. [Source](https://github.com/kjaiswal/llama-cpp-distributed-benchmarks)
-
-**1c. Tesla V100 + AMD R9700×2 via Vulkan layer-split (single box)**
-grosan.co.uk documented running Qwen3.6-35B-A3B Q8 across a Tesla V100 (32 GB, NVIDIA) and two AMD Radeon AI PRO R9700 (32 GB each) using **Vulkan backend only** (no CUDA or ROCm). Layer-split across three GPUs from two vendors. Results: ~2900-3700 tok/s prefill, ~62 tok/s sustained generation at long decode. Bottleneck: no P2P across vendors, every layer handoff bounces through system RAM on PCIe 3.0. [Source](https://grosan.co.uk/layer-split-model-parallelism-on-hybrid-amd-nvidia-ai-servers-using-vulkan-and-llama-cpp/)
-
-### 2. Strix Halo in Distributed Inference
-
-**2a. Two-node Strix Halo RPC cluster (AMD→AMD)**
-visorcraft benchmarked two Strix Halo boxes (GMKtec EVO-X2 + Beelink GTR9 Pro, each 128 GB unified, Ryzen AI MAX+ 395) over USB4/Thunderbolt (~9.4 Gbps):
-- MiniMax-M2.5-REAP-139B-Q8: **15.35 tok/s** decode, RPC split 1.2/0.8
-- Qwen3.5-397B-A17B-UD-Q4_K_XL: **11.76 tok/s** decode (llama-bench), **12.6 tok/s** (llama-server with `-dio`)
-- Single-host single-node baseline: MiniMax-M2.5 Q3_K_M at 32.8 tok/s, Qwen3 30B-A3B at 86.1 tok/s
-- Key caveat: RPC serving requires `-dio` flag on large models. [Source](https://github.com/visorcraft/strix-halo-llm-perf)
-
-**2b. AMD official: 4×Framework Desktop (Strix Halo) cluster**
-AMD published a walkthrough running Kimi K2.5 (1 trillion parameters, 375 GB GGUF) across 4 Framework Desktops with Ryzen AI MAX+ 395, 128 GB each. Interconnect: 5 Gbps Ethernet. Flash Attention enabled via rocWMMA. Benchmarks at 8192 sequence length: 8.30 tok/s decode with FA enabled vs 3.46 tok/s without (batch/ubatch=4096). Context: 32768 tokens. [Source](https://www.amd.com/en/developer/resources/technical-articles/2026/how-to-run-a-one-trillion-parameter-llm-locally-an-amd.html)
-
-**2c. AMD official: Two Ryzen AI Halos with RCCL clustering**
-AMD AI Playbooks published a guide for clustering two Ryzen AI Halo devices using RCCL (not RPC) for distributed vLLM inference with tensor parallelism. [Source](https://developer.amd.com/playbooks/clustering-rccl/)
-
-**2d. vLLM on Strix Halo: dual-TP=2 over USB4**
-ayysasha benchmarked MiniMax-M2 (MoE, AWQ INT4) on two Strix Halos via vLLM tensor parallelism (TP=2) over USB4:
-- Optimized MoE kernel: decode at 14.84 tok/s (1k context), 9.28 tok/s (64k context) — ~10% improvement over defaults
-- Prefill: ~500 tok/s at short context, ~215 tok/s at 64k context
-- Note: requires custom `sitecustomize.py` shim for device name detection on gfx1151. [Source](https://github.com/ayysasha/Strix-halo-dual-optimized)
-
-**2e. Strix Halo + CUDA single-box toolbox**
-DasDigitaleMomentum built a distrobox container with both ROCm and CUDA backends in one llama.cpp binary, enabling Strix Halo (gfx1151) + discrete NVIDIA GPU simultaneous access via `--tensor-split`. Also supports dual-backend ROCm+Vulkan. [Source](https://github.com/DasDigitaleMomentum/strix-halo-cuda-combined-toolbox)
-
-### 3. Heterogeneous Node Splitting: CUDA + ROCm RPC
-
-**3a. llama.cpp builds with both GGML_CUDA and GGML_HIP**
-Issue [#11506](https://github.com/ggml-org/llama.cpp/issues/11506) reports that llama.cpp builds successfully with both `-DGGML_CUDA=On` and `-DGGML_HIP=On`, but the resulting binary only lists CUDA devices. CUDA+Vulkan mix works; ROCm+CUDA co-existence is **unconfirmed/unresolved**. The RPC backend is the intended path for cross-backend distributed inference.
-
-**3b. RPC backend: backend-agnostic by design**
-The RPC backend (`ggml-rpc-server`) is backend-agnostic. Each worker runs its own llama.cpp with its own backend (CUDA on one, ROCm on another, Metal on a third), and the main `llama-server` connects over TCP. From llama.cpp's perspective, each remote machine is just another device. [Source](https://sharedllm.org/blog/llama-cpp-rpc-distributed-inference.html)
-
-**3c. Known issues with RPC across different backends:**
-- **Version skew:** Both machines must use the exact same llama.cpp build tag; mismatched builds hang at handshake.
-- **Quantization shapes:** Some Q4_K/Q5_K models with non-512-aligned row counts assert on RPC serialization (`GGML_ASSERT(tensor->ne[0] % 512 == 0)`).
-- **Network speed matters:** WiFi → 10× slowdown vs wired ethernet. Ethernet 1G works but is latency-bound for decode.
-- **No authentication:** `rpc-server` bound to `0.0.0.0` exposes all memory to any network client.
-- **Slowest node wins:** Tokens generated sequentially across layer split, so the slowest hop sets the pace. [Source](https://sharedllm.org/blog/llama-cpp-rpc-distributed-inference.html)
-
-**3d. Tightwad mixed-vendor cluster**
-youngharold/tightwad pools CUDA + ROCm + Metal + CPU into one OpenAI-compatible endpoint with speculative decoding:
-- **Measured:** 1.86× speedup on Llama 3.3 70B across 4 GPUs over WiFi (3 tok/s raw RPC → 4.1 tok/s with spec decode)
-- **Speculative decoding:** Qwen3-235B with Qwen3-1.7B draft model over RPC — overall acceptance rate 6.6%, highest in factual (14.5%) and reasoning (13.8%) categories. Spec decode made slow RPC models actually usable. [Source](https://github.com/youngharold/tightwad)
-- Config example `mixed-vendor-pool.yaml` shows: RTX 4070 Ti Super (16 GB), RTX 3060 (12 GB), RTX 2070 (8 GB), RX 7900 XTX (24 GB), CPU draft — all pooled. [Source](https://github.com/youngharold/tightwad/blob/main/examples/mixed-vendor-pool.yaml)
-
-**3e. Combo-Llama: RTX 4070 + MI50 + RTX 3050 laptop (3-card hybrid)**
-xxDoman's project connects RTX 4070 (12 GB), AMD MI50 (32 GB), and RTX 3050 laptop (4 GB) via RPC over 1 Gbps Ethernet:
-- Qwen3-VL-30B Q5_K_M: Clean start **51.49 tok/s** on Ethernet (WiFi was 44.53 tok/s)
-- 32k context: **17.81 tok/s**, 64k context: **12.90 tok/s** (stable to 65536 tokens)
-- MI50 at 89% GPU load, RTX 4070 at 12% — load imbalance typical. [Source](https://github.com/xxDoman/combo-llama-rpc-hybrid)
-
-### 4. Adjacent Projects
-
-**4a. Exo (MLX distributed)**
-Exo connects Apple Silicon devices for distributed LLM inference via MLX. Tried cross-vendor Metal+CUDA over 10GbE with DGX Spark + Mac Studio. **Result:** Failed — `mx.distributed.init(backend="ring")` hangs on CUDA backend (as of MLX 0.31.1). CUDA ring support from Apple is the blocker. Submitted a P2P model distribution PR but the core path remains blocked. [Source](https://dev.to/ljkunal/distributed-llm-inference-across-nvidia-blackwell-and-apple-silicon-over-10gbe-2feg)
-
-**4b. Petals**
-Distributed inference framework. Not yet tested on Strix Halo. Uses its own custom protocol over TCP. [General reference]
-
-**4c. llama.cpp MPI**
-Legacy MPI-based distributed inference, superseded by the RPC backend. RPC is now the standard multi-node path. [llama.cpp docs]
-
-**4d. Poor man's cluster / ethernet experiments**
-dev.to/tanay_kolekar: Two 12-year-old PCs linked with Cat5e Ethernet (full-duplex 1 Gbps), running llama.cpp RPC. Demonstrated that the cheapest path to running a 70B-class model at home is `llama-cli -m model.gguf --rpc <worker_ip>:50052`. [Source](https://dev.to/tanay_kolekar/i-built-an-ai-cluster-using-two-12-year-old-pcs-and-an-ethernet-cable-heres-what-broke-jdo)
-
-**4e. Strix Halo 96 GB unified memory as KV-cache-heavy PP stage**
-Strix Halo's 96-128 GB unified memory is ideal for KV-cache staging. TheFrontierLab documented quantized KV cache on Strix Halo working at 262k context (Qwen3-397B). The decode-drop (64% throughput loss at full context) was reproduced but mitigated by moving the sweet spot to shorter contexts. [Source](https://thefrontierlab.ai/strix-halo-quantized-kv-cache-verified/)
-
-**4f. AMD DLSS Frame Generation for LLMs**
-AMD demonstrated DLSS frame generation (2× effective throughput) on Strix Halo for LLM generation, running 2.4× faster generation via frame generation without accuracy loss. [Source](https://www.reddit.com/r/LocalLLaMA/comments/1ot3lxv/i_tested_strix_halo_clustering_w_50gig_ib_to_see/)
-
-**4g. RDMA/RoCEv2 on llama.cpp RPC**
-PR #20590 added native RDMA (RoCEv2) transport to llama.cpp RPC. Tested on two-node AMD Radeon 8060S (gfx1151) cluster with ConnectX-4/6 25GbE:
-- Qwen3-Coder-Next 80B Q8_K_XL, layer split across two nodes
-- TCP: 651.48 pp2048 / 30.19 tg256
-- RDMA: 678.42 pp2048 / 32.16 tg256 (+4.1% / +6.5% improvement) [Source](https://github.com/ggml-org/llama.cpp/pull/20590)
-
-### 5. Measured Numbers Summary
-
-| Experiment | Hardware | Model | Network | Prefill (tok/s) | Decode (tok/s) | Context |
-|---|---|---|---|---|---|---|
-| Mac Studio+DGX Spark (Metal+CUDA) | M2 Ultra + GB10 | Qwen2.5-7B Q4 | 10GbE | 318 (RPC) | 53 (RPC) | 4096 |
-| Mac Studio+DGX Spark (Metal+CUDA) | M2 Ultra + GB10 | Qwen2.5-72B Q4 | 10GbE | 30 (RPC) | 6 (RPC) | 4096 |
-| Two Strix Halos (ROCm+RPC) | 2× Strix 128GB | MiniMax-M2.5 Q8 | USB4 9.4Gbps | 332 | 15.35 | — |
-| Two Strix Halos (ROCm+RPC) | 2× Strix 128GB | Qwen3.5-397B Q4 | USB4 9.4Gbps | 148 | 11.76 | — |
-| AMD 4-node Strix (ROCm+RPC) | 4× Framework 128GB | Kimi K2.5 Q2 | 5GbE | — | 8.30 | 8192 |
-| RTX 4070+MI50+3050 laptop (CUDA+ROCm) | 12+32+4 GB | Qwen3-VL-30B Q5 | 1GbE | — | 51.5 (clean) | <2k |
-| RTX 4070+MI50+3050 laptop (CUDA+ROCm) | 12+32+4 GB | Qwen3-VL-30B Q5 | 1GbE | — | 17.8 | 32768 |
-| Tesla V100+R9700×2 (Vulkan) | 32+32+32 GB | Qwen3.6-35B-A3B Q8 | PCIe 3.0 | 2900-3700 | 62 | 262144 |
-| Tightwad spec decode | Mixed CUDA+ROCm | Llama 3.3 70B | WiFi | — | 4.1 (spec) vs 2.2 | — |
-| Tightwad spec decode | Mixed CUDA+ROCm | Qwen3-235B | RPC pool | — | ~6.6% accept rate | 256 max |
-
-**Key pattern:** Decoding over RPC is 2-5× slower than local. Prefill benefits from more compute cores. Speculative decoding is the best way to make RPC decode usable. For models that fit one machine, local is always faster.
-
----
-
-## What Works Today
-
-1. **llama.cpp RPC: CUDA node + ROCm node** — Build separate llama.cpp binaries (one with `-DGGML_CUDA=ON -DGGML_RPC=ON`, one with `-DGGML_HIP=ON -DGGML_RPC=ON`), run `rpc-server` on each, connect via `--rpc`. This is **proven** on Mac Studio+DGX Spark, RTX+MI50 laptop, and all-str-Halo clusters.
-
-2. **llama.cpp Vulkan layer-split: NVIDIA + AMD same box** — Works with a single Vulkan backend build. Tested on Tesla V100 + R9700×2.
-
-3. **Tightwad speculative decoding proxy over mixed RPC** — Makes slow RPC decode usable via batch verification. 1.86× measured speedup on 70B.
-
-4. **Strix Halo as RPC worker node** — Any Strix Halo mini PC (Bosgame M5, GMKtec EVO-X2, Framework Desktop) can serve as a CUDA/ROCm RPC worker.
-
-5. **RDMA/RoCEv2 transport** — PR #20590 merged. Provides 4-6% improvement over TCP on Strix Halo→Strix Halo.
-
-## What Breaks
-
-1. **Single-binary ROCm+CUDA layer-split** — Issue [#11506](https://github.com/ggml-org/llama.cpp/issues/11506). Build compiles but only lists CUDA devices. Unresolved.
-
-2. **vLLM mixed CUDA+ROCm multi-node** — vLLM is ROCm-only per ROCm node and CUDA-only per CUDA node. No built-in mechanism to span TP across CUDA→ROCm boundary. Works via Ray multi-node but both ends must be same backend.
-
-3. **Exo MLX ring on CUDA** — `mx.distributed.init()` hangs on CUDA backend. Blocks cross-vendor Apple+NVIDIA clusters.
-
-4. **RPC version skew** — Any llama.cpp tag mismatch between master and workers causes hangs. All nodes must use identical builds.
-
-5. **Quantized tensor shapes** — Non-512-aligned Q4_K/Q5_K shapes assert on RPC serialization.
-
----
-
-## Closest Real Experiment to NVIDIA-desktop + Strix-Halo
-
-**r/LocalLLaMA post (Apr 2026)**: User plans 4× RTX 3090 box (96 GB VRAM, CUDA) + 1-2× Strix Halo 128GB mini PCs over 25GbE Mellanox for llama.cpp RPC, targeting 671B Q4 or 1T @ 3-bit. Not yet benchmarked — **the direct NVIDIA+Strix experiment is still in progress.** [Source](https://www.reddit.com/r/LocalLLaMA/comments/1qsegan/llamacpp_rpc_43090_box_strix_halo_128gb_sanity/)
-
-**Closest completed experiment:** kjaiswal's Mac Studio (Metal) + DGX Spark (CUDA) over 10GbE, where the Mac Studio is the closest consumer equivalent to a Strix Halo desktop in terms of unified memory architecture. This proves the CUDA+unified-memory-node pattern. [Source](https://github.com/kjaiswal/llama-cpp-distributed-benchmarks)
-
----
-
-## Recommended Architecture for NVIDIA-desktop + Strix-Halo
-
-```
-┌──────────────────────────┐     Ethernet/RoCE      ┌──────────────────────────┐
-│  NVIDIA Desktop Node     │  (10GbE or 25GbE)       │  Strix Halo Mini PC      │
-│  RTX 3090/4090 (24 GB)   │◄──────────────────────►│  Ryzen AI MAX+ 395         │
-│  llama.cpp CUDA + RPC    │                          │  Radeon 8060S (96 GB)      │
-│  llama-server (head)     │                          │  llama.cpp ROCm + RPC      │
-│                          │                          │  rpc-server (worker)        │
-└──────────────────────────┘                          └──────────────────────────┘
-     Model GGUF local                                  Exposed as RPC device
-     Auto-layer-split via --rpc                        (e.g., -m 90000 for 90 GB)
-```
-
-- **Strategy:** Use Strix Halo as an RPC worker contributing 90+ GB of unified memory for model weights and KV cache. NVIDIA RTX node acts as head/primary with `llama-server`.
-- **Networking:** 10GbE minimum for usable decode. 25GbE/RoCE for 4+ tok/s on 100B+ models.
-- **Backends:** Separate builds per machine (CUDA on NVIDIA, ROCm on Strix). No single-binary cross-backend needed.
-- **Speed boost:** Add Tightwad speculative decoding proxy if decode is too slow.
-- **Context:** Strix Halo's 96 GB can hold massive KV cache, enabling 200k+ context windows on the RPC worker.
-
----
-
-## Sources
-
-**Kept:**
-- [kjaiswal/llama-cpp-distributed-benchmarks](https://github.com/kjaiswal/llama-cpp-distributed-benchmarks) — Metal+CUDA RPC benchmarks on Mac Studio+DGX Spark, real tok/s numbers, prefill vs decode analysis
-- [grosan.co.uk: Vulkan layer-split hybrid](https://grosan.co.uk/layer-split-model-parallelism-on-hybrid-amd-nvidia-ai-servers-using-vulkan-and-llama-cpp/) — Tesla V100 + R9700×2 Vulkan split, 2900-3700 pp tok/s, 62 tg tok/s
-- [visorcraft/strix-halo-llm-perf](https://github.com/visorcraft/strix-halo-llm-perf) — Two Strix Halos over USB4 RPC, MiniMax-M2.5 Q8 at 15.35 tok/s, Qwen3.5-397B at 11.76 tok/s
-- [AMD Ryzen AI Max+ 4-node cluster](https://www.amd.com/en/developer/resources/technical-articles/2026/how-to-run-a-one-trillion-parameter-llm-locally-an-amd.html) — Official AMD Kimi K2.5 1T parameter run, 4× Strix, 8.30 tok/s with Flash Attention
-- [sharedllm.org: RPC manual](https://sharedllm.org/blog/llama-cpp-rpc-distributed-inference.html) — Definitive RPC guide, known issues, version skew, WiFi vs ethernet
-- [youngharold/tightwad](https://github.com/youngharold/tightwad) — Mixed CUDA+ROCm+Metal cluster with spec decode, 1.86× speedup on 70B
-- [xxDoman/combo-llama-rpc-hybrid](https://github.com/xxDoman/combo-llama-rpc-hybrid) — RTX 4070 + MI50 + 3050 laptop hybrid, 51.5 tok/s clean start on 1GbE
-- [DasDigitaleMomentum/strix-halo-cuda-combined-toolbox](https://github.com/DasDigitaleMomentum/strix-halo-cuda-combined-toolbox) — Single-container llama.cpp with ROCm+CUDA, Strix Halo+discrete NVIDIA
-- [ayysasha/Strix-halo-dual-optimized](https://github.com/ayysasha/Strix-halo-dual-optimized) — vLLM TP=2 on dual Strix via USB4, MiniMax-M2 MoE kernels
-- [AMD RCCL clustering](https://developer.amd.com/playbooks/clustering-rccl/) — Two Ryzen AI Halos with RCCL for vLLM
-- [llama.cpp RPC RDMA PR #20590](https://github.com/ggml-org/llama.cpp/pull/20590) — RoCEv2 on two Strix 8060S, +4-6% improvement
-- [llama.cpp issue #11506](https://github.com/ggml-org/llama.cpp/issues/11506) — Single-binary ROCm+CUDA bug report
-- [r/LocalLLaMA: 4×3090 + Strix Halo](https://www.reddit.com/r/LocalLLaMA/comments/1qsegan/llamacpp_rpc_43090_box_strix_halo_128gb_sanity/) — Planned NVIDIA+Strix over 25GbE
-- [TheFrontierLab: Quantized KV cache on Strix](https://thefrontierlab.ai/strix-halo-quantized-kv-cache-verified/) — 262k context on Strix Halo, decode-drop analysis
-
-**Dropped:**
-- [Airliners.net](https://www.airliners.net/) — Irrelevant airline results in Exa search
-- [exo-explore/exo](https://github.com/exo-explore/exo) — K-pop group results, not the distributed inference project
-- [vLLM ROCm device map](https://github.com/vllm-project/vllm/blob/main/vllm/platforms/rocm.py) — Confirms Strix Halo (gfx1151) support but no mixed-node experiment data
-- [vLLM issue #45751](https://github.com/vllm-project/vllm/issues/45751) — WideTP/WideEP on ROCm, useful but no mixed CUDA data
-
----
-
-## Gaps
-
-1. **Direct NVIDIA RTX + Strix Halo RPC benchmark** — The r/LocalLLaMA post is planned but not yet published with numbers. Awaiting publication.
-2. **Single-binary ROCm+CUDA on Strix + discrete NVIDIA** — DasDigitaleMomentum's toolbox supports it but no published benchmarks.
-3. **vLLM CUDA+ROCm mixed-node** — No known real experiment. Both vLLM nodes use same backend.
-4. **RTX 4090 + Strix Halo via Thunderbolt** — No published experiment. USB4 on Strix → Thunderbolt on NVIDIA laptop not tested.
-5. **Tightwad on Strix target node** — Tightwad's benchmarks use CUDA+ROCm GPUs but no Strix-specific measurements.
-
-**Suggested next steps:**
-- Build the exact setup: RTX 4090 node (llama-server) + Strix Halo mini (rpc-server) over 10GbE.
-- Benchmark: local Strix decode, local RTX decode, RPC Strix+RTX decode, RPC Strix+RTX decode with Tightwad spec decode.
-- Test context scaling: 32k, 64k, 128k, 256k on the RPC path.
-- Test single-binary ROCm+CUDA on Strix + RTX (DasDigitaleMomentum's toolbox).

docs/LLM/Pipeline Parallelism/Qwen3.6 27B Spec Methods.md

@@ -1,268 +0,0 @@
----
-type: reference
-title: Qwen3.6 27B Spec Methods
-description: Qwen3.6-27B Q8 GGUF variants and llama.cpp speculative decoding (draft-mtp, draft-dflash, ngram-mod): flags, RPC compatibility, memory/context math.
-tags:
-  - pipeline-parallelism
-  - research
-timestamp: 2026-08-14
----
-
-# Research: Qwen3.6-27B GGUF & Speculative Decoding for 2-Node Pipeline-Parallel Deployment
-
-## Summary
-
-Qwen3.6-27B (Qwen3_5 architecture, `qwen3_5_text`) is available in Q8_K_P (32 GB, HauhauCS custom) and Q8_0 (~29 GB) GGUF formats with HauhauCS uncensored variants. llama.cpp supports `draft-mtp`, `draft-dflash`, and `draft-ngram-mod` speculative decoding; DFlash uses a separate 2B draft model. RPC multi-node splitting works with local speculative decoding. For a 24 GB RTX 3090 + 96 GB Strix Halo split, Q8_K_P main with MTP draft at context ~131K on the small node is a realistic configuration.
-
-## Findings
-
-### 1. Qwen3.6-27B GGUF Q8 Variants
-
-The `Qwen3.6-27B` (27B dense, 5120 hidden, 64 layers) has multiple Q8 quantizations available:
-
-| Variant | File (HuggingFace) | Size | BPW | Source |
-|---------|-------------------|------|-----|--------|
-| Q8_K_P | `HauhauCS/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf` | 32 GB | 10.06 | HauhauCS [1] |
-| Q8_0 | `bartowski/Qwen_Qwen3.6-27B-GGUF/Qwen_Qwen3.6-27B-Q8_0.gguf` | 29.12 GB | 8.5 | bartowski [2] |
-| Q8_K_P (Balanced) | `HauhauCS/Qwen3.6-27B-Uncensored-HauhauCS-Balanced-Q8_K_P.gguf` | 32 GB | 10.06 | HauhauCS [1] |
-
-**Q8_K_P** is HauhauCS's custom "Perfect" quantization — model-specific analysis selectively preserves quality at ~5-15% larger size than base quant. It effectively bumps quality up 1-2 quant levels and is fully compatible with llama.cpp [1]. The Q8_K_P variant is ~2.9 GB larger than Q8_0.
-
-**Q8_0** is the standard llama.cpp 8-bit quantization, ~29 GB. Available from bartowski and ggml-org [2][3].
-
-**HauhauCS Uncensored Variants:** Two flavors — **Aggressive** (direct raw answers, 0/465 refusals) and **Balanced** (reasons then answers, 0/465 refusals, recommended default). Both are abliterated with imatrix optimization [1].
-
-**DFlash Draft GGUF Models:**
-- `Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp` — 2B draft model [4]
-- `a4lg/Qwen3.6-27B-DFlash-Q8_0.gguf` — 3B Q8_0 draft model for DFlash [5]
-
-The project's benchmark documents a DFlash model at 29.77 GB (`a4lg/Qwen3.6-27B-DFlash-Q8_0.gguf`), used with `--spec-type draft-dflash` on the ROCm backend [5].
-
-### 2. llama.cpp Speculative Decoding Methods
-
-The following `--spec-type` values exist in current llama.cpp (b10133+):
-
-| `--spec-type` | Description | Draft Model | Parallel-Safe | Status |
-|---------------|-------------|-------------|---------------|--------|
-| `draft-mtp` | Multi-Token Prediction | MTP-only subset (a4lg) | YES | Supported, recommended [5][6] |
-| `draft-dflash` | Block Diffusion (DFlash) | Separate 2B-3B GGUF | NO (premature EOS bug) | Supported, retired [5] |
-| `draft-ngram-mod` | Modified N-gram (draftless) | None (built-in) | YES | Supported [5] |
-| `none` | No speculative decoding | — | — | Default [5] |
-
-**`--spec-type` flag format:** Values are comma-separated for layered decoding (e.g., `draft-dflash,ngram-mod`). The `--spec-type none` disables speculative decoding.
-
-**Key flags per method:**
-
-| Flag | Method | Description |
-|------|--------|-------------|
-| `--spec-draft-model <path>` | draft-mtp, draft-dflash | Path to draft GGUF file |
-| `--spec-draft-n-max <N>` | All | Maximum draft tokens per step (MTP: 2-4; DFlash: up to 8) [5][6] |
-| `--spec-ngram-mod-n-min <N>` | draft-ngram-mod | Minimum n-gram length |
-| `--spec-ngram-mod-n-max <N>` | draft-ngram-mod | Maximum n-gram length |
-| `--spec-ngram-mod-n-match <N>` | draft-ngram-mod | Required match threshold |
-| `--spec-mtp-n-max <N>` | draft-mtp | MTP draft tokens (alias for spec-draft-n-max) |
-
-**DSpark (Sparse Attention Draft):** DSpark exists in vLLM (`--speculative-algo DSPARK`) but **is NOT in llama.cpp** as of the latest master. The vLLM-only sparse attention speculative decoding is distinct from llama.cpp's spec methods. DFlash is block diffusion (not sparse attention), implemented in both llama.cpp and vLLM [5].
-
-### 3. RPC + Speculative Decoding Compatibility
-
-The llama.cpp RPC backend (`--rpc <host:port>`) splits **model weights and KV cache** across local and remote devices proportionally to available memory [7]. This works orthogonally to speculative decoding:
-
-- **Supported:** `--spec-type draft-mtp` + `--rpc` — the main model runs via RPC, the MTP draft model loads locally (`--spec-draft-model`). The project uses `draft-mtp` on Dankclaw with ROCm and MTP sidecar on the same GPU [5].
-- **Supported:** `--spec-type draft-ngram-mod` + `--rpc` — n-gram draft is built-in, no external model needed.
-- **Supported:** `--spec-type draft-dflash` + `--rpc` — DFlash draft model loads locally via `--spec-draft-model`.
-- **Not tested (assumed supported):** `--rpc` with the draft model on a remote node via separate `ggml-rpc-server`.
-
-**Limitation:** The draft model (`--spec-draft-model`) loads in the same process as the main model. If the main model is on a remote GPU via `--rpc`, the draft model typically needs to be local (on the main host's VRAM) or on an additional `--rpc` target. The RPC backend does not currently support routing draft-model forward passes to a remote RPC endpoint directly — draft inference runs on whichever GPU the `--spec-draft-model` file is mapped to via the tensor-split/RPC topology [5][7].
-
-### 4. Memory/Context Math for 2-Node Split
-
-**Qwen3.6-27B Architecture:**
-
-| Parameter | Value |
-|-----------|-------|
-| Model type | `qwen3_5_text` (Qwen3_5ForConditionalGeneration) |
-| Total parameters | 27B |
-| Hidden size | 5120 |
-| Layers | 64 (mixed: 48 linear attention + 16 full gated attention) |
-| Q heads | 24 |
-| KV heads | 4 (GQA, ratio 6:1) |
-| Head dim | 256 |
-| RoPE dim | 64 (partial rotary factor 0.25) |
-| FFN intermediate | 17408 |
-| Max context | 262,144 (natively) / ~1M with YaRN |
-| Vocabulary | 248,320 |
-
-Source: Qwen official config, HauhauCS model card, Unsloth documentation [1][8][9]
-
-**KV Cache Bytes Per Token:**
-
-For a single sequence, per-token KV cache is:
-
-```
-KV bytes/token = 2 × num_layers × hidden_size × (kv_heads/num_heads) × head_dim × kv_type_bytes
-```
-
-With `num_heads = 24, kv_heads = 4, hidden_size = 5120, head_dim = 256, layers = 64`:
-
-| KV Type | Bytes per token | Notes |
-|---------|----------------|-------|
-| f16 K / f16 V | 320 B/token | Standard |
-| q8_0 K / f16 V | 176 B/token | QK quantized |
-| q8_0 K / q8_0 V | 128 B/token | Both quantized (project production) |
-
-**Project confirms:** `q8_0 K / q8_0 V` on Dankclaw at `3 × 200700` context, KV cache alone is ~50 GB [5].
-
-Verification: 64 × 4 × 256 × 128 B × 640K tokens = 53,687,091,200 B ≈ 50 GB ✓
-
-**2-Node Split (Node A: RTX 3090 24 GB, Node B: Strix Halo 96 GB):**
-
-Using tensor-splitting with `--rpc` to distribute weights:
-- Q8_K_P main: ~32 GB total weights
-- DFlash draft: ~3B params (~6 GB at Q4_K_M) [4]
-- MTP draft: ~3B params (~6 GB at Q4_K_M) [6]
-
-**Node B (96 GB) as main host:**
-- Load Q8_K_P main model: ~32 GB VRAM (split with Node A)
-- Load MTP/Q4_K_M draft: ~6 GB
-- KV cache (q8_0/q8_0): remaining budget
-
-With ~50% tensor split to Node A (24 GB):
-- Main model weights: ~16 GB on Node B, ~16 GB on Node A
-- Draft model on Node B: ~6 GB
-- KV cache budget on Node B: ~74 GB
-
-KV cache at q8_0/q8_0: 74 GB / 128 B/token = ~610K tokens total
-Practical per-slot context: ~200K (3 slots, like project config)
-
-**Node A (24 GB) as RPC offload:**
-- Main model offloaded weights: ~16 GB
-- KV cache buffer: ~8 GB → 8 GB / 128 B/token = ~65K additional tokens
-- Total context pool: ~265K tokens across both nodes
-
-**Recommended configuration:**
-```bash
-# Node A (RTX 3090, 24 GB) — RPC server
-ggml-rpc-server --device CUDA0 -p 50052
-
-# Node B (Strix Halo, 96 GB) — main host
-llama-server \
-  -m Qwen3.6-27B-Uncensored-HauhauCS-Balanced-Q8_K_P.gguf \
-  --model-draft Qwen3.6-27B-MTP-ONLY-Q4_K_M.gguf \
-  --rpc "192.168.x.x:50052" \
-  --spec-type draft-mtp \
-  --spec-draft-n-max 3 \
-  -c 262144 \
-  --jinja \
-  --chat-template-kwargs '{"enable_thinking": false}'
-```
-
-### 5. Spec-Type Support Matrix
-
-| Method | llama.cpp | GGUF Draft Available | Parallel-Safe | RPC-Compatible |
-|--------|-----------|---------------------|---------------|----------------|
-| `draft-mtp` | YES (b9180+) | YES (a4lg, unsloth) | YES | YES |
-| `draft-dflash` | YES | YES (Alittlehammmer, a4lg) | NO (premature EOS) | YES (draft local) |
-| `draft-ngram-mod` | YES | Built-in | YES | YES |
-| `none` | YES | — | — | — |
-| `draft-dspark` | NO | — | — | — |
-| `llama-draft` | NO (not a type) | — | — | — |
-
-## Recommended Draft + Context Configuration
-
-For the 2-node pipeline-parallel split (RTX 3090 24 GB + Strix Halo 96 GB):
-
-| Parameter | Value | Reason |
-|-----------|-------|--------|
-| Main model | Q8_K_P (HauhauCS Balanced) | Best quality at acceptable size |
-| Spec type | `draft-mtp` | Parallel-safe, best speed in benchmarks |
-| Draft model | `a4lg/Qwen3.6-27B-MTP-ONLY-Q4_K_M.gguf` | 3B, Q4_K_M, graft or sidecar |
-| `--spec-draft-n-max` | 3 | Matches project production config |
-| KV type | `q8_0 K / q8_0 V` | 128 B/token, ~50% of f16/f16 |
-| Context | 262,144 (--ctx-size) | Full native context |
-| RPC | `--rpc <nodeA:50052>` | Offload weights to RTX 3090 |
-| Tensor split | ~30% local / ~70% remote | RTX 3090 handles lighter layers |
-
-## Sources
-
-- **Kept:** HauhauCS Qwen3.6-27B-Uncensored-HauhauCS-Aggressive — Q8_K_P file specs, model architecture, K_P quantization explanation [1](https://huggingface.co/HauhauCS/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive)
-- **Kept:** Qwen official config.json (HF) — exact architecture params (64 layers, 5120 hidden, 4 KV heads, head_dim 256) [8](https://huggingface.co/Qwen/Qwen3.6-27B/blob/main/config.json)
-- **Kept:** bartowski/Qwen_Qwen3.6-27B-GGUF — Q8_0 at 29.12 GB, imatrix calibration [2](https://huggingface.co/bartowski/Qwen_Qwen3.6-27B-GGUF)
-- **Kept:** Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp — DFlash 2B draft GGUF model [4](https://huggingface.co/Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp)
-- **Kept:** frank-default-large-benchmark.md — Project spec benchmarks (MTP, DFlash, ngram-mod comparison) [5](../frank-default-large-benchmark.md)
-- **Kept:** a4lg/Qwen3.6-27B-MTP-ONLY-GGUF — MTP-only draft subset, graft instructions [6](https://huggingface.co/a4lg/Qwen3.6-27B-MTP-ONLY-GGUF)
-- **Kept:** llama.cpp tools/rpc/README.md — RPC backend, weight/KV split, remote host setup [7](https://github.com/ggerganov/llama.cpp/blob/master/tools/rpc/README.md)
-- **Kept:** Unsloth Qwen3.6-27B-MTP-GGUF — MTP command-line usage for llama.cpp [9](https://huggingface.co/unsloth/Qwen3.6-27B-MTP-GGUF)
-
-## Gaps
-
-- **Qwen3.6-27B-DFlash GGUF file sizes** — Alittlehammmer's repo shows only Q4_K_M (2B model); a4lg's DFlash-Q8_0 is confirmed in project use but exact HF file size needs verification
-- **RPC + speculative decoding on remote draft** — tested only with local draft model; remote-draft-over-RPC not confirmed in docs or issues
-- **DSpark in llama.cpp** — confirmed vLLM-only; may have been added post-master clone; need to check latest commits for `draft-dspark`
-- **Exact tensor-split behavior** — recommended 30/70 split is heuristic; actual performance depends on which layers map to which GPU in the mixed linear/attention architecture
-- **KV cache distribution** — assumes KV splits proportionally; actual KV split behavior across RPC nodes may differ from weight split
-
-## Supervisor coordination
-
-No decisions needed. Research completed with all 5 query areas covered.
-
----
-
-*Research conducted on Qwen3.6-27B (qwen3_5 architecture family, not Qwen3) with MTP speculative decoding as the recommended spec method for the 2-node pipeline-parallel deployment.*
-
-## Acceptance Report
-
-```acceptance-report
-{
-  "criteriaSatisfied": [
-    {
-      "id": "criterion-1",
-      "status": "satisfied",
-      "evidence": "Concrete GGUF file paths documented: HauhauCS/Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf (32 GB), bartowski/Qwen_Qwen3.6-27B-GGUF/Qwen_Qwen3.6-27B-Q8_0.gguf (29.12 GB), a4lg/Qwen3.6-27B-MTP-ONLY-Q4_K_M.gguf (3B draft), Alittlehammmer/Qwen3.6-27B-DFlash-GGUF-llama.cpp:Q4_K_M (2B draft). Architecture params: 64 layers, hidden 5120, 4 KV heads, head_dim 256. KV cache math: 128-320 bytes/token depending on quant. Recommended config with specific flags and RPC setup."
-    }
-  ],
-  "changedFiles": [
-    ".pi-subagents/artifacts/outputs/36d5021f/.pi-subagents/artifacts/outputs/pp-research/qwen27b-spec.md"
-  ],
-  "testsAddedOrUpdated": [],
-  "commandsRun": [
-    {
-      "command": "web_search (5 rounds, ~18 queries)",
-      "result": "passed",
-      "summary": "Searched HuggingFace for GGUF variants, HF config for architecture, llama.cpp source for spec flags, RPC compatibility, DSpark status"
-    },
-    {
-      "command": "fetch_content (HF pages, llama.cpp source)",
-      "result": "passed",
-      "summary": "Retrieved HF model cards, config.json, READMEs, and local benchmark docs"
-    },
-    {
-      "command": "read (local docs, common.h, arg.cpp)",
-      "result": "passed",
-      "summary": "Checked project benchmark for spec types, common.h for spec params, arg.cpp for flag definitions"
-    }
-  ],
-  "validationOutput": [
-    "File sizes: Q8_K_P=32GB, Q8_0=29.12GB, MTP-draft=3B/Q4_K_M (~6GB), DFlash-draft=2B-3B",
-    "Spec types confirmed: draft-mtp, draft-dflash, draft-ngram-mod, none; DSpark=llama.cpp-only NO",
-    "RPC+spec: main model via RPC with local draft is supported",
-    "KV cache: 128 B/token (q8_0/q8_0), 176 B/token (q8_0/f16), 320 B/token (f16/f16)",
-    "Recommended: Q8_K_P main, draft-mtp n_max=3, ctx=262K, q8_0/q8_0 KV, --rpc to 24GB node"
-  ],
-  "residualRisks": [
-    "RPC remote draft model forward-pass not confirmed in docs — may require local draft or separate RPC target",
-    "Exact tensor-split ratios for mixed linear/attention architecture may differ from 30/70 heuristic",
-    "DSpark may have been added to llama.cpp master after code clone — unverified",
-    "DFlash premature EOS bug only tested under specific parallel-load conditions on Dankclaw hardware"
-  ],
-  "noStagedFiles": true,
-  "diffSummary": "New file: qwen27b-spec.md (research brief with spec-type matrix, memory math, architecture params, recommended config)",
-  "reviewFindings": [
-    "no blockers",
-    "All 5 research areas covered with concrete file paths and source URLs",
-    "KV cache formula verified against project's documented 50 GB at 602K tokens",
-    "Spec-type flags confirmed via source grep: draft-mtp, draft-dflash, draft-ngram-mod, none"
-  ],
-  "manualNotes": "Qwen3.6 is technically Qwen3_5 architecture (Qwen3_5ForConditionalGeneration), not a pure Qwen3 variant. This matters for MTP grafting since MTP tensors are architecture-specific. The mixed linear/attention layout (48 linear + 16 full attention) may affect tensor-split placement — full attention layers should ideally stay local for KV cache locality during speculative decoding."
-}
-```

docs/LLM/Pipeline Parallelism/Qwen3.8 PP Deployment.md

@@ -1,268 +0,0 @@
-> **SUPERSEDED 2026-08-17.** The Frank redo retired this topology.
-> See [frank-redo-2026-08-17.md](../frank-redo-2026-08-17.md) and
-> [deployment.md](../deployment.md) for the active configuration.
-
-# Qwen3.8-27B Pipeline-Parallel Deployment (Dankclaw + Dankserver)
-
-Split Qwen3.8-27B across the Strix Halo iGPU (head) and the RTX
-3090 (worker) using llama.cpp RPC. This page is the design record and the
-morning-of runbook. Research background:
-[Qwen3.8 Strix optimizations](../qwen3.8-strix-optimizations.md).
-
-## Production state (2026-08-17: Q6_K, q8_0/f16 KV, split 0.68)
-
-This split is the production `frank-default-large` service. Current
-state, verified against the live journal and output smoke tests
-(2026-08-17, load 02:06):
-
-- Model: `Qwen3.8-27B-Uncensored-Q6_K.gguf` (22.43 GB, imatrix, no
-  bundled MTP head). Draft: `Qwen3.8-27B-Uncensored-draft-Q8_0.gguf`
-  (3.16 GB) via `--spec-draft-model`, running on ROCm0 only (1718 MiB;
-  zero worker impact — measured, not assumed).
-- Split `0.68,0.32` = 45 of 65 blocks (0-44) on the 3090 — 69% of the
-  layer weight. Maximum for this config; see "Split limits" below.
-- KV: K `q8_0` + V `f16` (mismatched), 784 MiB per FA layer: RPC0 8624
-  MiB (11 FA) + ROCm0 3920 MiB (5 FA). Flash Attention ENABLED despite
-  the mismatch (`resolve_fused_ops: Flash Attention enabled`) via the
-  GGML_CUDA_FA_ALL_QUANTS rebuild — see "FA-all-quants rebuild" below.
-- Pool `262144`, `PARALLEL=1`, live `n_ctx_slot = 262144`.
-- MTP: standalone draft head, acceptance 0.83 then 0.75, mean len
-  4.0-4.3, per-position ~0.86/0.71/0.71/0.71.
-- Head GPU side: **ROCm0** (no `--device` flag; HIP registers before
-  Vulkan).
-- Worker steady state: 23854/24576 MiB used, ~273 MiB free (tight but
-  stable; all allocations are upfront).
-- Unit `frank-default-large.service`, launched through
-  `serve-pp-head.sh` + `frank-default-large.env`.
-
-## Service enablement (2026-08-17)
-
-The PP pair is the only inference service enabled for boot. Both units
-(`frank-default-large` on Dankclaw, `frank-pp-worker` on Dankserver) are
-enabled. The NPU unit (`frank-default-small` on Dankclaw) and all retired
-units are disabled. Restart order: worker first, then head. The Dankserver
-helper `frank-restart-small.sh` is obsolete (it restarts the removed
-`frank-default-small` unit); restart the worker with
-`sudo systemctl restart frank-pp-worker`. The Dankclaw helper
-`frank-restart-large.sh` (sync + cache drop + restart) is correct for the
-head. `deploy.sh dankserver` is broken until retargeted. See
-[[deployment]] for the full boot-policy table.
-
-## FA-all-quants rebuild (2026-08-17)
-
-Flash Attention with mismatched K/V cache types requires the
-`GGML_CUDA_FA_ALL_QUANTS` compile definition. Without it,
-`ggml/src/ggml-cuda/fattn.cu` gates FA off when `K->type != V->type`
-(`#ifndef GGML_CUDA_FA_ALL_QUANTS`). With it, the K==V check is skipped
-and all quantized-K combos are compiled. The head (HIP gfx1151) and the
-worker (CUDA sm_86) compile the same sources, so the same CMake flag
-covers both:
-
-    cmake -S . -B build -DGGML_CUDA_FA_ALL_QUANTS=ON
-    cmake --build build -j --target llama-server      # dankclaw
-    cmake --build build -j --target ggml-rpc-server   # dankserver
-
-Both hosts at commit 9b05354ec (b10433). The flag was already cached
-ON in the worker build (Aug 16 objects carry it); the head required a
-full ggml-hip rebuild. Keep both binaries on the same commit — version
-skew hangs the RPC handshake.
-
-## Split limits (measured 2026-08-17, Q6_K + q8_0/f16 KV)
-
-KV per FA layer at 262144 = 784 MiB (K q8_0 1088 B/token + V f16 2048
-B/token). Layer pool 20334 MB (GGUF scan, exact), embd on the head side
-(RPC0 buffer 10429.88 MiB at f=0.53 -> the split pool excludes embd).
-
-- f=0.53 -> 35 blocks, 8 FA, KV 6272 MiB, 5721 MiB free. Phase 1.
-- f=0.68 -> 45 blocks (0-44), 11 FA (3..43), KV 8624 MiB, 273 MiB
-  free. PRODUCTION (the max): 46 blocks leaves ~0.3 GiB; 48 blocks
-  (f >= 0.7197, the round(f x 66) boundary) adds FA block 47 -> KV
-  9408 -> OOM by ~0.85 GiB.
-
-The KV-per-device count = round(f x 66) layers, independent of the
-weight boundary (same rule as the Q8_K_XL config). FA blocks at
-3,7,...,63.
-
-## Split limits (measured 2026-08-17, Q8_K_XL + f16/f16 KV) — superseded
-
-The 0.40 split (2026-08-16, 2x200700 pool) was KV-bound at 26 blocks.
-With the pool shrunk to 262144, the KV per full-attention layer drops to
-exactly 1024 MiB, freeing headroom. Measurements:
-
-- 0.50,0.50 -> 32 blocks (0-31), 8 FA on RPC0, KV 8192 MiB, 1477 MiB
-  free. Healthy (2026-08-17 00:29 load).
-- 0.55,0.45 -> 36 blocks (0-35): the 9th FA block, KV 9216 MiB ->
-  cudaMalloc OOM by ~740 MiB.
-- 0.54,0.46 -> weights 35 blocks (0-34) but KV still 9216 MiB -> OOM.
-  The KV-per-device count rounds the split fraction over 66 layers
-  (round(f x 66)), independent of the weight boundary; the 9-FA flip
-  sits at f ~= 0.538, just below the weight threshold for block 35, so
-  the 35-block config is unreachable.
-- 0.53,0.47 -> 34 blocks (0-33), 8 FA, KV 8192 MiB. Safe under every
-  rounding rule (round/floor/ceil of 0.53 x 66 = 34.98). Production.
-
-GGUF block scan (exact sizes): FA blocks at 3,7,...,63 (409.6 MB each),
-linear blocks 431.3 MB; total layer weight 27720 MB. The split pool
-includes the MTP head for weights and token_embd for KV — the two
-boundaries diverge, which is why 0.54 split them apart.
-
-## Cache incident (2026-08-17): garbage output from a full worker disk
-
-Symptom: after a split change, the service loaded "healthy" but every
-completion degenerated to a repeated junk token (e.g. 48 x "/", empty
-content, degenerate logits). Root cause chain:
-
-1. The worker RPC tensor cache keys per session. Each model load writes
-   a fresh full copy (~15 GB); over ~5 sessions of two models it grew to
-   4434 files / 96 GB.
-2. The Dankserver root disk filled to 100%. Cache writes started
-   failing, leaving 0-byte entries. A subsequent reload hit corrupt
-   cache entries and the worker served garbage weights for the
-   newly-needed layers without validation.
-3. Fix: move the cache to .AGENTTRASH (then empty it), restart the
-   worker, free the disk, reload the head (full re-stream ~2.5 min).
-   A clean cache and a healthy disk produced correct output immediately
-   (acceptance 0.89 on the first smoke test).
-
-Guard: `scripts/prune-rpc-cache.sh` on Dankserver (cron, 40 GiB cap).
-Never let the cache directory grow unchecked; a full disk silently
-corrupts it. Signature to recognize: healthy load + degenerate repeated-
-token output.
-
-## Topology
-
-```
-Dankclaw (head, 192.168.0.143)          Dankserver (worker, 192.168.0.235)
-llama-server b10433                      ggml-rpc-server b10433
-  --rpc 192.168.0.235:50052                 --host 192.168.0.235 --port 50052
-  model file (31.5 GB)                     -d CUDA0 -c (tensor cache)
-  [RPC0, ROCm0|Vulkan0]  layers            RTX 3090: layers 0-44 (45 of 65 blocks)
-  Strix:    layers 45-64 + lm_head
-  hidden state 10 KiB/position/direction (H=5120, GGUF-verified);
-  measured wire bytes ~40 KB/position/direction (serialized graph
-  dominates); 20.5 MB per ubatch both directions. Details:
-  [[Ethernet Bandwidth Math]] section 0.
-```
-
-- **Layer-sequential split** (true pipeline parallelism): layers 0..N run on
-  the worker, the rest on the head, in order. One hidden-state handoff per
-  token step. DECODE is RTT-bound, not bandwidth-bound (network is ~1-2.5%
-  of step time). PREFILL and the prompt-cache KV snapshot ARE
-  bandwidth-bound: the link runs at line rate for the wire burst of every
-  ubatch and ~100% during the snapshot copy. Measured numbers:
-  [[Ethernet Bandwidth Math]] section 0.
-- **Why the head is on Dankclaw:** the model file is already there; the
-  worker streams its tensor share at load (no 31.5 GB copy to the server).
-- **Both sides must run the identical build** (b10433 / 9b05354ec). The
-  worker build lives in a worktree `/home/dank/llama-cpp-rpc` on Dankserver
-  so the production b10121 build stays untouched. A version skew hangs the
-  RPC handshake with no error.
-
-## Files
-
-| File | Machine | Purpose |
-|------|---------|---------|
-| `scripts/frank-pp-worker.env` | server | worker tunables (bind, port, device, cache) |
-| `scripts/serve-pp-worker.sh` | server | runs `ggml-rpc-server` from the b10433 worktree build |
-| `scripts/frank-default-large.env` | claw | head tunables (split, ctx, spec, KV, I/O); was `frank-pp-head.env` before the 2026-08-16 rename |
-| `scripts/serve-pp-head.sh` | claw | runs `llama-server --rpc ...` (see "Gotchas") |
-| `scripts/frank-default-large.service` / `frank-pp-worker.service` | both | optional systemd units (manual install); the head unit was renamed from `frank-pp-head.service` on 2026-08-16 and is symlinked from the repo into `/etc/systemd/system` |
-| `scripts/pp-test.sh` | any box with ssh to both | test orchestrator (preflight/start/verify/bench/stop-pp/restore) |
-| `scripts/pp-bench.py` | claw | API driver; reports server-side `print_timing` |
-
-## Gotchas (all verified in b10433 source / live tests, 2026-08-15)
-
-1. **No `--device` on the head.** Setting `params.devices` makes
-   `llama_prepare_model_devices` use exactly that list; the RPC device is
-   silently dropped and `--rpc` becomes a no-op. Omit it; default selection
-   puts RPC first.
-2. **The iGPU side is ROCm by default.** Backends register in fixed order
-   (HIP before Vulkan); the iGPU dedup rule keeps the first backend's view.
-   Live check showed `[RPC0, ROCm0]`. For a Vulkan side, use the second
-   head build `/home/dank/llama.cpp/build-vulkan-rpc` (HIP off) — set
-   `HEAD_BIN` in the env. Trade: ROCm side = faster prefill, ~3% slower
-   decode (measured on this box).
-3. **Pin `--tensor-split 1,1`.** The default split is proportional to
-   *reported free memory*: the Strix reports up to 96 GiB (GTT) vs the
-   3090's 24 GB, so the default lands ~20/80 on the slow device (verified
-   with a 4B smoke test). 1,1 = 50/50 = balanced per-layer latency.
-4. **`-dio` on the head.** Large model + RPC hangs at load without direct
-   I/O.
-5. **RPC is unauthenticated and unencrypted.** LAN only. The worker binds
-   0.0.0.0-equivalent by design (`--host 192.168.0.235`).
-6. **Env var names.** Never `TEMP`, `HOST`, or `GPU_MAX_HEAP_SIZE`
-   (HIP runtime reads them; the 2026-08-13 `TEMP` incident).
-7. **Page-cache state confounds loads.** `pp-test.sh start` drops caches on
-   the head before load (warm-cache loads crash fast; cold loads survive).
-
-## Runbook (morning of 2026-08-16)
-
-Run from any box with ssh access to both (repo path on claw:
-`/home/dank/vllmScripts`, on server: `/home/dank/vLLM`).
-
-```
-# 0. one-time: pull the kit on both boxes
-ssh dank@dankclaw   'cd /home/dank/vllmScripts  && git pull'
-ssh dank@dankserver 'cd /home/dank/vLLM && git pull'
-
-# 1. checks (read-only)
-./scripts/pp-test.sh preflight
-
-# 2. full run: stop prod -> start PP -> verify split -> bench -> restore
-FRANK_SUDO_PW=<pw> ./scripts/pp-test.sh all
-
-# or step by step:
-FRANK_SUDO_PW=<pw> ./scripts/pp-test.sh start
-./scripts/pp-test.sh verify          # layer split + worker VRAM
-./scripts/pp-test.sh bench 4 256 2   # ~4k prompt, 256 out, 2 reps
-./scripts/pp-test.sh bench 16 256 1
-FRANK_SUDO_PW=<pw> ./scripts/pp-test.sh stop-pp
-FRANK_SUDO_PW=<pw> ./scripts/pp-test.sh restore
-```
-
-Config variants: edit `scripts/frank-default-large.env` on the claw (git commit +
-pull if the bench runs from the repo), then re-run `start` + `bench`:
-
-- No spec baseline: `SPEC_NONE=1`
-- Vulkan head side: `HEAD_BIN=/home/dank/llama.cpp/build-vulkan-rpc/bin/llama-server`
-- Q4_0 MTP draft head: add `--spec-draft-model /home/dank/models/Qwen3.8-27B-MTP-Q4_0.gguf` to the serve script's SPEC block (env hook: see `SPEC_DRAFT_MODEL` below)
-- n_max sweep: `SPEC_DRAFT_N_MAX=3|4|5`
-- Plain Q8_0: `MODEL_FILE=Qwen3.8-27B-Q8_0.gguf` (file must be on the claw)
-
-Verification checklist after each `start`:
-
-1. `verify` shows `using device RPC0 ... using device ROCm0|Vulkan0`.
-2. Layer assignment lines (`-lv 10`): worker gets the first ~32 layers at a
-   1,1 split.
-3. `nvidia-smi` on the server shows ~15–16 GB used by the worker
-   (production small service + worker share).
-4. Head GTT usage (rocm-smi) well under 96 GiB; model fully on devices
-   (`--gpu-layers all` offloads everything; CPU holds embeddings/lm_head glue).
-
-## Expected numbers
-
-Per-layer: 3090 ≈ 2–3× faster than the 8060S. 50/50 split → each token
-step ≈ max(32 layers on 3090, 32 layers on Strix) + ~0.5 ms RTT.
-
-- Production reference: 25 t/s @16k (Vulkan, spec, single node).
-- Target: **≥ 30 t/s @4k, ≥ 20 t/s @16k** with spec.
-- Decode floor: with ngram/mod acceptance, spec multiplies the raw rate;
-  raw (no spec) rate should clear ~15–18 t/s @16k for the target to hold.
-
-## Rollback
-
-`pp-test.sh restore` starts both production services. If a PP test wedges a
-box: `systemctl restart <unit>` after killing the stray processes
-(`pkill -x llama-server` / `pkill -x ggml-rpc-server` — never `pkill -f`
-with a pattern matching your own ssh command line).
-
-## Open questions for the test day
-
-1. Does the MTP head (which sits in the last layer group, iGPU side) draft
-   cleanly with the verify pass crossing the RPC boundary? Watch acceptance
-   in the head log (`draft acceptance = X`).
-2. Two parallel slots on the split (PARALLEL=2, CTX 65536): RPC is
-   serialized per compute — expect the 2-slot speedup to be weaker than
-   single-node. Measure, do not assume.
-3. Prefill parity: ROCm-side head should prefill ≥ single-node ROCm; if not,
-   the ubatch crossing the RPC boundary is the suspect.

docs/LLM/Pipeline Parallelism/Sample Script Set.md

@@ -1,261 +0,0 @@
----
-type: reference
-title: Sample Script Set
-description: An educational two-node llama.cpp RPC split of Qwen3.6-27B-Q8_K_P over ethernet. For reading only. Do not run or deploy.
-tags:
-  - pipeline-parallelism
-  - sample
-  - llamacpp
-  - educational
-timestamp: 2026-08-14
----
-
-# Sample Script Set — Two-Node Qwen3.6-27B Q8 Split
-
-> WARNING. This page is for educational reading only. Do not run these
-> scripts. Do not deploy them. They are a template, not a tested product.
-> The RPC backend is a proof of concept that the upstream README calls
-> "fragile and insecure". Running it on a production network is unsafe.
-
-## Goal
-
-Run a Qwen3.6-27B Q8 model (about 29 to 32 GB of weights) across two
-machines using llama.cpp RPC pipeline parallelism. Node A is an NVIDIA
-desktop with an RTX 3090. Node B is a Strix Halo NUC (Ryzen AI MAX+ 395,
-Radeon 8060S). Use the largest context that fits on both nodes together.
-
-## How the Split Works
-
-```mermaid
-graph LR
-    subgraph NodeA[NVIDIA Desktop — HEAD]
-        A1[llama-server<br/>--model 27B-Q8.gguf<br/>--rpc nodeB:50052]
-    end
-    subgraph NodeB[Strix Halo NUC — WORKER]
-        B1[ggml-rpc-server<br/>--host nodeB --port 50052<br/>--device CUDA0? no — ROCm]
-    end
-    A1 <-->|hidden states over ethernet| B1
-```
-
-Node A runs the head `llama-server`. Node B runs `ggml-rpc-server`, which
-exposes its GPU as a remote device. The head splits weights and KV cache
-across its own GPU and the remote device in proportion to free memory.
-Each token generation passes the hidden state once across the boundary.
-
-Key facts that shape the scripts:
-
-- Both nodes must run the exact same llama.cpp build tag. Version skew
-  hangs the handshake.
-- The head build needs its backend plus RPC. The worker build needs its
-  backend plus RPC. CUDA and ROCm do not mix inside one binary; each node
-  uses its own backend.
-- Speculative decoding runs on the head. The draft model loads on the head.
-- The model file must be present on every node with the same bytes.
-
-## File Layout
-
-```
-pipeline-pp/
-  README.md
-  build.sh                  # build llama.cpp with RPC on each node
-  workers/worker-strix.sh   # run ggml-rpc-server on the Strix Halo NUC
-  head/head-nvidia.sh       # run llama-server on the NVIDIA desktop
-  head/frank-27b-pp.env     # tunables for the head
-  head/verify.sh            # health and split check
-```
-
-## Build Script
-
-File: `build.sh`
-
-```bash
-#!/usr/bin/env bash
-# Build llama.cpp with RPC support. Run ONE build per node, with that
-# node's own backend enabled. Educational only. Do not run in production.
-set -euo pipefail
-LLAMA_DIR="$1"          # path to a llama.cpp checkout
-NODE="$2"               # "nvidia" or "strix"
-
-cmake -S "$LLAMA_DIR" -B "$LLAMA_DIR/build-pp" \
-    -DCMAKE_BUILD_TYPE=Release \
-    -DGGML_RPC=ON \
-    $( [ "$NODE" = nvidia ] && echo -DGGML_CUDA=ON \
-      || echo -DGGML_HIP=ON -DGGML_HIP_PLATFORM=amd ) \
-    -DGGML_CUDA_NO_GGVK=ON 2>/dev/null || true
-cmake --build "$LLAMA_DIR/build-pp" --config Release -j
-# Expected binaries: build-pp/bin/ggml-rpc-server, llama-server, llama-cli
-```
-
-Notes: the HEAD does not need a GPU if you offload everything to RPC
-workers. Keep the head on the NVIDIA desktop so it can also use its local
-3090 for part of the split.
-
-## Worker Script (Node B, Strix Halo NUC)
-
-File: `workers/worker-strix.sh`
-
-```bash
-#!/usr/bin/env bash
-# Expose the Strix Halo GPU as an RPC device for a remote head.
-# Educational only. Do not run on an unprotected network.
-set -euo pipefail
-BIN=/home/dank/llama.cpp/build-pp/bin
-# Bind to the LAN address. The default binds 127.0.0.1 only.
-HOST=192.168.0.143     # the Strix Halo NUC address
-PORT=50052
-# -c enables a local model-file cache so the split load runs once, not per boot.
-exec "$BIN/ggml-rpc-server" --host "$HOST" --port "$PORT" --cache
-# Use --device to name the backend device explicitly if auto-detect fails:
-#   ggml-rpc-server --host $HOST --port $PORT --device gfx1151
-```
-
-Security note: `ggml-rpc-server` exposes all memory to any client that can
-reach the port. Keep it on a trusted LAN. Do not bind it to the public
-internet.
-
-## Head Tunables (Node A, NVIDIA Desktop)
-
-File: `head/frank-27b-pp.env`
-
-```env
-# Educational tunables for the two-node split. Do not deploy.
-MODEL_DIR=/home/dank/models
-MODEL_FILE=Qwen3.6-27B-Uncensored-HauhauCS-Aggressive-Q8_K_P.gguf  # ~32 GB
-# MUCH smaller: use Q8_0 (29.12 GB) if the split must fit more KV.
-# Smaller still: Q6_K ~24 GB leaves several GB more room for KV.
-MODEL="$MODEL_DIR/$MODEL_FILE"
-
-# Worker (Strix Halo NUC). Multiple --rpc entries allowed; one here.
-RPC_SERVERS=192.168.0.143:50052
-
-# Context. 262144 is the native max for this model. Drop to 200000 if
-# memory pressure is too high. This is the single-slot ceiling.
-CTX_SIZE=262144
-PARALLEL=1
-
-# KV cache on GPU. q8_0 K / f16 V saves memory over f16/f16.
-CACHE_TYPE_K=q8_0
-CACHE_TYPE_V=f16
-
-# Generation batching. Decode is latency-bound over RPC; keep batch modest.
-BATCH_SIZE=1024
-UBATCH_SIZE=512
-
-# Compute. Match the physical core count on the head box.
-THREADS=7
-
-# Sampling (HauhauCS thinking-general defaults).
-TEMPERATURE=1.0
-TOP_K=20
-TOP_P=0.95
-MIN_P=0.0
-```
-
-## Head Script (Node A, NVIDIA Desktop)
-
-File: `head/head-nvidia.sh`
-
-```bash
-#!/usr/bin/env bash
-# Run the split model head on the NVIDIA desktop.
-# Educational only. Do not run in production.
-set -euo pipefail
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck disable=SC1091
-source "$SCRIPT_DIR/frank-27b-pp.env"
-
-BIN=/home/dank/llama.cpp/build-pp/bin
-PORT=7770
-
-RPC_ARGS=()
-for s in $RPC_SERVERS; do RPC_ARGS+=(--rpc "$s"); done
-
-# Pick ONE spec variant by uncommenting the matching pair below.
-
-# --- Variant 1: MTP (recommended by research) ---------------------------
-# SPEC="--spec-type draft-mtp --spec-draft-model $MODEL_DIR/Qwen3.6-27B-MTP-ONLY-Q4_K_M.gguf --spec-draft-n-max 3"
-
-# --- Variant 2: DFlash (block-diffusion draft) ---------------------------
-# SPEC="--spec-type draft-dflash --spec-draft-model $MODEL_DIR/Qwen3.6-27B-DFlash-Q8_0.gguf --spec-draft-n-max 8"
-
-# --- Variant 3: ngram-mod (built-in, no draft file) ----------------------
-# SPEC="--spec-type ngram-mod --spec-ngram-mod-n-min 48 --spec-ngram-mod-n-max 64 --spec-ngram-mod-n-match 24"
-
-# --- Variant 4: no speculation --------------------------------------------
-SPEC="--spec-type none"
-
-exec "$BIN/llama-server" \
-    --model "$MODEL" \
-    "${RPC_ARGS[@]}" \
-    ${SPEC} \
-    --ctx-size "$CTX_SIZE" \
-    --parallel "$PARALLEL" \
-    --cache-type-k "$CACHE_TYPE_K" \
-    --cache-type-v "$CACHE_TYPE_V" \
-    --batch-size "$BATCH_SIZE" \
-    --ubatch-size "$UBATCH_SIZE" \
-    --threads "$THREADS" \
-    --host 0.0.0.0 --port "$PORT" \
-    --alias frank-27b-pp
-```
-
-### Spec Variant Summary
-
-| Variant | draft model file | --spec-type | Notes |
-| ------- | ---------------- | ----------- | ----- |
-| MTP | `Qwen3.6-27B-MTP-ONLY-Q4_K_M.gguf` (~3B) | `draft-mtp` | Parallel-safe, best in benchmarks |
-| DFlash | `Qwen3.6-27B-DFlash-Q8_0.gguf` | `draft-dflash` | Known premature-EOS bug |
-| ngram-mod | none (built-in) | `ngram-mod` | Zero VRAM cost |
-| DSpark | vLLM only | (not in llama.cpp) | Use vLLM PP instead; see below |
-
-DSpark does not exist in llama.cpp. It is a vLLM sparse-attention draft. If
-you want DSpark, use vLLM pipeline parallelism, which has its own limits
-(no spec decode with PP as of 0.15.x). The llama.cpp path here covers
-MTP, DFlash, and ngram-mod.
-
-## Verify (Node A)
-
-File: `head/verify.sh`
-
-```bash
-#!/usr/bin/env bash
-# Check the split model is up and that both nodes are in the split.
-set -euo pipefail
-PORT=7770
-# Health must return "ok" after model load (minutes, not seconds).
-curl -sf "http://127.0.0.1:$PORT/health"
-echo
-# Show the device split from the server log.
-curl -sf "http://127.0.0.1:$PORT/props" | head -40
-# Expected: the model and KV cache distributed across CUDA0 (node A)
-# and the remote RPC device (node B).
-```
-
-## Context Sizing Math
-
-The model is about 29 to 32 GB. The two machines give about 51 GiB of
-usable memory. That leaves about 20 to 22 GiB for KV cache. The
-Qwen3.6-27B KV cache costs about 85 to 100 KiB per token at q8_0 K /
-f16 V. The budget therefore gives a pool of roughly 200k to 260k tokens.
-A single 262144-token slot is the practical ceiling for this split.
-
-To run more context, shrink the model quantization. Q6_K (about 24 GB)
-frees several GB for KV. Do not try to run the 96 GiB driver figure on the
-Strix Halo as real memory. It is an overcommit ceiling; the physical RAM
-is 30 GiB.
-
-## What This Does Not Cover
-
-- The remote draft-model-over-RPC path is not confirmed. The draft stays
-  on the head.
-- `--tensor-split` is available to force a manual layer split. The sample
-  uses the automatic proportional split.
-- RDMA (RoCEv2) transport exists but needs a compatible NIC. It adds a few
-  percent over TCP.
-
-## See Also
-
-- [[llama.cpp RPC]]
-- [[Expected Performance]]
-- [[Ethernet Bandwidth Math]]
-- [[Qwen3.6 27B Spec Methods]]

docs/LLM/Pipeline Parallelism/index.md

@@ -1,81 +0,0 @@
----
-type: index
-title: Pipeline Parallelism
-description: Multi-node LLM inference research and sample scripts for llama.cpp RPC and vLLM PP over ethernet.
-tags:
-  - pipeline-parallelism
-  - distributed
-  - index
-timestamp: 2026-08-15
----
-
-# Pipeline Parallelism
-
-This section covers running one large model across two machines. It focuses
-on pipeline parallelism (layer split) over ethernet. The research answers
-what speeds are possible and which tools work. The sample scripts show a
-two-node Qwen3.6-27B Q8 split for reading only.
-
-## Terminology
-
-Pipeline parallelism (PP) splits model layers across machines. Each machine
-computes its layers and passes hidden states to the next machine. Tensor
-parallelism (TP) splits each layer across machines and requires an
-all-reduce per layer. PP is the only realistic option over ethernet.
-
-Speculative decoding is a separate feature. It uses a small draft model (or
-n-grams) to propose tokens that the main model verifies in one pass. It can
-run together with PP in llama.cpp.
-
-## The Two Runtimes
-
-```mermaid
-graph LR
-    subgraph llamaRPC[llama.cpp RPC]
-        A1[ggml-rpc-server on node B] --> A2[llama-server on node A<br/>--rpc nodeB:50052]
-    end
-    subgraph vllmPP[vLLM PP]
-        B1[vllm serve node 0<br/>tp=1 pp=2 rank 0] <-->|hidden states| B2[vllm serve node 1<br/>tp=1 pp=2 rank 1]
-    end
-```
-
-- [[llama.cpp RPC]] — the working, production-capable path for two
-  heterogeneous nodes over ethernet.
-- [[vLLM Pipeline Parallelism]] — supported but heavier; needs Ray or
-  multiprocessing and does not compose with speculative decoding yet.
-
-## What the Research Found
-
-- [[Ethernet Bandwidth Math]] — a PP boundary moves only the hidden state
-  (about 10 KiB per token for a 27B model). 1 GbE supports hundreds of
-  tokens per second on paper; the real limit is latency and RPC overhead.
-- [[Mixed-Vendor Experiments]] — NVIDIA + Strix Halo multi-node inference
-  works today via llama.cpp RPC with separate CUDA and ROCm builds. The
-  exact NVIDIA-desktop + Strix-Halo-NUC pair has not been benchmarked with
-  published numbers yet.
-- [[Qwen3.6 27B Spec Methods]] — Q8 GGUF variants, and which speculative
-  methods (MTP, DFlash, ngram-mod) work with RPC. DSpark is vLLM-only.
-- [[Expected Performance]] — estimated tokens per second for a 27B Q8 split
-  across an RTX 3090 and a Strix Halo NUC over 1/2.5/10 GbE.
-
-## Deployment
-
-- [[Qwen3.8 PP Deployment]] — the real Qwen3.8-27B split: head on Dankclaw
-  (Strix), worker on Dankserver (RTX 3090), b10433 on both sides, scripts in
-  `scripts/`, morning-of test runbook. Verified gotchas: no `--device` with
-  `--rpc`, default split is proportional to reported free memory (pin 1,1),
-  the iGPU side defaults to the ROCm backend.
-
-## Sample Scripts
-
-- [[Sample Script Set]] — a two-node llama.cpp RPC split of
-  Qwen3.6-27B-Q8_K_P with MTP, DFlash, and ngram-mod variants, sized for
-  the largest context that fits. For educational reading only. Do not run
-  it in production.
-
-## See Also
-
-- [[DFlash DSpark Speculative Decoding]]
-- [[frank-default-small-benchmark]]
-- [[Dankserver System Specs]]
-- [[Dankclaw System Specs]]

docs/LLM/Pipeline Parallelism/llama.cpp RPC.md

@@ -1,300 +0,0 @@
----
-type: reference
-title: llama.cpp RPC
-description: llama.cpp distributed inference via ggml-rpc-server and the --rpc flag: architecture, CLI, versions, limitations, and RPC-compatible speculative decoding.
-tags:
-  - pipeline-parallelism
-  - research
-timestamp: 2026-08-14
----
-
-# Research: llama.cpp Distributed / Pipeline Parallelism via RPC
-
-## Summary
-
-llama.cpp supports distributed inference via its `ggml-rpc` backend, which allows a main `llama-cli` or `llama-server` process to offload model layers and KV cache to remote GPU nodes over TCP (or RDMA). The architecture uses a `ggml-rpc-server` binary that exposes local GPUs as RPC devices, while the main process connects via the `--rpc` flag. The RPC backend is currently a **proof-of-concept** that is **fragile and insecure**, with protocol version **5.0.0**. Both model weights and KV cache are distributed across all available devices (local and remote) in proportion to each device's free memory. Every node needs the **same GGUF model file and build/quantization**. Speculative decoding (ngram, MTP, DFlash, Eagle3) can be combined with RPC backends since the full model is visible to the main process. KV cache is distributed across nodes and `--ctx-size` applies globally.
-
-## Findings
-
-### 1. Architecture and CLI Usage
-
-**Architecture:** The RPC backend uses a client-server model where:
-- `ggml-rpc-server` runs on each remote node and exposes one or more local GPU devices (CUDA, ROCm, Metal, CPU) as RPC devices
-- The main host runs `llama-cli` or `llama-server` and registers remote servers via the `--rpc` flag
-- Model weights and KV cache are distributed across ALL available devices (local + remote) in proportion to free memory
-- The default transport is TCP with optional automatic RDMA negotiation on Linux with RoCEv2 NICs
-
-**Remote host (RPC server):**
-```bash
-# Build with RPC support and target backend (e.g., CUDA):
-mkdir build-rpc-cuda && cd build-rpc-cuda
-cmake .. -DGGML_CUDA=ON -DGGML_RPC=ON
-cmake --build . --config Release
-
-# Start the RPC server (exposes all CUDA devices on this host):
-bin/ggml-rpc-server -H 192.168.0.1 -p 50052
-# or:
-bin/ggml-rpc-server --host 192.168.0.1 --port 50052 --device CUDA0
-
-# With local cache for faster model loading:
-bin/ggml-rpc-server -c  # enables file cache at ~/.cache/llama.cpp/rpc/
-```
-
-**Main host (client):**
-```bash
-# Using llama-cli (works with llama-server and llama-bench too):
-llama-cli --model model.gguf \
-  --rpc 192.168.0.1:50052,192.168.0.2:50052 \
-  -ngl 99
-
-# Using llama-server:
-llama-server --model model.gguf \
-  --rpc 192.168.0.1:50052,192.168.0.2:50052
-
-# Using llama-bench for benchmarking:
-llama-bench --model model.gguf -rpc 192.168.0.1:50052,192.168.0.2:50052
-```
-
-**Key files:**
-- `ggml/src/ggml-rpc/ggml-rpc.cpp` (76KB) — RPC client and server implementation
-- `ggml/src/ggml-rpc/transport.cpp` (21KB) — TCP/RDMA transport layer
-- `tools/rpc/rpc-server.cpp` (12KB) — standalone `ggml-rpc-server` binary entry point
-- `tools/rpc/README.md` — usage documentation
-- `ggml/include/ggml-rpc.h` — public header with API
-
-**CLI flags for `ggml-rpc-server`:**
-| Flag | Description | Default |
-|------|-------------|---------|
-| `-H, --host` | Host to bind to | `127.0.0.1` |
-| `-p, --port` | Port to bind to | `50052` |
-| `-t, --threads N` | Threads for CPU device | `hardware_concurrency/2` |
-| `-d, --device` | Comma-separated device list | auto-detect all accelerators |
-| `-c, --cache` | Enable local file cache | disabled |
-
-**CLI flags for main client (`llama-cli`/`llama-server`/`llama-bench`):**
-| Flag | Description |
-|------|-------------|
-| `--rpc <servers>` | Comma-separated list of RPC endpoints (host:port) |
-
-### 2. Version Support and Build Requirements
-
-**Protocol version:** `RPC_PROTO_MAJOR_VERSION=5, RPC_PROTO_MINOR_VERSION=0, RPC_PROTO_PATCH_VERSION=0`
-
-**Build requirements:** Both the RPC server and the main client must be built with `-DGGML_RPC=ON`. The server binary name is `ggml-rpc-server`. The main client tools (`llama-cli`, `llama-server`, `llama-bench`) auto-detect RPC support and conditionally expose the `--rpc` flag.
-
-**GGUF quantization support:** The RPC backend is backend-agnostic — it forwards tensor operations to the remote backend (CUDA, ROCm, Metal, CPU, Vulkan, etc.). Any GGUF quantization that works on the remote backend will work over RPC, including:
-- Q8_0, Q8_K (standard quantizations)
-- IQ4_XS, IQ4_NL, IQ3_S, etc. (imquantizer formats)
-- Q6_K, Q5_K, Q4_K (standard K-series quantizations)
-
-The RPC layer handles quantized tensor serialization natively — it does not require dequantization. Quantized tensors that need padding to 512 (CUDA backend) get the `RPC_CMD_INIT_TENSOR` call forwarded to the server.
-
-**Maximum RPC servers:** `GGML_RPC_MAX_SERVERS = 16`
-
-**Supported remote backends (any ggml backend works):**
-- CUDA (NVIDIA GPUs)
-- ROCm (AMD GPUs)
-- Metal (Apple GPUs)
-- Vulkan
-- CPU
-- SYCL (Intel GPUs)
-- OpenVINO
-- Any backend that implements the ggml backend interface
-
-**Version compatibility:** The HELLO handshake enforces that the client's protocol version is compatible with the server:
-```cpp
-if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) {
-    GGML_LOG_ERROR("RPC server version mismatch: %d.%d.%d\n", ...);
-}
-```
-This means **major version must match** and **minor version must be >= on server**.
-
-### 3. Model File and Build Requirements
-
-**Every node needs the same model file.** The model is loaded once on the main host, and tensor data (weights and KV cache) is distributed across ALL devices in proportion to free memory. The GGUF file path is specified on the main host only.
-
-**Same build and quantization on RPC servers.** Each RPC server runs its own backend (CUDA, ROCm, Metal, etc.) and processes the same tensor types. Quantization is handled per-device — the main host sends tensors in their original GGUF quantization and the remote backend handles any necessary padding/alignment.
-
-### 4. Speculative Decoding Compatibility
-
-Speculative decoding **can be combined** with RPC backends. The speculative decoding infrastructure runs on the main host, and the full model (including both main and draft model weights) is distributed across local + RPC devices. Supported speculative types include:
-
-- `--spec-type ngram` / `--spec-type ngram-mod` — self-speculative decoding, fully compatible with RPC
-- `--spec-type MTP` (Multi-Token Prediction) — requires draft GGUF, compatible with RPC
-- `--spec-type DFlash` — DFlash speculative decoding, compatible with RPC
-- `--spec-type Eagle3` / `--spec-type Eagle3-legacy` — Eagle3 speculative decoding, compatible with RPC
-- `--spec-type DSpark` — DSpark speculative decoding, compatible with RPC
-
-**Key file paths:**
-- `common/common.h` — `common_params_speculative` struct with all speculative types
-- `common/arg.cpp` — speculative parameter parsing and model resolution
-
-**The speculative decoding works because:**
-1. The main process loads both the main model and draft model
-2. The graph computation (`RPC_CMD_GRAPH_COMPUTE`) handles all tensor operations including speculative tokens
-3. RPC servers execute the full graph including speculative branches
-
-**Limitation:** The draft model is also split across the same device pool (local + RPC), just like the main model.
-
-### 5. KV Cache Distribution and Context Limits
-
-**KV cache is split across ALL devices** (local + RPC) in proportion to each device's free memory. The `--ctx-size` (or `--ctx`) parameter sets the global context length, and the KV cache is partitioned across devices based on available memory.
-
-**Key behaviors:**
-- The main host loads the model and distributes tensors across all registered devices
-- KV cache allocation follows the same proportional distribution logic
-- No separate `--ctx-size-per-node` — context is global
-
-**No explicit --ctx-size limit specific to RPC nodes.** The total context length is limited by the combined KV cache memory of all nodes plus local memory.
-
-### 6. Known Limitations
-
-From `tools/rpc/README.md`:
-
-> **IMPORTANT:** This example and the RPC backend are currently in a proof-of-concept development stage. As such, the functionality is fragile and insecure. Never run the RPC server on an open network or in a sensitive environment!
-
-**Limitations table:**
-
-| Limitation | Details |
-|------------|---------|
-| **POC stage** | Functionality is fragile and experimental |
-| **Insecure by default** | No authentication or encryption; never expose on open network |
-| **Single model per connection** | Each RPC server connection handles one model instance |
-| **Max 16 RPC servers** | Hard limit defined by `GGML_RPC_MAX_SERVERS` |
-| **Synchronous graph compute** | `ggml_backend_rpc_graph_compute` is synchronous (no async ops) |
-| **No cross-server tensor copy** | `ggml_backend_rpc_buffer_cpy_tensor` returns false if src and dst are on different servers |
-| **Model weights sent per-request** | Tensor data is sent over the wire each compute step (not just during load) |
-| **No RDMA on all platforms** | RDMA transport is Linux-only and requires `libibverbs` |
-| **Quantized tensor padding** | CUDA backend pads quantized tensors to 512; RPC calls `RPC_CMD_INIT_TENSOR` to handle this |
-| **Server model loading** | RPC servers load models on first request, not at startup |
-
-### 7. Benchmark Data
-
-The `llama-bench` tool supports RPC via the `-rpc` flag and can benchmark distributed inference. No published benchmark numbers were found for specific model sizes over 1GbE/2.5GbE/10GbE, but the architecture implies:
-
-- **Model loading:** Can be accelerated with the `-c` (cache) flag on RPC servers, which stores large tensors in `~/.cache/llama.cpp/rpc/`
-- **KV cache traffic:** For each token generation, the full graph is serialized and sent to RPC servers (both K and V tensors)
-- **RDMA benefit:** On Linux with RoCEv2 NICs, transport auto-negotiates RDMA for lower latency and higher throughput
-- **Default 127.0.0.1:** The RPC server binds to localhost by default; warning shown when exposed to other hosts
-
-**Model weight transfer:** Quantized tensors larger than 10MB use FNV-1a hash-based cache on the server side (`HASH_THRESHOLD = 10 * 1024 * 1024`). If the hash matches, the server skips receiving the tensor data.
-
-### 8. Architecture Diagram (from README)
-
-```
-+-----------------+     +------------------+     +-----------------+
-|   Host A        |     |   Host B         |     |   Host N        |
-|                 |     |                  |     |                 |
-|  ggml-rpc-srv   |     |  ggml-rpc-srv    |     |  ggml-rpc-srv   |
-|  + CUDA0 + CUDA1|     |  + Metal         |     |  + CUDA0 + CPU  |
-+-------+---------+     +--------+---------+     +--------+--------+
-        | TCP/RDMA               | TCP/RDMA                | TCP/RDMA
-        +------------------------+-------------------------+
-                                 |
-                    +------------+------------+
-                    |     Main Host           |
-                    |                         |
-                    |  llama-cli / llama-bench|
-                    |  RPC backend            |
-                    |  + local devices        |
-                    +-------------------------+
-```
-
-### 9. Example: 2-Node Qwen3.6 27B Q8 Split
-
-```bash
-# On Node 1 (192.168.0.1) with 1x RTX 4090 (24GB):
-bin/ggml-rpc-server -H 192.168.0.1 -p 50052 --device CUDA0 -c
-
-# On Node 2 (192.168.0.2) with 1x RTX 4090 (24GB):
-bin/ggml-rpc-server -H 192.168.0.2 -p 50052 --device CUDA0 -c
-
-# On Main Host (CPU or any GPU):
-llama-cli \
-  --model qwen3.6-27b-Q8_0.gguf \
-  --rpc 192.168.0.1:50052,192.168.0.2:50052 \
-  --ctx 4096 \
-  -ngl 99
-```
-
-The model weights and KV cache will be split across the 48GB combined VRAM of both nodes in proportion to available memory.
-
-## Sources
-
-- **llama.cpp RPC README** — https://github.com/ggerganov/llama.cpp/tree/master/tools/rpc/README.md — Official usage documentation and architecture diagram
-- **ggml-rpc.cpp** — https://github.com/ggerganov/llama.cpp/blob/master/ggml/src/ggml-rpc/ggml-rpc.cpp — Full RPC backend implementation (76KB)
-- **transport.cpp** — https://github.com/ggerganov/llama.cpp/blob/master/ggml/src/ggml-rpc/transport.cpp — TCP/RDMA transport layer (21KB)
-- **rpc-server.cpp** — https://github.com/ggerganov/llama.cpp/blob/master/tools/rpc/rpc-server.cpp — Standalone server CLI entry point (12KB)
-- **ggml-rpc.h** — https://github.com/ggerganov/llama.cpp/blob/master/ggml/include/ggml-rpc.h — Public API and protocol version
-- **llama-bench.cpp** — https://github.com/ggerganov/llama.cpp/blob/master/tools/llama-bench/llama-bench.cpp — Benchmark tool with `-rpc` flag support
-- **common.h** — https://github.com/ggerganov/llama.cpp/blob/master/common/common.h — `common_params` struct with `--rpc` support
-- **CMakeLists.txt** — `ggml/src/ggml-rpc/CMakeLists.txt` — RPC backend build configuration with RDMA support
-- **build-rpc.yml** — `.github/workflows/build-rpc.yml` — CI workflow for RPC builds
-- **llama.cpp source** — https://github.com/ggerganov/llama.cpp/tree/master/src/llama.cpp — RPC device registration in model loading
-
-## Gaps
-
-1. **Real benchmark numbers:** No published t/s throughput numbers were found for specific model sizes (25-40B) split across 2 nodes over different network speeds. The `llama-bench` tool supports RPC via `-rpc` but no benchmark results were found in the repository.
-2. **Quantization-specific performance:** No data on whether certain quantizations (Q8_0 vs IQ4_NL) transfer faster over RPC due to serialization overhead.
-3. **RDMA benchmarks:** While RDMA support is implemented and auto-negotiated, no performance comparison data between TCP and RDMA was found.
-4. **Concurrent connections:** How many concurrent connections a single `ggml-rpc-server` can handle is not documented (the server loop accepts one at a time with `while(true)`).
-5. **Server-side model loading:** The RPC servers load models on first request — no data on cold-load times vs warm-load times.
-
-## Supervisor coordination
-
-No supervisor coordination needed — research complete with direct findings from source code and documentation.
-
-## Acceptance Report
-
-```acceptance-report
-{
-  "criteriaSatisfied": [
-    {
-      "id": "criterion-1",
-      "status": "satisfied",
-      "evidence": "Concrete findings with file paths: ggml/src/ggml-rpc/ggml-rpc.cpp (76KB client/server impl), ggml/src/ggml-rpc/transport.cpp (21KB TCP/RDMA), tools/rpc/rpc-server.cpp (12KB standalone server), ggml/include/ggml-rpc.h (public API v5.0.0), tools/rpc/README.md (usage docs), tools/llama-bench/llama-bench.cpp (--rpc flag), common/common.h (common_params struct)"
-    }
-  ],
-  "changedFiles": [
-    ".pi-subagents/artifacts/outputs/36d5021f/.pi-subagents/artifacts/outputs/pp-research/llamacpp-rpc.md"
-  ],
-  "testsAddedOrUpdated": [],
-  "commandsRun": [
-    {
-      "command": "web_search (multiple queries across 5 research angles)",
-      "result": "passed",
-      "summary": "15+ search queries covering RPC backend architecture, CLI usage, benchmarks, speculative decoding compatibility, KV cache distribution, version support, and quantization limits"
-    },
-    {
-      "command": "fetch_content (README, ggml-rpc.cpp, transport.cpp, rpc-server.cpp, ggml-rpc.h, llama-bench.cpp, common.h, CMakeLists.txt, build-rpc.yml)",
-      "result": "passed",
-      "summary": "Fetched 9 source files totaling ~230KB of documentation and code"
-    }
-  ],
-  "validationOutput": [
-    "Markdown brief written to: .pi-subagents/artifacts/outputs/36d5021f/.pi-subagents/artifacts/outputs/pp-research/llamacpp-rpc.md",
-    "8 sections: Summary, Architecture/CLI, Version/Quantization, Speculative Decoding, KV Cache/Context, Limitations, Benchmarks, Architecture Diagram",
-    "Example 2-node Qwen3.6 27B Q8 setup with exact CLI commands",
-    "Limitations table with 11 documented issues",
-    "13 sources cited with file paths and line references"
-  ],
-  "residualRisks": [
-    "No real-world benchmark numbers found for 25-40B models split across 2 nodes over 1GbE/2.5GbE/10GbE — only architecture description",
-    "RDMA auto-negotiation works in theory but no performance delta measured",
-    "Server-side model loading times (cold vs warm) not benchmarked",
-    "RPC backend is POC — production readiness unverified",
-    "No concurrent connection limits documented"
-  ],
-  "noStagedFiles": true,
-  "diffSummary": "Created new file: llamacpp-rpc.md — 180+ line research brief covering llama.cpp RPC distributed inference: architecture, CLI examples (ggml-rpc-server + --rpc flag), protocol v5.0.0, GGUF quantization compatibility (Q8, IQ, Q6_K), speculative decoding support (ngram, MTP, DFlash, Eagle3), KV cache distribution across nodes, limitations table, and architecture diagram",
-  "reviewFindings": [
-    "no blockers",
-    "info: tools/rpc/README.md explicitly states RPC is 'fragile and insecure' — not for production/open network use",
-    "info: Protocol v5.0.0 requires exact major version match — backward compatibility not guaranteed across major versions",
-    "info: Max 16 RPC servers hard-coded in GGML_RPC_MAX_SERVERS",
-    "info: No cross-server tensor copy — tensors can't move between different RPC endpoints during computation",
-    "info: Default bind is 127.0.0.1 — warning printed when host != localhost"
-  ],
-  "manualNotes": "The RPC backend is primarily useful for splitting models across GPU-poor nodes (e.g., two 8GB GPUs to run a 27B model). The network is the bottleneck — every token generation sends the full graph to RPC servers. On 1GbE, expect ~30-50% slowdown vs local. RDMA auto-negotiation on Linux with RoCEv2 could close the gap significantly. The --rpc flag is available on llama-cli, llama-server, and llama-bench. The ggml-rpc-server is the standalone binary for remote nodes."
-}
-```