frank-medium: serve the new R9700s, retire frank-small

The RTX 3090 left Dankserver; two AMD Radeon AI PRO R9700 cards (Navi 48,
gfx1201, 32,624 MiB each) replaced it. FreeToken/CUDA cannot run there at all,
so the `frank-small` service is retired and `frank-medium` takes the endpoint.

New service: charlie12345/ROCmFPX llama.cpp (built for gfx1201 against the
fork's self-contained ROCm 7.14 toolchain, so the host's stale /opt/rocm-6.3.3
never participates) serving kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF --
a Q8_0_ROCMFPX_AGENT (ftype 115) checkpoint that stock llama.cpp cannot load.
Placeholder config: --split-mode tensor across both cards, 131072 context,
1 slot, f16 KV, MTP draft head on ROCm0.

deploy-remote.sh grows retire_root_llm_small(): the CUDA-era unit is stopped,
disabled and moved to ~/.AGENTTRASH/20260915-frank-small-retire/host-units
before frank-medium starts, because `systemctl disable` has been observed to
delete the unit file itself on this host (2026-08-26 incident).

The retired frank-small repo files (env, unit, serve script) move to
scripts/.AGENTTRASH/, and the planner, dispatcher, service allow-list and the
deployment test seams now name frank-medium.

scripts/deploy-plan.py

@@ -60,7 +60,7 @@ SERVICE_ORDER = (
     "lidarr",
     "readarr",
     "frank-large",
-    "frank-small",
+    "frank-medium",
 )
 
 DANKCLAW_SERVICES = {
@@ -107,7 +107,7 @@ DANKSERVER_SERVICES = {
     "sonarr",
     "lidarr",
     "readarr",
-    "frank-small",
+    "frank-medium",
 }
 
 # FrankBook-Pro runs the *arr stack + qBittorrent inside a podman machine on
@@ -368,12 +368,13 @@ def services_for_path(path: str, target: str) -> set[str]:
 
     if parts[0] == "scripts":
         name = p.name
-        if name in {"frank-small.env", "frank-small.service", "serve-frank-small.sh"}:
+        if name in {"frank-medium.env", "frank-medium.service", "serve-frank-medium.sh"}:
-            _add(services, "frank-small")
+            _add(services, "frank-medium")
         elif name == "chat-template.jinja":
-            # Only frank-small consumes this template now: frank-large runs
+            # frank-medium reads this template only when its env file points
-            # halogen-flash-server, which owns its own chat handling.
+            # at it (the GGUF's embedded template is the default); frank-large
-            _add(services, "frank-small")
+            # runs halogen-flash-server, which owns its own chat handling.
+            _add(services, "frank-medium")
         elif name in {"homepage-status.py", "homepage-status.service", "homepage-status.timer"}:
             _add(services, "homepage")
         # Deployment/test/benchmark scripts and other repo tooling do not

scripts/deploy-remote.sh

@@ -95,7 +95,7 @@ service_is_allowed() {
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
         dankserver:prowlarr|dankserver:qbittorrent|dankserver:radarr|\
         dankserver:sonarr|dankserver:lidarr|dankserver:readarr|\
-        dankserver:frank-small|\
+        dankserver:frank-medium|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)
@@ -181,7 +181,6 @@ add_pending_services() {
             continue
         fi
         [[ -n "$recorded" && "$recorded" != "$REVISION" ]] || continue
-        if [[ "$TARGET" == dankclaw && "$service" == frank-small ]]; then continue; fi
         if [[ "$TARGET" == dankserver && "$service" == frank-large ]]; then continue; fi
         if ! changed="$(python3 scripts/deploy-plan.py --target "$TARGET" \
             --repo "$REMOTE_DIR" --base "$recorded" --head "$REVISION")"; then
@@ -992,14 +991,35 @@ wait_for_llm_large_ready() {
     return 0
 }
 
-deploy_llm_small() {
+# deploy_llm_medium — frank-medium replaced frank-small on 2026-09-15 when the
-    run_privileged systemctl stop frank-default-small || true
+# RTX 3090 left the host for two Radeon AI PRO R9700 cards. The old unit is a
-    run_privileged systemctl disable frank-default-small || true
+# CUDA-era FreeToken service that can only crash-loop on this hardware, so it
-    relink_unit frank-small
+# is retired here (stop, disable, move the installed unit to the trash) before
+# its replacement starts. Same failure mode as frank-large's retirement: this
+# host's `systemctl disable` has been observed to delete the unit file itself
+# (2026-08-26 incident), so both outcomes are tolerated.
+retire_root_llm_small() {
+    local trash="$HOME/.AGENTTRASH/20260915-frank-small-retire/host-units"
+    run_privileged systemctl stop frank-default-small 2>/dev/null || true
+    run_privileged systemctl disable frank-default-small 2>/dev/null || true
+    run_privileged systemctl stop frank-small.service 2>/dev/null || true
+    run_privileged systemctl disable frank-small.service 2>/dev/null || true
+    if [[ -e /etc/systemd/system/frank-small.service || -L /etc/systemd/system/frank-small.service ]]; then
+        mkdir -p "$trash"
+        run_privileged mv -- /etc/systemd/system/frank-small.service \
+            "$trash/frank-small.service"
+        echo "retired root unit frank-small.service -> $trash"
+    fi
     run_privileged systemctl daemon-reload
-    run_privileged systemctl enable frank-small
+}
-    run_privileged systemctl restart frank-small
+
-    wait_for_health frank-small http://127.0.0.1:7770/health 1200
+deploy_llm_medium() {
+    retire_root_llm_small
+    relink_unit frank-medium
+    run_privileged systemctl daemon-reload
+    run_privileged systemctl enable frank-medium
+    run_privileged systemctl restart frank-medium
+    wait_for_health frank-medium http://127.0.0.1:7770/health 1200
 }
 
 wait_for_health() {
@@ -1187,7 +1207,7 @@ deploy_service() {
         seerr) deploy_seerr ;;
         marreta) deploy_marreta ;;
         frank-large) deploy_llm_large ;;
-        frank-small) deploy_llm_small ;;
+        frank-medium) deploy_llm_medium ;;
         hermes|hermes-dashboard|hermes-webui) deploy_hermes ;;
         romm|romm-mariadb|romm-redis|mariadb|redis) deploy_romm_stack "$service" ;;
         jellyfin) deploy_jellyfin ;;

scripts/deploy.sh

@@ -35,7 +35,7 @@ Options:
 Examples:
   ./scripts/deploy.sh dankclaw
   ./scripts/deploy.sh dankclaw --service searxng
-  ./scripts/deploy.sh dankserver --service frank-small
+  ./scripts/deploy.sh dankserver --service frank-medium
   ./scripts/deploy.sh dankclaw --dry-run
 EOF
 }
@@ -145,7 +145,7 @@ valid_service() {
         dankserver:copyparty|dankserver:jellyfin|dankserver:calibre-web|dankserver:plex|dankserver:tautulli|\
         dankserver:qbittorrent|dankserver:radarr|dankserver:sonarr|\
         dankserver:lidarr|dankserver:readarr|dankserver:node-exporter|\
-        dankserver:frank-small|\
+        dankserver:frank-medium|\
         frankbook-pro:gluetun|frankbook-pro:qbittorrent|frankbook-pro:prowlarr|\
         frankbook-pro:radarr|frankbook-pro:sonarr|frankbook-pro:lidarr|\
         frankbook-pro:readarr)

scripts/download-q8agent.sh

@@ -1,12 +1,18 @@
 #!/usr/bin/env bash
 # Download the kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT payload for
-# frank-large from HuggingFace into /home/dank/models/.../Q8_0-AGENT.
+# frank-medium from HuggingFace into
+# /home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT.
 #
-# Uses the token in ~/.cache/huggingface/token so the ~28 GB main file does not
+# Run on Dankserver (the box that serves it).  The repo is public, so the
-# hit the anonymous 10 GB/30 min rate limit. Run on Dankclaw.
+# download works anonymously; a token in ~/.cache/huggingface/token is used
+# when present, which keeps the ~28 GB main file clear of the anonymous
+# rate limit at full speed.
 #
 #   bash scripts/download-q8agent.sh
 #   bash scripts/download-q8agent.sh --verify-only
+#
+# The script is resumable (curl -C -) and verifies every artifact's size
+# afterwards, so a truncated file fails loudly instead of load-crashing.
 set -euo pipefail
 
 REPO="kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF"
@@ -16,6 +22,12 @@ EXPECTED=(
   "mmproj-Qwen3.8-27B-BF16.gguf:931145888"
   "mtp-Qwen3.8-27B-Q4_0.gguf:1680271648"
 )
+# The model card's MTP prompt-cache fix ships in the same repo as a patch;
+# fetched for reference only.  This fork's main already carries the
+# speculative boundary in the context checkpoint, so nothing applies it.
+EXTRA=(
+  "patches/mtp-prompt-cache-fix.patch"
+)
 
 cd -- "$(dirname -- "$0")/.."
 
@@ -25,30 +37,38 @@ main() {
   mkdir -p "$DIR"
   cd "$DIR"
 
-  local token
+  local token="" auth=()
-  token="$(cat "$HOME/.cache/huggingface/token")"
+  if [[ -r "$HOME/.cache/huggingface/token" ]]; then
-  [[ -n "$token" ]] || { echo "no HF token in $HOME/.cache/huggingface/token" >&2; exit 1; }
+    token="$(cat "$HOME/.cache/huggingface/token")"
+    auth=(-H "Authorization: Bearer $token")
+  else
+    echo "no HF token at $HOME/.cache/huggingface/token; downloading anonymously"
+  fi
 
   dl() {
     local f="$1"
-    curl -fsSL -C - -o "$f" --retry 5 --retry-delay 3 --retry-max-time 3600 \
+    mkdir -p "$(dirname "$f")"
-      -H "Authorization: Bearer $token" \
+    curl -fsSL -C - -o "$f" --retry 10 --retry-delay 5 --retry-max-time 10800 \
+      "${auth[@]}" \
       "https://huggingface.co/$REPO/resolve/main/$f"
     echo "  done: $f ($(du -h "$f" | cut -f1))"
   }
 
   if [[ "$mode" == "download" ]]; then
     echo "downloading $REPO into $DIR"
-    dl "Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf"
+    for entry in "${EXPECTED[@]}"; do
-    dl "mmproj-Qwen3.8-27B-BF16.gguf"
+      dl "${entry%%:*}"
-    dl "mtp-Qwen3.8-27B-Q4_0.gguf"
+    done
+    for f in "${EXTRA[@]}"; do
+      dl "$f"
+    done
   fi
 
   echo "verifying sizes"
   local ok=1 f want act
   for entry in "${EXPECTED[@]}"; do
     f="${entry%%:*}"; want="${entry##*:}"
-    act="$(stat -c%s "$f")"
+    act="$(stat -c%s "$f" 2>/dev/null || echo 0)"
     if [[ "$act" -eq "$want" ]]; then
       echo "  OK   $f ($act bytes)"
     else
@@ -56,6 +76,9 @@ main() {
       ok=0
     fi
   done
+  if [[ -z "$token" ]]; then
+    echo "note: no token was used; sizes above are the integrity check"
+  fi
   [[ "$ok" -eq 1 ]] || exit 1
   echo "all files present and sized correctly"
 }

scripts/frank-medium.env

@@ -0,0 +1,132 @@
+# Tunables for frank-medium (kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF,
+# charlie12345/ROCmFPX llama.cpp, Dankserver, 2x AMD Radeon AI PRO R9700).
+#
+# Hardware (2026-09-15): the RTX 3090 left Dankserver and two Radeon AI PRO
+# R9700 cards (Navi 48, gfx1201, RDNA4, 32,624 MiB each, PCIe 5.0 x16) took
+# its place.  The Raphael iGPU (gfx1036) is still enumerated by HIP as the
+# third device.  There is no XGMI link between the two cards and they sit in
+# different IOMMU groups, so any cross-GPU traffic goes over PCIe.
+#
+# Runtime: charlie12345/ROCmFPX (llama.cpp fork, ROCmFPX quant family; the
+# Q8_0_ROCMFPX_AGENT tensor type exists nowhere else - stock llama.cpp cannot
+# load this GGUF).  Built for gfx1201 with the fork's self-contained ROCm
+# 7.14 toolchain so the host's stale /opt/rocm-6.3.3 install is never used:
+#
+#   cd /home/dank/GitPackages/charlie-ROCmFPX
+#   env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh
+#
+# gfx1201 must be the build target: a gfx1200 binary links, loads the model
+# and then segfaults on this card (fork's docs/BUILD-AMD-ARCHITECTURES.md).
+#
+# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT - Q8_0_ROCMFPX_AGENT
+# (ftype 115), 26.28 GiB, 8.39 bpw, arch qwen35 (MoE), quantized from a
+# 51.3 GiB BF16 source.  The AGENT routing keeps 12 more attention-output
+# projections at full Q8_0 than the plain 8-bit build, which buys a higher
+# MTP draft acceptance (0.953 vs 0.911) and that is the whole point of this
+# checkpoint.  Model card documents 131,072 context.
+#
+# Parallelism (--split-mode tensor): tensor parallel splits the weights AND
+# the KV cache across both cards through the fused "meta device".  It is the
+# only split mode here that gives each card half the KV cache; layer split
+# would give each card the KV of the layers it owns, with one PCIe handoff
+# per token per boundary.  TENSOR mode's own constraints, all met below:
+# flash attention ON, non-quantized KV cache, and no auto-fit.
+#
+# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
+# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
+# incident).  llama.cpp reads --host/--port/--temp as CLI flags, never as
+# env vars.
+
+# charlie12345/ROCmFPX llama-server (self-contained ROCm 7.14 build, RPATH
+# points at the bundled runtime libs; no LD_LIBRARY_PATH needed).
+LLAMA_BIN=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server
+
+# Model package, fetched with scripts/download-q8agent.sh:
+#   Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf   26.28 GiB  main weights
+#   mtp-Qwen3.8-27B-Q4_0.gguf              1.68 GiB  MTP draft head
+#   mmproj-Qwen3.8-27B-BF16.gguf           0.93 GiB  vision tower (unused)
+MODEL_DIR=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT
+MODEL_FILE=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf
+DRAFT_MODEL_FILE=mtp-Qwen3.8-27B-Q4_0.gguf
+
+# Alias + network.  The port is the fleet's dankserver endpoint; the alias is
+# what clients ask for, and it replaces the retired `frank-small`.
+ALIAS=frank-medium
+BIND_ADDR=0.0.0.0
+LISTEN_PORT=7770
+
+# Device placement.  HIP_VISIBLE_DEVICES hides the iGPU (device 2) so nothing
+# can drift onto shared system RAM; DEVICES is what llama.cpp splits across.
+HIP_VISIBLE_DEVICES=0,1
+DEVICES=ROCm0,ROCm1
+SPLIT_MODE=tensor
+
+# TENSOR_SPLIT empty = an even split between the two cards.
+TENSOR_SPLIT=
+
+NGL=999
+# Tensor mode ignores auto-fit, and auto-fit has no business resizing a
+# context we set explicitly.
+FIT=off
+FLASH_ATTN=on
+
+# 131,072 tokens, one slot (the model card's documented window).  Tensor mode
+# puts half the KV cache on each card, so the window is not limited by a
+# single card's free memory.
+CTX_SIZE=131072
+PARALLEL=1
+
+# 26.28 GiB of weights + MTP draft + KV cache leaves ~15 GiB per card at this
+# context, so the batch shape is chosen for prefill throughput, not memory.
+BATCH_SIZE=2048
+UBATCH_SIZE=512
+NUM_THREADS=8
+THREAD_BATCH=8
+
+# f16 K/V: tensor mode refuses a quantized KV cache, and an agent checkpoint
+# is the wrong place to spend KV precision anyway.
+CACHE_TYPE_K=f16
+CACHE_TYPE_V=f16
+CTX_CHECKPOINTS=8
+CACHE_RAM=8192
+
+# Sampling: the checkpoint's instruct table.
+TEMPERATURE=0.7
+TOP_P=0.8
+TOP_K=20
+MIN_P=0.0
+PRESENCE_PENALTY=1.5
+
+# Reasoning on; DeepSeek format projects the trace into reasoning_content
+# instead of literal thinking tags in content.
+REASONING=on
+REASONING_FORMAT=deepseek
+REASONING_BUDGET=-1
+
+# Chat template: empty = the template embedded in the GGUF.  Point this at
+# /home/dank/serverAppConfigs/scripts/chat-template.jinja to force the repo's
+# Sharp agent template instead.
+CHAT_TEMPLATE_FILE=
+CHAT_TEMPLATE_KWARGS='{"enable_thinking":true,"preserve_thinking":true}'
+
+# MTP speculative decoding.  n_max 6 / p_min 0.6 is the fork's measured
+# sweet spot for a MTP head with high acceptance; p_min 0.0 drafts
+# low-probability tokens that get rejected and makes MTP a net loss.
+SPEC_TYPE=draft-mtp
+SPEC_DRAFT_N_MAX=6
+SPEC_DRAFT_N_MIN=0
+SPEC_DRAFT_P_MIN=0.6
+SPEC_DRAFT_P_SPLIT=0.10
+SPEC_DRAFT_NGL=999
+SPEC_DRAFT_TYPE_K=f16
+SPEC_DRAFT_TYPE_V=f16
+SPEC_DRAFT_POLL=1
+SPEC_DRAFT_POLL_BATCH=1
+# The draft head must run on ONE device.  With --split-mode tensor the draft
+# otherwise joins the meta-device allocation and the KV-cache build aborts:
+#   ggml/src/ggml-backend.cpp:119: GGML_ASSERT(buffer) failed
+#   <- ggml_backend_meta_alloc_ctx_tensors_from_buft <- llama_kv_cache ctor
+# (measured 2026-09-15 on the built runtime: identical flags are healthy with
+# --spec-draft-device ROCm0 and abort with it empty).  ROCm0 is the card that
+# also holds the target model's main GPU.
+SPEC_DRAFT_DEVICE=ROCm0

scripts/frank-medium.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=charlie12345/ROCmFPX llama.cpp serving kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT qwen35 MoE (dankserver, frank-medium, 2x Radeon AI PRO R9700)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=dank
+Group=dank
+Environment="FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env"
+ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh
+Restart=on-failure
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target

scripts/frank-small.env

@@ -1,56 +0,0 @@
-# Tunables for frank-small (batsclamp/Huihui-Qwen3.6-35B-A3B-
-# abliterated-FP8, qwen3_5_moe hybrid MoE, Dankserver, RTX 3090).
-#
-# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine).
-# Installed via uv into /home/dank/freetoken/.venv (CPython 3.12).
-# FreeToken keeps expert weights in host RAM with an LRU GPU cache;
-# KV is paged with radix prefix sharing.  Only the 10 full-attention
-# layers page KV (~20 KiB/token); the 30 GDN layers use constant-size
-# per-sequence state (mamba slot pool).
-#
-# Model: batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 (HuggingFace
-# safetensors).  Abliterated (refusal-removed) Huihui finetune of
-# Qwen3.6-35B-A3B, quantized to the SAME native format the previous
-# official checkpoint used: modelopt block-fp8 e4m3, dynamic
-# activations, 128x128 weight_block_size, bf16 weight_scale_inv;
-# lm_head/embeds/norms/router gates/visual/MTP kept BF16.  Un-fused
-# per-expert layout matches the official Qwen FP8, so the FreeToken
-# loader path is identical; ~35 GB on disk.  Ships the fixed Qwen3.6
-# chat template (froggeric/Qwen-Fixed-Chat-Templates) — FreeToken
-# reads it from the checkpoint; no template override needed.
-# Replaced Qwen/Qwen3.6-35B-A3B-FP8 on 2026-08-30; pool numbers
-# unchanged — same arch, same weight mass.
-#
-# Pool sizing: GPU budget (~17.6 GiB after weights + fixed caches) is
-# split MoE-first.  MOE_CACHE_SIZE pins resident expert slots; NUM_TOKENS
-# pins the KV pool.  At 2048 slots / 453005 tokens the split is ~20%
-# residency (2048/10240) and ~8.6 GiB KV.  Live resize via
-# `ft ctl cache --kv N --moe M` can grow KV by shrinking MoE without
-# restart (the reverse requires a restart).
-#
-# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE.
-# The ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
-# incident).  This box is NVIDIA/CUDA, but the rule keeps the env portable.
-
-# Model directory (safetensors checkpoint, ~35 GB, block-fp8).
-MODEL_DIR=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8
-
-# FreeToken venv (uv-managed CPython 3.12, ft CLI, flashinfer, torch+cu130).
-FT_VENV=/home/dank/freetoken/.venv
-CUDA_HOME_DIR=/home/dank/freetoken/.venv/lib/python3.12/site-packages/nvidia/cu13
-
-ALIAS=frank-small
-BIND_ADDR=0.0.0.0
-LISTEN_PORT=7770
-
-# Expert residency: 2048 of 10240 total expert instances (20%).
-# Each slot ~3 MiB → ~6.0 GiB VRAM for the MoE cache.
-MOE_CACHE_SIZE=2048
-
-# KV pool: 453005 tokens (~8.6 GiB at ~20 KiB/token).
-# Covers 4 concurrent sessions at ~113k unique context each
-# (more with radix prefix sharing for common prefixes).
-NUM_TOKENS=453005
-
-# Max concurrent decoding requests (also caps mamba slot allocation).
-MAX_RUNNING_REQUESTS=4

scripts/frank-small.service

@@ -1,16 +0,0 @@
-[Unit]
-Description=FreeToken Huihui-Qwen3.6-35B-A3B-abliterated-FP8 qwen3_5_moe hybrid MoE (dankserver, frank-small)
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User=dank
-Group=dank
-Environment="FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env"
-ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh
-Restart=on-failure
-RestartSec=10
-
-[Install]
-WantedBy=multi-user.target

scripts/serve-frank-medium.sh

@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+# Serve kingjones777/Qwen3.8-27B-ROCmFPX-Q8_0-AGENT-GGUF as frank-medium via
+# the charlie12345/ROCmFPX llama.cpp runtime on Dankserver
+# (2x AMD Radeon AI PRO R9700, Navi 48, gfx1201, 32 GB each).
+#
+# Runtime: charlie12345/ROCmFPX main (c49ebdb, 2026-08-22), built for
+# gfx1201 with the fork's self-contained ROCm toolchain:
+#
+#   cd /home/dank/GitPackages/charlie-ROCmFPX
+#   env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh
+#   -> build-rdna4-rocm714/bin/llama-server
+#
+# That script downloads the gfx120X-all TheRock ROCm 7.14 toolchain into
+# ~/.local/share/rocm-7.14.0a20260624, builds with
+# -DCMAKE_HIP_ARCHITECTURES="gfx1200;gfx1201", copies the ROCm runtime
+# (hip/hsa/rocblas/hipblaslt + their kernel libraries) next to the binary and
+# sets RPATH=$ORIGIN/../lib on it.  The resulting service is self-contained:
+# no LD_LIBRARY_PATH, and the host's stale /opt/rocm-6.3.3 cannot leak in.
+# gfx1201 is NOT interchangeable with gfx1200 on this card (the fork's
+# docs/BUILD-AMD-ARCHITECTURES.md: a gfx1200 build links, loads and then
+# segfaults), and the build verifies the emitted code objects afterwards.
+#
+# Model: kingjones Qwen3.8-27B-ROCmFPX-Q8_0-AGENT (Q8_0_ROCMFPX_AGENT,
+# ftype 115, arch qwen35 MoE), 26.28 GiB main file, plus a separate
+# MTP draft head and an mmproj vision tower.  8.39 bpw; the AGENT routing
+# keeps 13 of the 17 attention-output projections at full Q8_0, which is
+# what makes the MTP draft land ~0.95 of its guesses.  A stock llama.cpp
+# cannot load this GGUF at all - Q8_0_ROCMFPX exists only in this fork.
+#
+# Parallelism: --split-mode tensor across both cards.  Tensor parallel
+# splits the weights AND the KV cache across the participating devices via
+# the fused "meta device" (unlike -sm layer, which pipelines layers and
+# keeps each layer's KV on its own card).  It requires flash attention and
+# a non-quantized KV cache, and it is disabled from auto-fit (--fit off).
+#
+# Tunables come from frank-medium.env unless FRANK_MEDIUM_ENV_FILE
+# overrides that path.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ENV_FILE="${FRANK_MEDIUM_ENV_FILE:-$SCRIPT_DIR/frank-medium.env}"
+if [[ -f "$ENV_FILE" ]]; then
+    set -a
+    # shellcheck disable=SC1090
+    source "$ENV_FILE"
+    set +a
+fi
+
+# --- defaults (used when the env file does not set them) -----------------
+: "${LLAMA_BIN:=/home/dank/GitPackages/charlie-ROCmFPX/build-rdna4-rocm714/bin/llama-server}"
+: "${MODEL_DIR:=/home/dank/models/kingjones-Qwen3.8-27B-ROCMFPX-Q8_0-AGENT}"
+: "${MODEL_FILE:=Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}"
+: "${DRAFT_MODEL_FILE:=mtp-Qwen3.8-27B-Q4_0.gguf}"
+: "${ALIAS:=frank-medium}"
+: "${BIND_ADDR:=0.0.0.0}"
+: "${LISTEN_PORT:=7770}"
+
+# Devices: ROCm0/ROCm1 are the two R9700s; the Raphael iGPU enumerates as
+# ROCm2 (gfx1036) and is hidden from HIP so nothing can drift onto it.
+: "${HIP_VISIBLE_DEVICES:=0,1}"
+: "${DEVICES:=ROCm0,ROCm1}"
+: "${SPLIT_MODE:=tensor}"
+: "${TENSOR_SPLIT:=}"
+: "${NGL:=999}"
+: "${FIT:=off}"
+: "${FLASH_ATTN:=on}"
+
+: "${CTX_SIZE:=131072}"
+: "${PARALLEL:=1}"
+: "${BATCH_SIZE:=2048}"
+: "${UBATCH_SIZE:=512}"
+: "${NUM_THREADS:=8}"
+: "${THREAD_BATCH:=8}"
+: "${CACHE_TYPE_K:=f16}"
+: "${CACHE_TYPE_V:=f16}"
+: "${CTX_CHECKPOINTS:=8}"
+: "${CACHE_RAM:=8192}"
+
+# Sampling: the checkpoint's instruct table.
+: "${TEMPERATURE:=0.7}"
+: "${TOP_P:=0.8}"
+: "${TOP_K:=20}"
+: "${MIN_P:=0.0}"
+: "${PRESENCE_PENALTY:=1.5}"
+
+# Reasoning: the model is an agent checkpoint; keep the trace and project it
+# into the API's reasoning_content field rather than literal thinking tags.
+: "${REASONING:=on}"
+: "${REASONING_FORMAT:=deepseek}"
+: "${REASONING_BUDGET:=-1}"
+
+# Chat template: empty means "use the template embedded in the GGUF".
+: "${CHAT_TEMPLATE_FILE:=}"
+: "${CHAT_TEMPLATE_KWARGS:={\"enable_thinking\":true,\"preserve_thinking\":true}}"
+
+# MTP draft head (separate GGUF, --spec-type draft-mtp).  SPEC_DRAFT_DEVICE
+# must name exactly one device: under tensor mode an unsplit draft joins the
+# meta-device KV allocation and aborts while building the KV cache
+# (GGML_ASSERT(buffer) in ggml-backend.cpp, 2026-09-15).
+: "${SPEC_TYPE:=draft-mtp}"
+: "${SPEC_DRAFT_N_MAX:=6}"
+: "${SPEC_DRAFT_N_MIN:=0}"
+: "${SPEC_DRAFT_P_MIN:=0.6}"
+: "${SPEC_DRAFT_P_SPLIT:=0.10}"
+: "${SPEC_DRAFT_NGL:=999}"
+: "${SPEC_DRAFT_DEVICE:=}"
+: "${SPEC_DRAFT_TYPE_K:=f16}"
+: "${SPEC_DRAFT_TYPE_V:=f16}"
+: "${SPEC_DRAFT_POLL:=1}"
+: "${SPEC_DRAFT_POLL_BATCH:=1}"
+
+# Text only: the server must not try to autoload the vision tower.  Vision
+# stays on frank-large (Dankclaw), which is where the fleet routes images.
+NO_MMPROJ="${NO_MMPROJ:-1}"
+
+export HIP_VISIBLE_DEVICES
+
+MODEL="$MODEL_DIR/$MODEL_FILE"
+if [[ ! -x "$LLAMA_BIN" ]]; then
+    echo "missing executable runtime: $LLAMA_BIN" >&2
+    echo "build it with: cd $(dirname "$(dirname "$LLAMA_BIN")")/.. && env JOBS=8 ./scripts/build-rocmfp4-rocm714-local.sh" >&2
+    exit 1
+fi
+if [[ ! -r "$MODEL" ]]; then
+    echo "missing or unreadable required artifact: $MODEL" >&2
+    echo "fetch it with: bash $SCRIPT_DIR/download-q8agent.sh" >&2
+    exit 1
+fi
+if [[ -n "$CHAT_TEMPLATE_FILE" && ! -r "$CHAT_TEMPLATE_FILE" ]]; then
+    echo "missing or unreadable chat template: $CHAT_TEMPLATE_FILE" >&2
+    exit 1
+fi
+
+# --- speculative decoding (MTP draft head) --------------------------------
+SPEC_ARGS=()
+if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then
+    SPEC_ARGS=(
+        --spec-type "$SPEC_TYPE"
+        --spec-draft-n-max "$SPEC_DRAFT_N_MAX"
+        --spec-draft-n-min "$SPEC_DRAFT_N_MIN"
+        --spec-draft-p-min "$SPEC_DRAFT_P_MIN"
+        --spec-draft-p-split "$SPEC_DRAFT_P_SPLIT"
+    )
+    if [[ "$SPEC_TYPE" == *draft-mtp* ]]; then
+        DRAFT_MODEL="$MODEL_DIR/$DRAFT_MODEL_FILE"
+        if [[ ! -r "$DRAFT_MODEL" ]]; then
+            echo "missing or unreadable draft model: $DRAFT_MODEL" >&2
+            exit 1
+        fi
+        SPEC_ARGS+=(
+            --spec-draft-model "$DRAFT_MODEL"
+            --spec-draft-ngl "$SPEC_DRAFT_NGL"
+            --spec-draft-type-k "$SPEC_DRAFT_TYPE_K"
+            --spec-draft-type-v "$SPEC_DRAFT_TYPE_V"
+            --spec-draft-poll "$SPEC_DRAFT_POLL"
+            --spec-draft-poll-batch "$SPEC_DRAFT_POLL_BATCH"
+        )
+        if [[ -n "$SPEC_DRAFT_DEVICE" ]]; then
+            SPEC_ARGS+=(--spec-draft-device "$SPEC_DRAFT_DEVICE")
+        fi
+    fi
+fi
+
+# --- template + placement -------------------------------------------------
+TEMPLATE_ARGS=()
+if [[ -n "$CHAT_TEMPLATE_FILE" ]]; then
+    TEMPLATE_ARGS+=(--chat-template-file "$CHAT_TEMPLATE_FILE")
+    TEMPLATE_ARGS+=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS")
+fi
+
+PLACEMENT_ARGS=(--device "$DEVICES" --split-mode "$SPLIT_MODE")
+if [[ -n "$TENSOR_SPLIT" ]]; then
+    PLACEMENT_ARGS+=(--tensor-split "$TENSOR_SPLIT")
+fi
+
+MMPROJ_ARGS=()
+if [[ "$NO_MMPROJ" == "1" ]]; then
+    MMPROJ_ARGS=(--no-mmproj)
+fi
+
+exec "$LLAMA_BIN" \
+    --model "$MODEL" \
+    --alias "$ALIAS" \
+    --host "$BIND_ADDR" \
+    --port "$LISTEN_PORT" \
+    --jinja \
+    --gpu-layers "$NGL" \
+    --fit "$FIT" \
+    --parallel "$PARALLEL" \
+    --ctx-size "$CTX_SIZE" \
+    --batch-size "$BATCH_SIZE" \
+    --ubatch-size "$UBATCH_SIZE" \
+    --flash-attn "$FLASH_ATTN" \
+    --cache-type-k "$CACHE_TYPE_K" \
+    --cache-type-v "$CACHE_TYPE_V" \
+    --threads "$NUM_THREADS" \
+    --threads-batch "$THREAD_BATCH" \
+    --ctx-checkpoints "$CTX_CHECKPOINTS" \
+    --cache-ram "$CACHE_RAM" \
+    --cache-prompt \
+    --metrics \
+    --temp "$TEMPERATURE" \
+    --top-p "$TOP_P" \
+    --top-k "$TOP_K" \
+    --min-p "$MIN_P" \
+    --presence-penalty "$PRESENCE_PENALTY" \
+    --reasoning "$REASONING" \
+    --reasoning-format "$REASONING_FORMAT" \
+    --reasoning-budget "$REASONING_BUDGET" \
+    "${TEMPLATE_ARGS[@]}" \
+    "${PLACEMENT_ARGS[@]}" \
+    "${MMPROJ_ARGS[@]}" \
+    "${SPEC_ARGS[@]}"

scripts/serve-frank-small.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-# Serve batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 as
-# frank-small via FreeToken on Dankserver (RTX 3090, 60 GB RAM).
-#
-# Runtime: FreeToken v0.1.2 (edge-native MoE serving engine).
-#   - Experts live in host RAM; an LRU cache of expert slots is resident
-#     on the GPU. Misses are streamed over PCIe.
-#   - Paged KV cache (radix prefix sharing).  Only the 10 full-attention
-#     layers page KV (~20 KiB/token); the 30 GDN/linear-attention layers
-#     hold constant-size per-sequence state (mamba slots).
-#   - OpenAI-compatible API (/v1/chat/completions, /v1/models).
-#
-# Model: batsclamp/Huihui-Qwen3.6-35B-A3B-abliterated-FP8 — Huihui
-# abliterated finetune of Qwen3.6-35B-A3B in the same native modelopt
-# block-fp8 format (e4m3, 128x128 weight_block_size, un-fused per-expert
-# layout) as the official FP8 checkpoint this service ran before; the
-# FreeToken loader path is unchanged.  40 layers (10 attention + 30
-# GDN), 256 experts / 8 used, 2 KV heads × 256 dim.  ~35 GB on disk.
-# The checkpoint ships the fixed Qwen3.6 chat template; FreeToken reads
-# it from the model directory.
-#
-# Pool sizing: the GPU budget is split MoE-first between expert residency
-# and KV pages.  --moe-cache-size pins the number of resident expert
-# slots; --num-tokens pins the KV pool capacity.  Live resize is possible
-# via `ft ctl cache --kv N --moe M` (grows KV by shrinking MoE; shrinking
-# KV requires a restart due to the swap-during-rebuild memory cost).
-#
-# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE
-# overrides that path.
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
-if [[ -f "$ENV_FILE" ]]; then
-    set -a
-    # shellcheck disable=SC1090
-    source "$ENV_FILE"
-    set +a
-fi
-
-# --- defaults (used when the env file does not set them) -----------------
-: "${MODEL_DIR:=/home/dank/models/huihui-qwen36-35b-a3b-abliterated-fp8}"
-: "${ALIAS:=frank-small}"
-: "${BIND_ADDR:=0.0.0.0}"
-: "${LISTEN_PORT:=7770}"
-: "${FT_VENV:=/home/dank/freetoken/.venv}"
-: "${CUDA_HOME_DIR:=${FT_VENV}/lib/python3.12/site-packages/nvidia/cu13}"
-
-# Pool sizing (expert residency vs KV cache tradeoff).
-: "${MOE_CACHE_SIZE:=2048}"
-: "${NUM_TOKENS:=453005}"
-: "${MAX_RUNNING_REQUESTS:=4}"
-
-# Paths
-FT_BIN="${FT_VENV}/bin/ft"
-MODEL="${MODEL_DIR}"
-
-if [[ ! -x "$FT_BIN" ]]; then
-    echo "missing FreeToken CLI: $FT_BIN" >&2
-    exit 1
-fi
-if [[ ! -d "$MODEL" ]]; then
-    echo "missing model directory: $MODEL" >&2
-    exit 1
-fi
-
-export PATH="${FT_VENV}/bin:${PATH}"
-export CUDA_HOME="${CUDA_HOME_DIR}"
-
-exec "$FT_BIN" serve \
-    --model "$MODEL" \
-    --served-model-name "$ALIAS" \
-    --host "$BIND_ADDR" \
-    --port "$LISTEN_PORT" \
-    --moe-cache-size "$MOE_CACHE_SIZE" \
-    --num-tokens "$NUM_TOKENS" \
-    --max-running-requests "$MAX_RUNNING_REQUESTS"

scripts/test-changed.sh

@@ -179,7 +179,7 @@ for path in "${CHANGED[@]}"; do
         scripts/mount-entropy.sh)
             add_test tests/mount_entropy.bats
             ;;
-        scripts/frank-small.env|scripts/serve-frank-small.sh)
+        scripts/frank-medium.env|scripts/serve-frank-medium.sh)
             add_test tests/llm_units.bats
             add_test tests/llm_secrets.bats
             add_test tests/scripts_syntax.bats

tests/changed_tests.bats

@@ -22,7 +22,7 @@ load helpers
 }
 
 @test "changed-test wrapper maps a service script to its focused tests" {
-    run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/serve-frank-small.sh
+    run bash "$REPO_ROOT/scripts/test-changed.sh" --list --paths scripts/serve-frank-medium.sh
     [ "$status" -eq 0 ]
     [[ "$output" == *"tests/scripts_syntax.bats"* ]]
     [[ "$output" == *"tests/llm_units.bats"* ]]

tests/deployment_flow.bats

@@ -69,11 +69,11 @@ load helpers
 
 @test "remote runner has separate handlers for both inference services" {
     grep -q 'deploy_llm_large' "$REPO_ROOT/scripts/deploy-remote.sh"
-    grep -q 'deploy_llm_small' "$REPO_ROOT/scripts/deploy-remote.sh"
+    grep -q 'deploy_llm_medium' "$REPO_ROOT/scripts/deploy-remote.sh"
 }
 
 @test "inference restarts are absent from the top-level dispatcher" {
-    ! grep -qE 'systemctl .*restart (frank-small|frank-large|frank-npu)' "$REPO_ROOT/scripts/deploy.sh"
+    ! grep -qE 'systemctl .*restart (frank-medium|frank-large|frank-npu)' "$REPO_ROOT/scripts/deploy.sh"
 }
 
 @test "remote runner health-checks an inference endpoint" {

tests/deployment_scoping.bats

@@ -29,23 +29,29 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
 }
 
 @test "shared chat-template changes map only to the service that reads it" {
-    # frank-large runs halogen-flash-server, which owns its own chat handling;
+    # frank-medium reads the tracked template only when its env file points at
-    # frank-small is the only consumer of the tracked template now, so a
+    # it (the GGUF's embedded template is the default); frank-large runs
-    # template edit is a no-op on Dankclaw.
+    # halogen-flash-server, which owns its own chat handling. So a template
+    # edit is a no-op on Dankclaw and selects frank-medium on Dankserver.
     run python3 "$PLANNER" --target dankclaw --paths scripts/chat-template.jinja
     [ "$status" -eq 0 ]
     [ -z "$output" ]
 
     run python3 "$PLANNER" --target dankserver --paths scripts/chat-template.jinja
     [ "$status" -eq 0 ]
-    [ "$output" = "frank-small" ]
+    [ "$output" = "frank-medium" ]
 }
 
-@test "deployment planner maps small-model changes to the small model only" {
+@test "deployment planner maps the dankserver model to frank-medium only" {
     run python3 "$PLANNER" --target dankserver --paths \
-        scripts/frank-small.service
+        scripts/frank-medium.service
     [ "$status" -eq 0 ]
-    [ "$output" = "frank-small" ]
+    [ "$output" = "frank-medium" ]
+}
+
+@test "the retired small endpoint is absent from the deployment inventory" {
+    ! grep -q '"frank-small"' "$PLANNER"
+    ! grep -qE 'frank-small\) deploy|dankserver:frank-small' "$REMOTE_DEPLOY" "$REPO_ROOT/scripts/deploy.sh"
 }
 
 @test "documentation-only changes plan no runtime deployment" {
@@ -157,7 +163,7 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
     # The runner may restart inference only from the service-dispatch path. It
     # must not contain a host-wide restart loop.
     grep -q 'deploy_llm_large' "$REMOTE_DEPLOY"
-    grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
+    grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
     grep -q 'deploy_service' "$REMOTE_DEPLOY"
     ! grep -qE 'for .*frank-(small|large|npu)-?' "$REMOTE_DEPLOY"
 }