frank-small: Q5_K_M GGUF on llama.cpp, two full native windows

One quant step down from Q6_K on the same CUDA build and the same shape:
2 x 262,144 with K f16 + V q8_0. The 0.86 GiB freed from the weights is
headroom, not a new window — the native 262,144 cap is already reached,
and f16 V would still cost 3.96 GiB more than the pool can carry.

scripts/deploy-remote.sh

@@ -1186,7 +1186,7 @@ deploy_llm_large() {
 #
 # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
 # not shared: frank-large's engine runs on the gfx1151 iGPU.
-LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
+LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
 LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
 
 deploy_llm_small() {
@@ -1202,7 +1202,7 @@ deploy_llm_small() {
     fi
     if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then
         echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2
-        echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q6_K first;" >&2
+        echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q5_K_M first;" >&2
         echo "frank-small: refusing to start a unit that cannot load." >&2
         return 1
     fi

scripts/frank-small.env

@@ -1,24 +1,30 @@
 # frank-small — llama.cpp (master, CUDA sm_86) serving
-# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771.
+# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771.
 #
-# 2026-10-01 night (Frank): swapped from SGLang INT8-W8A16-AutoRound to the
-# GGUF path specifically for the ASYMMETRIC KV contract — K unquantized
-# (f16), V q8_0 — which only llama.cpp exposes per-tensor. Q6_K is 7.79 GiB
-# of weights (above the fleet's 6-bit accuracy floor).
+# 2026-10-01 night (Frank): one quant step down from the Q6_K file this
+# endpoint ran for a few hours, on the same runtime and the same shape — the
+# llama.cpp GGUF path is kept for the ASYMMETRIC KV contract (K unquantized
+# f16, V q8_0) that no other runtime on this host exposes per-tensor, and the
+# freed 0.86 GiB of weights is what keeps the two full windows bootable with
+# margin for CUDA graphs.
 #
 # KV arithmetic: attention KV exists only in the 8 full-attention layers
-# (4 KV heads x 256 dim). K f16 = 16 KiB/token; V q8_0 ~ 8.5 KiB/token
-# (34 B per 32-element block) => ~24.5 KiB/token. Two full native windows
-# (2 x 262,144 = 524,288 tokens) cost ~12.6 GiB; weights 7.8 + KV 12.6 +
-# compute buffers and CUDA graphs ~1.5-2 => ~22-23 GiB against the card's
-# 24 GiB — this is the "two contexts with room for checkpoints" fit. If the
-# graph capture OOMs, lower CTX_SIZE (262144 = two 131072 windows) rather
+# (4 KV heads x 256 head_dim). K f16 = 16 KiB/token; V q8_0 = 8.5 KiB/token
+# (34 B per 32-element block) => 24.5 KiB/token. Two full native windows
+# (2 x 262,144 = 524,288 tokens) cost 12.5 GiB; weights 6.40 + KV 12.5 + the
+# fixed Gated DeltaNet state (~25 MiB/slot) + compute buffers and CUDA graphs
+# ~1.5-2 => ~20.5 GiB against the card's 24 GiB. Q6_K measured 20.9/24.6 here,
+# which is the same arithmetic with 7.26 GiB of weights.
+#
+# Why V is not f16: full-precision V costs 16 KiB/token instead of 8.5, i.e.
+# +3.96 GiB on a 524,288-token pool — 24.5 GiB total, past the card. If the
+# graph capture OOMs, lower CTX_SIZE (262144 = two 131,072 windows) rather
 # than quantizing K.
 LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
-MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf
+MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf
 
-# Fleet endpoint: same port and alias as the SGLang era; the harness
-# models.yml id does not move.
+# Fleet endpoint: same port and alias as every frank-small before it, so the
+# harness models.yml id and the homepage probe do not move.
 ALIAS=frank-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7771

scripts/frank-small.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q6_K GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small)
+Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small)
 After=network-online.target
 Wants=network-online.target
 

scripts/serve-frank-small.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on
+# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on
 # Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
 #
 # 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
@@ -32,7 +32,7 @@ fi
 
 # --- defaults (used when the env file does not set them) -------------------
 : "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
-: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
+: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
 : "${ALIAS:=frank-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7771}"