frank-small: three contexts, vision on, 2-3 GiB of the 3090 left free

CTX_SIZE 589824 / PARALLEL 3 -> three independent 196,608-token conversations
(a mamba hybrid cannot share one KV cache, so per-slot window = pool/slots;
3 x 262,144 would need 18.4 GiB of KV alone and busts the card).

Vision comes back via bartowski's mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf
(918,166,048 B, sha256 ff348f31..., fetched + verified by
scripts/download-frank-small.sh): the Q5_K_M GGUF ships no v.* tensors, so
--mmproj IS the image path. MMPROJ_FILE without a colon default so an explicit
empty value is a real text-only opt-out, and the deployer now gates on the
projector exactly as the serve script does.

Budget is measured, not guessed: the 2 x 262,144 shape used 20,190/24,576 MiB,
which prices non-KV cost at 7,390 MiB. Three slots + projector land near
22.5 GiB, i.e. ~2 GiB spare - the bottom of the requested band, to be
confirmed by measurement and laddered if it lands outside 2-3 GiB.

scripts/deploy-remote.sh

@@ -1188,6 +1188,10 @@ deploy_llm_large() {
 # not shared: frank-large's engine runs on the gfx1151 iGPU.
 LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
 LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
+# The projector IS the vision capability here: the Q5_K_M GGUF ships no `v.*`
+# tensors, so a deploy that starts without this file silently ships a
+# text-only endpoint. Checked unconditionally, like frank-large's.
+LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
 
 deploy_llm_small() {
     # Refuse rather than restart-loop: a unit whose runtime or payload is
@@ -1200,12 +1204,14 @@ deploy_llm_small() {
         echo "frank-small: && cmake --build build-cuda --target llama-server) first." >&2
         return 1
     fi
-    if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then
-        echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2
-        echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q5_K_M first;" >&2
-        echo "frank-small: refusing to start a unit that cannot load." >&2
-        return 1
-    fi
+    for f in "$LLM_SMALL_MODEL_FILE" "$LLM_SMALL_MMPROJ"; do
+        if [[ ! -f "$f" ]]; then
+            echo "ERROR: frank-small payload file absent at $f" >&2
+            echo "frank-small: run scripts/download-frank-small.sh first;" >&2
+            echo "frank-small: refusing to start a unit that cannot load." >&2
+            return 1
+        fi
+    done
     relink_unit frank-small
     run_privileged systemctl daemon-reload
     run_privileged systemctl enable frank-small.service

scripts/download-frank-small.sh

@@ -1,7 +1,10 @@
 #!/usr/bin/env bash
-# Download bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M for frank-small
-# (llama.cpp on the RTX 3090). Public repo — no auth header, and never a token
-# on the command line (it is world-readable in /proc/<pid>/cmdline).
+# Download the frank-small payload for llama.cpp on the RTX 3090:
+# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M plus its vision projector
+# (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so
+# without this file the endpoint is text-only by construction). Public repo —
+# no auth header, and never a token on the command line (it is world-readable
+# in /proc/<pid>/cmdline).
 #
 # Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized file
 # on a 31 GiB payload); -f/--fail-with-body so an HTTP 401 is not a success,
@@ -33,6 +36,7 @@ DEST="${DEST:-/home/dank/models/MiMo-Q5KM}"
 # name bytes sha256
 FILES=(
   "MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf 6876124704 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f"
+  "mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf 918166048 ff348f3180a63188aa7285db85f550fe38acb61dd013c599eb8bad08d2cc2576"
 )
 
 mkdir -p "$DEST"

scripts/frank-small.env

@@ -1,39 +1,51 @@
 # frank-small — llama.cpp (master, CUDA sm_86) serving
 # bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771.
 #
-# 2026-10-01 night (Frank): one quant step down from the Q6_K file this
-# endpoint ran for a few hours, on the same runtime and the same shape — the
-# llama.cpp GGUF path is kept for the ASYMMETRIC KV contract (K unquantized
-# f16, V q8_0) that no other runtime on this host exposes per-tensor, and the
-# freed 0.86 GiB of weights is what keeps the two full windows bootable with
-# margin for CUDA graphs.
+# 2026-10-02 (Frank): THREE contexts, vision on, and a hard instruction to
+# leave 2-3 GiB of the card free. Both new demands cost memory, so the per-slot
+# window shrinks — see the budget below.
 #
-# KV arithmetic: attention KV exists only in the 8 full-attention layers
-# (4 KV heads x 256 head_dim). K f16 = 16 KiB/token; V q8_0 = 8.5 KiB/token
-# (34 B per 32-element block) => 24.5 KiB/token. Two full native windows
-# (2 x 262,144 = 524,288 tokens) cost 12.5 GiB; weights 6.40 + KV 12.5 + the
-# fixed Gated DeltaNet state (~25 MiB/slot) + compute buffers and CUDA graphs
-# ~1.5-2 => ~20.5 GiB against the card's 24 GiB. Q6_K measured 20.9/24.6 here,
-# which is the same arithmetic with 7.26 GiB of weights.
+# Measured on this card, this runtime, 2 x 262,144 (2026-10-02, steady state):
+# 20,190 MiB used of 24,576. KV at that pool was 524,288 tokens x 24.5 KiB =
+# 12,800 MiB, so everything that is NOT KV — weights 6,557 + Gated DeltaNet
+# state ~25/slot + compute buffers and CUDA graphs ~800 — is 7,390 MiB.
 #
-# Why V is not f16: full-precision V costs 16 KiB/token instead of 8.5, i.e.
-# +3.96 GiB on a 524,288-token pool — 24.5 GiB total, past the card. If the
-# graph capture OOMs, lower CTX_SIZE (262144 = two 131,072 windows) rather
-# than quantizing K.
+# KV arithmetic (unchanged): attention KV exists only in the 8 full-attention
+# layers (4 KV heads x 256 head_dim). K f16 = 16 KiB/token, V q8_0 =
+# 8.5 KiB/token (34 B per 32-element block) => 24.5 KiB/token. Three slots at
+# 196,608 = 589,824 tokens cost 14,113 MiB. Add the non-KV 7,390, a third
+# DeltaNet state (~25) and the vision tower + its graph (~1,030: 876 MiB of
+# mmproj weights plus buffer) => ~22.5 GiB used, ~2.0 GiB free — the bottom of
+# the requested band. Step CTX_SIZE down by 12,288 (=4096/slot) if measurement
+# says tighter, up if it says looser; the native 262,144 per slot is NOT
+# reachable three times (3 x 262,144 x 24.5 KiB = 18.4 GiB of KV alone).
+#
+# Why V stays q8_0 and K stays f16: full-precision V would add 8 KiB/token —
+# +4.5 GiB on this pool — which is what pushes the card past its 24 GiB. K is
+# the tensor the asymmetric-KV contract exists to protect.
 LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
 MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf
 
+# Vision: the Q5_K_M GGUF ships no `v.*` tensors, so the projector file IS the
+# vision capability (sha256 ff348f31..., 918,166,048 B, fetched by
+# scripts/download-frank-small.sh). Empty string = text-only on purpose.
+MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf
+
 # Fleet endpoint: same port and alias as every frank-small before it, so the
 # harness models.yml id and the homepage probe do not move.
 ALIAS=frank-small
 BIND_ADDR=0.0.0.0
 LISTEN_PORT=7771
 
-# Two contexts at the model's native window each. llama.cpp divides
-# --ctx-size across --parallel slots (n_ctx_slot = CTX_SIZE/PARALLEL), and
-# without unified KV each slot gets its own cache.
-CTX_SIZE=524288
-PARALLEL=2
+# Three independent conversations. llama.cpp divides --ctx-size across
+# --parallel slots (n_ctx_slot = CTX_SIZE/PARALLEL), and a mamba-hybrid model
+# cannot use unified KV, so each slot gets its own cache and its own window.
+# 196,608 per slot is BELOW the native 262,144: a single request longer than
+# ~196k tokens now needs compaction instead of one shot (the 2026-10-01
+# 2 x 262,144 shape was chosen to hold a 216,269-token compaction request;
+# three contexts is the trade Frank asked for).
+CTX_SIZE=589824
+PARALLEL=3
 
 # The KV contract (Frank, 2026-10-01 night): unquantized K, q8_0 V.
 # q8_0 on V is only served by the flash-attention path; the serve script

scripts/frank-small.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small)
+Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 3x196608, f16 K + q8_0 V, mmproj vision) on the RTX 3090 (dankclaw, frank-small)
 After=network-online.target
 Wants=network-online.target
 

scripts/serve-frank-small.sh

@@ -11,9 +11,10 @@
 #
 # Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
 # DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
-# the DeltaNet state is per-slot fixed size. GGUF path is TEXT-ONLY: no
-# mmproj, the in-checkpoint vision tower does not exist here (models.yml
-# declares frank-small text-only since this swap).
+# the DeltaNet state is per-slot fixed size. Vision comes from the repo's
+# separate projector (the Q5_K_M GGUF carries no `v.*` tensors at all), so
+# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented
+# text-only opt-out.
 #
 # Sampling: llama.cpp takes it per request (the fleet harness sends its
 # recipe from models.yml); greedy is the harness default.
@@ -36,20 +37,30 @@ fi
 : "${ALIAS:=frank-small}"
 : "${BIND_ADDR:=0.0.0.0}"
 : "${LISTEN_PORT:=7771}"
-# ctx-size is the POOL llama.cpp divides across slots: 524288 / 2 = TWO
-# full native 262144 windows.
-: "${CTX_SIZE:=524288}"
-: "${PARALLEL:=2}"
+# ctx-size is the POOL llama.cpp divides across slots: 589824 / 3 gives
+# 196608 per slot — three independent conversations, each with a 192k window
+# (the native 262144 no longer fits three times on a 24 GiB card; see
+# frank-small.env for the arithmetic that lands ~2.5 GiB of card free).
+: "${CTX_SIZE:=589824}"
+: "${PARALLEL:=3}"
 : "${CACHE_TYPE_K:=f16}"
 : "${CACHE_TYPE_V:=q8_0}"
 : "${NGL:=99}"
+# Vision projector. No colon on purpose: an explicitly empty value means
+# "text-only, on purpose" and must NOT be replaced by the default path.
+: "${MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
 
 # Fail loudly rather than half-loading: a missing file is a download that
-# has not finished, and Restart=on-failure would turn it into a crash loop
-# that reads like a runtime fault.
+# has not finished, and Restart=always would turn it into a crash loop that
+# reads like a runtime fault.
 for path in "$LLAMA_BIN" "$MODEL_FILE"; do
     [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
 done
+EXTRA_ARGS=()
+if [[ -n "$MMPROJ_FILE" ]]; then
+    [[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; }
+    EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
+fi
 
 # The UVM device node is created lazily by nvidia-modprobe on the first CUDA
 # open after boot; make sure it exists before the server needs it.
@@ -58,17 +69,19 @@ nvidia-modprobe -u -c=0 2>/dev/null || true
 # q8_0 on V REQUIRES flash attention (the quantized-value kernels only exist
 # on the FA path); an FA-less build silently falls back and the KV contract
 # in frank-small.env becomes a lie, so pin the flag pair here.
-exec "$LLAMA_BIN" \
-    --model "$MODEL_FILE" \
-    --alias "$ALIAS" \
-    --host "$BIND_ADDR" \
-    --port "$LISTEN_PORT" \
-    --ctx-size "$CTX_SIZE" \
-    --parallel "$PARALLEL" \
-    --n-gpu-layers "$NGL" \
-    --flash-attn on \
-    --cache-type-k "$CACHE_TYPE_K" \
-    --cache-type-v "$CACHE_TYPE_V" \
-    --jinja \
-    --reasoning auto \
+ARGS=(
+    --model "$MODEL_FILE"
+    --alias "$ALIAS"
+    --host "$BIND_ADDR"
+    --port "$LISTEN_PORT"
+    --ctx-size "$CTX_SIZE"
+    --parallel "$PARALLEL"
+    --n-gpu-layers "$NGL"
+    --flash-attn on
+    --cache-type-k "$CACHE_TYPE_K"
+    --cache-type-v "$CACHE_TYPE_V"
+    --jinja
+    --reasoning auto
     --reasoning-format deepseek
+)
+exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}