From d82c5c03b742a70b58d58fcca13d41689d3479e8 Mon Sep 17 00:00:00 2001 From: Frank Date: Fri, 2 Oct 2026 00:00:17 -0700 Subject: [PATCH] frank-small: Q5_K_M GGUF on llama.cpp, two full native windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One quant step down from Q6_K on the same CUDA build and the same shape: 2 x 262,144 with K f16 + V q8_0. The 0.86 GiB freed from the weights is headroom, not a new window — the native 262,144 cap is already reached, and f16 V would still cost 3.96 GiB more than the pool can carry. --- scripts/deploy-remote.sh | 4 ++-- scripts/frank-small.env | 34 ++++++++++++++++++++-------------- scripts/frank-small.service | 2 +- scripts/serve-frank-small.sh | 4 ++-- tests/frank_small_unit.bats | 18 +++++++++++------- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 5dd493e..e0b9bf9 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1186,7 +1186,7 @@ deploy_llm_large() { # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # not shared: frank-large's engine runs on the gfx1151 iGPU. -LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}" +LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" deploy_llm_small() { @@ -1202,7 +1202,7 @@ deploy_llm_small() { fi if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2 - echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q6_K first;" >&2 + echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q5_K_M first;" >&2 echo "frank-small: refusing to start a unit that cannot load." >&2 return 1 fi diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 362932c..70c3272 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -1,24 +1,30 @@ # frank-small — llama.cpp (master, CUDA sm_86) serving -# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771. +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771. # -# 2026-10-01 night (Frank): swapped from SGLang INT8-W8A16-AutoRound to the -# GGUF path specifically for the ASYMMETRIC KV contract — K unquantized -# (f16), V q8_0 — which only llama.cpp exposes per-tensor. Q6_K is 7.79 GiB -# of weights (above the fleet's 6-bit accuracy floor). +# 2026-10-01 night (Frank): one quant step down from the Q6_K file this +# endpoint ran for a few hours, on the same runtime and the same shape — the +# llama.cpp GGUF path is kept for the ASYMMETRIC KV contract (K unquantized +# f16, V q8_0) that no other runtime on this host exposes per-tensor, and the +# freed 0.86 GiB of weights is what keeps the two full windows bootable with +# margin for CUDA graphs. # # KV arithmetic: attention KV exists only in the 8 full-attention layers -# (4 KV heads x 256 dim). K f16 = 16 KiB/token; V q8_0 ~ 8.5 KiB/token -# (34 B per 32-element block) => ~24.5 KiB/token. Two full native windows -# (2 x 262,144 = 524,288 tokens) cost ~12.6 GiB; weights 7.8 + KV 12.6 + -# compute buffers and CUDA graphs ~1.5-2 => ~22-23 GiB against the card's -# 24 GiB — this is the "two contexts with room for checkpoints" fit. If the -# graph capture OOMs, lower CTX_SIZE (262144 = two 131072 windows) rather +# (4 KV heads x 256 head_dim). K f16 = 16 KiB/token; V q8_0 = 8.5 KiB/token +# (34 B per 32-element block) => 24.5 KiB/token. Two full native windows +# (2 x 262,144 = 524,288 tokens) cost 12.5 GiB; weights 6.40 + KV 12.5 + the +# fixed Gated DeltaNet state (~25 MiB/slot) + compute buffers and CUDA graphs +# ~1.5-2 => ~20.5 GiB against the card's 24 GiB. Q6_K measured 20.9/24.6 here, +# which is the same arithmetic with 7.26 GiB of weights. +# +# Why V is not f16: full-precision V costs 16 KiB/token instead of 8.5, i.e. +# +3.96 GiB on a 524,288-token pool — 24.5 GiB total, past the card. If the +# graph capture OOMs, lower CTX_SIZE (262144 = two 131,072 windows) rather # than quantizing K. LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server -MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf +MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf -# Fleet endpoint: same port and alias as the SGLang era; the harness -# models.yml id does not move. +# Fleet endpoint: same port and alias as every frank-small before it, so the +# harness models.yml id and the homepage probe do not move. ALIAS=frank-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 diff --git a/scripts/frank-small.service b/scripts/frank-small.service index ac0807f..af9062e 100644 --- a/scripts/frank-small.service +++ b/scripts/frank-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q6_K GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small) +Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index bef244c..ee13c79 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on +# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on # Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86). # # 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound @@ -32,7 +32,7 @@ fi # --- defaults (used when the env file does not set them) ------------------- : "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}" +: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 488f11c..2276085 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats # Seam: the frank-small llama.cpp root unit on dankclaw. # frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves -# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M via a CUDA (sm_86) build of # llama.cpp master, chosen for the ASYMMETRIC KV Frank asked for: K f16 # (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. This # seam runs the serve script against a stub engine and asserts on the argv @@ -14,7 +14,7 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh" ENV_FILE="$REPO_ROOT/scripts/frank-small.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" -TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf" +TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf" setup() { STAGE="$(mktemp -d)" @@ -72,9 +72,9 @@ argv_value() { grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT" } -@test "the shipped env points at the llama.cpp runtime and the Q6_K GGUF" { +@test "the shipped env points at the llama.cpp runtime and the Q5_K_M GGUF" { grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE" - grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q6K/$TARGET_STUB$" "$ENV_FILE" + grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q5KM/$TARGET_STUB$" "$ENV_FILE" } @test "the shipped env serves the fleet shape: two full native windows" { @@ -105,12 +105,16 @@ argv_value() { @test "the KV budget fits the 3090 with room for checkpoints" { # K f16 (16 KiB/token) + V q8_0 (~8.5, budget 25 with slack) over the - # whole pool, plus 7.8 GiB weights and ~2 GiB of compute buffers and + # whole pool, plus 6.4 GiB weights and ~2 GiB of compute buffers and # CUDA graphs, must stay under the card's 24 GiB. This is arithmetic, - # not a boot log: change CTX_SIZE and it bites before deploy does. + # not a boot log: change CTX_SIZE and it bites before deploy does. The + # same arithmetic with f16 V (16 KiB/token, so 32+2 overhead per token) + # lands past the card, which is why the contract is asymmetric. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 25 / 1048576)) - [ $((kv_gib + 8 + 2)) -le 24 ] + [ $((kv_gib + 7 + 2)) -le 24 ] + f16v_gib=$((ctx * 34 / 1048576)) + [ $((f16v_gib + 7 + 2)) -gt 24 ] } @test "no speculative draft ships by default" {