dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-small: Q5_K_M GGUF on llama.cpp, two full native windows

Browse Source 
One quant step down from Q6_K on the same CUDA build and the same shape:
2 x 262,144 with K f16 + V q8_0. The 0.86 GiB freed from the weights is
headroom, not a new window — the native 262,144 cap is already reached,
and f16 V would still cost 3.96 GiB more than the pool can carry.
This commit is contained in:
Frank
2026-10-02 00:00:17 -07:00
parent 4bb10cd20e
commit d82c5c03b7
5 changed files with 36 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/deploy-remote.sh
+2 -2
View File
@@ -1186,7 +1186,7 @@ deploy_llm_large() {
#
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
# not shared: frank-large's engine runs on the gfx1151 iGPU.
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
deploy_llm_small() {
@@ -1202,7 +1202,7 @@ deploy_llm_small() {
fi
if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then
echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2
echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q6_K first;" >&2
echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q5_K_M first;" >&2
echo "frank-small: refusing to start a unit that cannot load." >&2
return 1
fi
scripts/frank-small.env
+20 -14
View File
@@ -1,24 +1,30 @@
# frank-small — llama.cpp (master, CUDA sm_86) serving
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771.
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771.
#
# 2026-10-01 night (Frank): swapped from SGLang INT8-W8A16-AutoRound to the
# GGUF path specifically for the ASYMMETRIC KV contract — K unquantized
# (f16), V q8_0 — which only llama.cpp exposes per-tensor. Q6_K is 7.79 GiB
# of weights (above the fleet's 6-bit accuracy floor).
# 2026-10-01 night (Frank): one quant step down from the Q6_K file this
# endpoint ran for a few hours, on the same runtime and the same shape — the
# llama.cpp GGUF path is kept for the ASYMMETRIC KV contract (K unquantized
# f16, V q8_0) that no other runtime on this host exposes per-tensor, and the
# freed 0.86 GiB of weights is what keeps the two full windows bootable with
# margin for CUDA graphs.
#
# KV arithmetic: attention KV exists only in the 8 full-attention layers
# (4 KV heads x 256 dim). K f16 = 16 KiB/token; V q8_0 ~ 8.5 KiB/token
# (34 B per 32-element block) => ~24.5 KiB/token. Two full native windows
# (2 x 262,144 = 524,288 tokens) cost ~12.6 GiB; weights 7.8 + KV 12.6 +
# compute buffers and CUDA graphs ~1.5-2 => ~22-23 GiB against the card's
# 24 GiB — this is the "two contexts with room for checkpoints" fit. If the
# graph capture OOMs, lower CTX_SIZE (262144 = two 131072 windows) rather
# (4 KV heads x 256 head_dim). K f16 = 16 KiB/token; V q8_0 = 8.5 KiB/token
# (34 B per 32-element block) => 24.5 KiB/token. Two full native windows
# (2 x 262,144 = 524,288 tokens) cost 12.5 GiB; weights 6.40 + KV 12.5 + the
# fixed Gated DeltaNet state (~25 MiB/slot) + compute buffers and CUDA graphs
# ~1.5-2 => ~20.5 GiB against the card's 24 GiB. Q6_K measured 20.9/24.6 here,
# which is the same arithmetic with 7.26 GiB of weights.
#
# Why V is not f16: full-precision V costs 16 KiB/token instead of 8.5, i.e.
# +3.96 GiB on a 524,288-token pool — 24.5 GiB total, past the card. If the
# graph capture OOMs, lower CTX_SIZE (262144 = two 131,072 windows) rather
# than quantizing K.
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf
MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf
# Fleet endpoint: same port and alias as the SGLang era; the harness
# models.yml id does not move.
# Fleet endpoint: same port and alias as every frank-small before it, so the
# harness models.yml id and the homepage probe do not move.
ALIAS=frank-small
BIND_ADDR=0.0.0.0
LISTEN_PORT=7771
scripts/frank-small.service
+1 -1
View File
@@ -1,5 +1,5 @@
[Unit]
Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q6_K GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small)
Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small)
After=network-online.target
Wants=network-online.target
scripts/serve-frank-small.sh
+2 -2
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
#
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
@@ -32,7 +32,7 @@ fi
# --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
: "${ALIAS:=frank-small}"
: "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}"
tests/frank_small_unit.bats
+11 -7
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bats
# Seam: the frank-small llama.cpp root unit on dankclaw.
# frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M via a CUDA (sm_86) build of
# llama.cpp master, chosen for the ASYMMETRIC KV Frank asked for: K f16
# (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. This
# seam runs the serve script against a stub engine and asserts on the argv
@@ -14,7 +14,7 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf"
TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf"
setup() {
STAGE="$(mktemp -d)"
@@ -72,9 +72,9 @@ argv_value() {
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
}
@test "the shipped env points at the llama.cpp runtime and the Q6_K GGUF" {
@test "the shipped env points at the llama.cpp runtime and the Q5_K_M GGUF" {
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q6K/$TARGET_STUB$" "$ENV_FILE"
grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q5KM/$TARGET_STUB$" "$ENV_FILE"
}
@test "the shipped env serves the fleet shape: two full native windows" {
@@ -105,12 +105,16 @@ argv_value() {
@test "the KV budget fits the 3090 with room for checkpoints" {
# K f16 (16 KiB/token) + V q8_0 (~8.5, budget 25 with slack) over the
# whole pool, plus 7.8 GiB weights and ~2 GiB of compute buffers and
# whole pool, plus 6.4 GiB weights and ~2 GiB of compute buffers and
# CUDA graphs, must stay under the card's 24 GiB. This is arithmetic,
# not a boot log: change CTX_SIZE and it bites before deploy does.
# not a boot log: change CTX_SIZE and it bites before deploy does. The
# same arithmetic with f16 V (16 KiB/token, so 32+2 overhead per token)
# lands past the card, which is why the contract is asymmetric.
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
kv_gib=$((ctx * 25 / 1048576))
[ $((kv_gib + 8 + 2)) -le 24 ]
[ $((kv_gib + 7 + 2)) -le 24 ]
f16v_gib=$((ctx * 34 / 1048576))
[ $((f16v_gib + 7 + 2)) -gt 24 ]
}
@test "no speculative draft ships by default" {