dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-small: Q5_K_M -> Q6_K rung at the held 2x200704 f16-KV shape

Browse Source 
The freed third slot funded exact V; the remaining ~3.5 GiB of honest
nvidia-smi free (3,571, not the 4,046 total-minus-used I quoted - the
driver holds ~450 MiB of reserve) funds the +875 MiB of Q6_K weights.
Projected landing ~2.7 GiB free, inside the 2-3 GiB floor; measure on
boot, step the pool one 12,288-token increment if it lands under 2 GiB.

- payload dir MiMo-Q6K/ made self-contained (mmproj hardlinked from
  MiMo-Q5KM/, link-count-2 verified)
- env/serve/deployer/bats repointed; download script now fetches and
  verifies Q6_K (7,793,710,624 B, ef96d05a...) with Q5_K_M documented as
  the rollback; payload sha verified live on the host before deploy
- seam green 14/14 against the stub engine
This commit is contained in:
Frank
2026-10-03 11:00:19 -07:00
parent b80886e318
commit 6ec0b1a85d
5 changed files with 35 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/deploy-remote.sh
+3 -3
View File
@@ -1198,12 +1198,12 @@ deploy_llm_large() {
# #
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
# not shared: frank-large's engine runs on the gfx1151 iGPU. # not shared: frank-large's engine runs on the gfx1151 iGPU.
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
# The projector IS the vision capability here: the Q5_K_M GGUF ships no `v.*` # The projector IS the vision capability here: the Q6_K GGUF ships no `v.*`
# tensors, so a deploy that starts without this file silently ships a # tensors, so a deploy that starts without this file silently ships a
# text-only endpoint. Checked unconditionally, like frank-large's. # text-only endpoint. Checked unconditionally, like frank-large's.
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
deploy_llm_small() { deploy_llm_small() {
# Refuse rather than restart-loop: a unit whose runtime or payload is # Refuse rather than restart-loop: a unit whose runtime or payload is
scripts/download-frank-small.sh
+8 -7
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Download the frank-small payload for llama.cpp on the RTX 3090: # Download the frank-small payload for llama.cpp on the RTX 3090:
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M plus its vision projector # bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K plus its vision projector
# (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so # (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so
# without this file the endpoint is text-only by construction). Public repo — # without this file the endpoint is text-only by construction). Public repo —
# no auth header, and never a token on the command line (it is world-readable # no auth header, and never a token on the command line (it is world-readable
@@ -11,10 +11,11 @@
# then byte count and the Hub LFS sha256. Idempotent: complete files are # then byte count and the Hub LFS sha256. Idempotent: complete files are
# skipped. # skipped.
# #
# The Q6_K file this endpoint served until 2026-10-01 night stays on disk as # 2026-10-03: the rung is Q6_K again — the freed third slot funds exact f16 V
# the rollback target (7793710624 B, LFS oid # AND the higher quant. The Q5_K_M file this script served from 2026-10-02
# ef96d05a2ddf2cbb450d1af1ac3860ec769d3575bafa70692ee5609bda3fad7d) — re-fetch # stays on disk at /home/dank/models/MiMo-Q5KM/ (6876124704 B, LFS oid
# it from the same repo with the same recipe if it is ever needed again. # 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f) as the
# rollback target; re-fetch it from the same repo with the same recipe.
# #
# Usage: # Usage:
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only] # scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
@@ -31,11 +32,11 @@ for arg in "$@"; do
*) DEST="$arg" ;; *) DEST="$arg" ;;
esac esac
done done
DEST="${DEST:-/home/dank/models/MiMo-Q5KM}" DEST="${DEST:-/home/dank/models/MiMo-Q6K}"
# name bytes sha256 # name bytes sha256
FILES=( FILES=(
"MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf 6876124704 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f" "MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf 7793710624 ef96d05a2ddf2cbb450d1af1ac3860ec769d3575bafa70692ee5609bda3fad7d"
"mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf 918166048 ff348f3180a63188aa7285db85f550fe38acb61dd013c599eb8bad08d2cc2576" "mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf 918166048 ff348f3180a63188aa7285db85f550fe38acb61dd013c599eb8bad08d2cc2576"
) )
scripts/frank-small.env
+15 -6
View File
@@ -1,5 +1,12 @@
# frank-small — llama.cpp (master, CUDA sm_86) serving # frank-small — llama.cpp (master, CUDA sm_86) serving
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771. # bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771.
#
# 2026-10-03 (Frank, second move): the rung went Q5_K_M -> Q6_K with the pool
# HELD at 401,408. The freed third slot funded exact f16 V first; the ~3.5 GiB
# still free funds the +875 MiB of weights (file delta 7,793,710,624 -
# 6,876,124,704). Projected landing ~2.7 GiB free — inside the 2-3 GiB floor,
# so MEASURE after the boot; if it lands under 2 GiB, step the pool down one
# 12,288-token (384 MiB) increment.
# #
# 2026-10-03 (Frank): TWO streams instead of three, each one WIDE — 200,704 # 2026-10-03 (Frank): TWO streams instead of three, each one WIDE — 200,704
# tokens per conversation, above the 200,700 the harness compacts at — and the # tokens per conversation, above the 200,700 the harness compacts at — and the
@@ -38,12 +45,14 @@
# and the third slot is gone. The flags stay explicit so the choice is visible # and the third slot is gone. The flags stay explicit so the choice is visible
# in the argv either way. # in the argv either way.
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf
# Vision: the Q5_K_M GGUF ships no `v.*` tensors, so the projector file IS the # Vision: the Q6_K GGUF ships no `v.*` tensors either, so the projector file
# vision capability (sha256 ff348f31..., 918,166,048 B, fetched by # IS the vision capability (sha256 ff348f31..., 918,166,048 B, fetched by
# scripts/download-frank-small.sh). Empty string = text-only on purpose. # scripts/download-frank-small.sh; hardlinked into this dir from MiMo-Q5KM/,
MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf # same bytes — the Q6_K dir is self-contained). Empty string = text-only on
# purpose.
MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf
# Fleet endpoint: same port and alias as every frank-small before it, so the # Fleet endpoint: same port and alias as every frank-small before it, so the
# harness models.yml id and the homepage probe do not move. # harness models.yml id and the homepage probe do not move.
scripts/serve-frank-small.sh
+4 -4
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on # Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86). # Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
# #
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound # 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
@@ -12,7 +12,7 @@
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated # Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
# DeltaNet, native 262144 window. KV lives only in the 8 attention layers; # DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
# the DeltaNet state is per-slot fixed size. Vision comes from the repo's # the DeltaNet state is per-slot fixed size. Vision comes from the repo's
# separate projector (the Q5_K_M GGUF carries no `v.*` tensors at all), so # separate projector (the Q6_K GGUF carries no `v.*` tensors at all), so
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented # --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented
# text-only opt-out. # text-only opt-out.
# #
@@ -33,7 +33,7 @@ fi
# --- defaults (used when the env file does not set them) ------------------- # --- defaults (used when the env file does not set them) -------------------
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" : "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" : "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
: "${ALIAS:=frank-small}" : "${ALIAS:=frank-small}"
: "${BIND_ADDR:=0.0.0.0}" : "${BIND_ADDR:=0.0.0.0}"
: "${LISTEN_PORT:=7771}" : "${LISTEN_PORT:=7771}"
@@ -54,7 +54,7 @@ fi
: "${NGL:=99}" : "${NGL:=99}"
# Vision projector. No colon on purpose: an explicitly empty value means # Vision projector. No colon on purpose: an explicitly empty value means
# "text-only, on purpose" and must NOT be replaced by the default path. # "text-only, on purpose" and must NOT be replaced by the default path.
: "${MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" : "${MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
# Fail loudly rather than half-loading: a missing file is a download that # Fail loudly rather than half-loading: a missing file is a download that
# has not finished, and Restart=always would turn it into a crash loop that # has not finished, and Restart=always would turn it into a crash loop that
tests/frank_small_unit.bats
+5 -5
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env bats #!/usr/bin/env bats
# Seam: the frank-small llama.cpp root unit on dankclaw. # Seam: the frank-small llama.cpp root unit on dankclaw.
# frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves # frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M via a CUDA (sm_86) build of # bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of
# llama.cpp master, chosen because it is the only runtime here that moves K and # llama.cpp master, chosen because it is the only runtime here that moves K and
# V independently. Since 2026-10-03 it runs TWO unquantized contexts of # V independently. Since 2026-10-03 it runs TWO unquantized contexts of
# 200,704 tokens each and carries a vision projector, and the card must keep # 200,704 tokens each and carries a vision projector, and the card must keep
@@ -16,7 +16,7 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-small.env" ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf" TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf"
# Same basename as the shipped projector: the test env repoints it into $STAGE. # Same basename as the shipped projector: the test env repoints it into $STAGE.
MM_STUB="mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf" MM_STUB="mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf"
@@ -79,9 +79,9 @@ argv_value() {
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT" grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
} }
@test "the shipped env points at the llama.cpp runtime and the Q5_K_M GGUF" { @test "the shipped env points at the llama.cpp runtime and the Q6_K GGUF" {
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE" grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q5KM/$TARGET_STUB$" "$ENV_FILE" grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q6K/$TARGET_STUB$" "$ENV_FILE"
} }
@test "the deployer gates on exactly the payload paths the env file ships" { @test "the deployer gates on exactly the payload paths the env file ships" {
@@ -169,7 +169,7 @@ argv_value() {
} }
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" { @test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
# The Q5_K_M GGUF carries no `v.*` tensors, so the projector argument IS # The Q6_K GGUF carries no `v.*` tensors, so the projector argument IS
# the vision capability: no flag, no eyes. # the vision capability: no flag, no eyes.
serve_argv serve_argv
[ "$status" -eq 0 ] [ "$status" -eq 0 ]