frank-small: Q5_K_M -> Q6_K rung at the held 2x200704 f16-KV shape
The freed third slot funded exact V; the remaining ~3.5 GiB of honest nvidia-smi free (3,571, not the 4,046 total-minus-used I quoted - the driver holds ~450 MiB of reserve) funds the +875 MiB of Q6_K weights. Projected landing ~2.7 GiB free, inside the 2-3 GiB floor; measure on boot, step the pool one 12,288-token increment if it lands under 2 GiB. - payload dir MiMo-Q6K/ made self-contained (mmproj hardlinked from MiMo-Q5KM/, link-count-2 verified) - env/serve/deployer/bats repointed; download script now fetches and verifies Q6_K (7,793,710,624 B, ef96d05a...) with Q5_K_M documented as the rollback; payload sha verified live on the host before deploy - seam green 14/14 against the stub engine
This commit is contained in:
Frank
@@ -1198,12 +1198,12 @@ deploy_llm_large() {
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: the Q5_K_M GGUF ships no `v.*`
|
||||
# The projector IS the vision capability here: the Q6_K GGUF ships no `v.*`
|
||||
# tensors, so a deploy that starts without this file silently ships a
|
||||
# text-only endpoint. Checked unconditionally, like frank-large's.
|
||||
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
|
||||
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-small payload for llama.cpp on the RTX 3090:
|
||||
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M plus its vision projector
|
||||
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K plus its vision projector
|
||||
# (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so
|
||||
# without this file the endpoint is text-only by construction). Public repo —
|
||||
# no auth header, and never a token on the command line (it is world-readable
|
||||
@@ -11,10 +11,11 @@
|
||||
# then byte count and the Hub LFS sha256. Idempotent: complete files are
|
||||
# skipped.
|
||||
#
|
||||
# The Q6_K file this endpoint served until 2026-10-01 night stays on disk as
|
||||
# the rollback target (7793710624 B, LFS oid
|
||||
# ef96d05a2ddf2cbb450d1af1ac3860ec769d3575bafa70692ee5609bda3fad7d) — re-fetch
|
||||
# it from the same repo with the same recipe if it is ever needed again.
|
||||
# 2026-10-03: the rung is Q6_K again — the freed third slot funds exact f16 V
|
||||
# AND the higher quant. The Q5_K_M file this script served from 2026-10-02
|
||||
# stays on disk at /home/dank/models/MiMo-Q5KM/ (6876124704 B, LFS oid
|
||||
# 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f) as the
|
||||
# rollback target; re-fetch it from the same repo with the same recipe.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
|
||||
@@ -31,11 +32,11 @@ for arg in "$@"; do
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/MiMo-Q5KM}"
|
||||
DEST="${DEST:-/home/dank/models/MiMo-Q6K}"
|
||||
|
||||
# name bytes sha256
|
||||
FILES=(
|
||||
"MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf 6876124704 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f"
|
||||
"MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf 7793710624 ef96d05a2ddf2cbb450d1af1ac3860ec769d3575bafa70692ee5609bda3fad7d"
|
||||
"mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf 918166048 ff348f3180a63188aa7285db85f550fe38acb61dd013c599eb8bad08d2cc2576"
|
||||
)
|
||||
|
||||
|
||||
+15
-6
@@ -1,5 +1,12 @@
|
||||
# frank-small — llama.cpp (master, CUDA sm_86) serving
|
||||
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771.
|
||||
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771.
|
||||
#
|
||||
# 2026-10-03 (Frank, second move): the rung went Q5_K_M -> Q6_K with the pool
|
||||
# HELD at 401,408. The freed third slot funded exact f16 V first; the ~3.5 GiB
|
||||
# still free funds the +875 MiB of weights (file delta 7,793,710,624 -
|
||||
# 6,876,124,704). Projected landing ~2.7 GiB free — inside the 2-3 GiB floor,
|
||||
# so MEASURE after the boot; if it lands under 2 GiB, step the pool down one
|
||||
# 12,288-token (384 MiB) increment.
|
||||
#
|
||||
# 2026-10-03 (Frank): TWO streams instead of three, each one WIDE — 200,704
|
||||
# tokens per conversation, above the 200,700 the harness compacts at — and the
|
||||
@@ -38,12 +45,14 @@
|
||||
# and the third slot is gone. The flags stay explicit so the choice is visible
|
||||
# in the argv either way.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
|
||||
MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf
|
||||
MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf
|
||||
|
||||
# Vision: the Q5_K_M GGUF ships no `v.*` tensors, so the projector file IS the
|
||||
# vision capability (sha256 ff348f31..., 918,166,048 B, fetched by
|
||||
# scripts/download-frank-small.sh). Empty string = text-only on purpose.
|
||||
MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf
|
||||
# Vision: the Q6_K GGUF ships no `v.*` tensors either, so the projector file
|
||||
# IS the vision capability (sha256 ff348f31..., 918,166,048 B, fetched by
|
||||
# scripts/download-frank-small.sh; hardlinked into this dir from MiMo-Q5KM/,
|
||||
# same bytes — the Q6_K dir is self-contained). Empty string = text-only on
|
||||
# purpose.
|
||||
MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf
|
||||
|
||||
# Fleet endpoint: same port and alias as every frank-small before it, so the
|
||||
# harness models.yml id and the homepage probe do not move.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on
|
||||
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on
|
||||
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
||||
#
|
||||
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
|
||||
@@ -12,7 +12,7 @@
|
||||
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
|
||||
# DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
|
||||
# the DeltaNet state is per-slot fixed size. Vision comes from the repo's
|
||||
# separate projector (the Q5_K_M GGUF carries no `v.*` tensors at all), so
|
||||
# separate projector (the Q6_K GGUF carries no `v.*` tensors at all), so
|
||||
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented
|
||||
# text-only opt-out.
|
||||
#
|
||||
@@ -33,7 +33,7 @@ fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}"
|
||||
: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
@@ -54,7 +54,7 @@ fi
|
||||
: "${NGL:=99}"
|
||||
# Vision projector. No colon on purpose: an explicitly empty value means
|
||||
# "text-only, on purpose" and must NOT be replaced by the default path.
|
||||
: "${MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
|
||||
: "${MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
|
||||
|
||||
# Fail loudly rather than half-loading: a missing file is a download that
|
||||
# has not finished, and Restart=always would turn it into a crash loop that
|
||||
|
||||
Reference in New Issue
Block a user