diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 72c2fa8..91edada 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1198,12 +1198,12 @@ deploy_llm_large() { # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # not shared: frank-large's engine runs on the gfx1151 iGPU. -LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" +LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}" LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -# The projector IS the vision capability here: the Q5_K_M GGUF ships no `v.*` +# The projector IS the vision capability here: the Q6_K GGUF ships no `v.*` # tensors, so a deploy that starts without this file silently ships a # text-only endpoint. Checked unconditionally, like frank-large's. -LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" +LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" deploy_llm_small() { # Refuse rather than restart-loop: a unit whose runtime or payload is diff --git a/scripts/download-frank-small.sh b/scripts/download-frank-small.sh index 3831ad0..1d69ead 100755 --- a/scripts/download-frank-small.sh +++ b/scripts/download-frank-small.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Download the frank-small payload for llama.cpp on the RTX 3090: -# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M plus its vision projector +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K plus its vision projector # (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so # without this file the endpoint is text-only by construction). Public repo — # no auth header, and never a token on the command line (it is world-readable @@ -11,10 +11,11 @@ # then byte count and the Hub LFS sha256. Idempotent: complete files are # skipped. # -# The Q6_K file this endpoint served until 2026-10-01 night stays on disk as -# the rollback target (7793710624 B, LFS oid -# ef96d05a2ddf2cbb450d1af1ac3860ec769d3575bafa70692ee5609bda3fad7d) — re-fetch -# it from the same repo with the same recipe if it is ever needed again. +# 2026-10-03: the rung is Q6_K again — the freed third slot funds exact f16 V +# AND the higher quant. The Q5_K_M file this script served from 2026-10-02 +# stays on disk at /home/dank/models/MiMo-Q5KM/ (6876124704 B, LFS oid +# 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f) as the +# rollback target; re-fetch it from the same repo with the same recipe. # # Usage: # scripts/download-frank-small.sh [DEST_DIR] [--verify-only] @@ -31,11 +32,11 @@ for arg in "$@"; do *) DEST="$arg" ;; esac done -DEST="${DEST:-/home/dank/models/MiMo-Q5KM}" +DEST="${DEST:-/home/dank/models/MiMo-Q6K}" # name bytes sha256 FILES=( - "MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf 6876124704 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f" + "MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf 7793710624 ef96d05a2ddf2cbb450d1af1ac3860ec769d3575bafa70692ee5609bda3fad7d" "mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf 918166048 ff348f3180a63188aa7285db85f550fe38acb61dd013c599eb8bad08d2cc2576" ) diff --git a/scripts/frank-small.env b/scripts/frank-small.env index da5b7f3..7c28b51 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -1,5 +1,12 @@ # frank-small — llama.cpp (master, CUDA sm_86) serving -# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771. +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K on the RTX 3090, :7771. +# +# 2026-10-03 (Frank, second move): the rung went Q5_K_M -> Q6_K with the pool +# HELD at 401,408. The freed third slot funded exact f16 V first; the ~3.5 GiB +# still free funds the +875 MiB of weights (file delta 7,793,710,624 - +# 6,876,124,704). Projected landing ~2.7 GiB free — inside the 2-3 GiB floor, +# so MEASURE after the boot; if it lands under 2 GiB, step the pool down one +# 12,288-token (384 MiB) increment. # # 2026-10-03 (Frank): TWO streams instead of three, each one WIDE — 200,704 # tokens per conversation, above the 200,700 the harness compacts at — and the @@ -38,12 +45,14 @@ # and the third slot is gone. The flags stay explicit so the choice is visible # in the argv either way. LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server -MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf +MODEL_FILE=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf -# Vision: the Q5_K_M GGUF ships no `v.*` tensors, so the projector file IS the -# vision capability (sha256 ff348f31..., 918,166,048 B, fetched by -# scripts/download-frank-small.sh). Empty string = text-only on purpose. -MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf +# Vision: the Q6_K GGUF ships no `v.*` tensors either, so the projector file +# IS the vision capability (sha256 ff348f31..., 918,166,048 B, fetched by +# scripts/download-frank-small.sh; hardlinked into this dir from MiMo-Q5KM/, +# same bytes — the Q6_K dir is self-contained). Empty string = text-only on +# purpose. +MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf # Fleet endpoint: same port and alias as every frank-small before it, so the # harness models.yml id and the homepage probe do not move. diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 9069b97..7e7f907 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q5_K_M) as frank-small on +# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on # Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86). # # 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound @@ -12,7 +12,7 @@ # Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated # DeltaNet, native 262144 window. KV lives only in the 8 attention layers; # the DeltaNet state is per-slot fixed size. Vision comes from the repo's -# separate projector (the Q5_K_M GGUF carries no `v.*` tensors at all), so +# separate projector (the Q6_K GGUF carries no `v.*` tensors at all), so # --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented # text-only opt-out. # @@ -33,7 +33,7 @@ fi # --- defaults (used when the env file does not set them) ------------------- : "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -: "${MODEL_FILE:=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" +: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}" : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" @@ -54,7 +54,7 @@ fi : "${NGL:=99}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. -: "${MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" +: "${MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" # Fail loudly rather than half-loading: a missing file is a download that # has not finished, and Restart=always would turn it into a crash loop that diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 41a2156..0b7817c 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -1,7 +1,7 @@ #!/usr/bin/env bats # Seam: the frank-small llama.cpp root unit on dankclaw. # frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves -# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M via a CUDA (sm_86) build of +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of # llama.cpp master, chosen because it is the only runtime here that moves K and # V independently. Since 2026-10-03 it runs TWO unquantized contexts of # 200,704 tokens each and carries a vision projector, and the card must keep @@ -16,7 +16,7 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh" ENV_FILE="$REPO_ROOT/scripts/frank-small.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" -TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf" +TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf" # Same basename as the shipped projector: the test env repoints it into $STAGE. MM_STUB="mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf" @@ -79,9 +79,9 @@ argv_value() { grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT" } -@test "the shipped env points at the llama.cpp runtime and the Q5_K_M GGUF" { +@test "the shipped env points at the llama.cpp runtime and the Q6_K GGUF" { grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE" - grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q5KM/$TARGET_STUB$" "$ENV_FILE" + grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q6K/$TARGET_STUB$" "$ENV_FILE" } @test "the deployer gates on exactly the payload paths the env file ships" { @@ -169,7 +169,7 @@ argv_value() { } @test "vision ships by default and MMPROJ_FILE empty opts out of the image path" { - # The Q5_K_M GGUF carries no `v.*` tensors, so the projector argument IS + # The Q6_K GGUF carries no `v.*` tensors, so the projector argument IS # the vision capability: no flag, no eyes. serve_argv [ "$status" -eq 0 ]