diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 996e82b..f26fb2b 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1188,6 +1188,10 @@ deploy_llm_large() { # not shared: frank-large's engine runs on the gfx1151 iGPU. LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf}" LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" +# The projector IS the vision capability here: the Q5_K_M GGUF ships no `v.*` +# tensors, so a deploy that starts without this file silently ships a +# text-only endpoint. Checked unconditionally, like frank-large's. +LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" deploy_llm_small() { # Refuse rather than restart-loop: a unit whose runtime or payload is @@ -1200,12 +1204,14 @@ deploy_llm_small() { echo "frank-small: && cmake --build build-cuda --target llama-server) first." >&2 return 1 fi - if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then - echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2 - echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q5_K_M first;" >&2 - echo "frank-small: refusing to start a unit that cannot load." >&2 - return 1 - fi + for f in "$LLM_SMALL_MODEL_FILE" "$LLM_SMALL_MMPROJ"; do + if [[ ! -f "$f" ]]; then + echo "ERROR: frank-small payload file absent at $f" >&2 + echo "frank-small: run scripts/download-frank-small.sh first;" >&2 + echo "frank-small: refusing to start a unit that cannot load." >&2 + return 1 + fi + done relink_unit frank-small run_privileged systemctl daemon-reload run_privileged systemctl enable frank-small.service diff --git a/scripts/download-frank-small.sh b/scripts/download-frank-small.sh index 46d2927..3831ad0 100755 --- a/scripts/download-frank-small.sh +++ b/scripts/download-frank-small.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash -# Download bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M for frank-small -# (llama.cpp on the RTX 3090). Public repo — no auth header, and never a token -# on the command line (it is world-readable in /proc//cmdline). +# Download the frank-small payload for llama.cpp on the RTX 3090: +# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M plus its vision projector +# (the GGUF carries no `v.*` tensors — the tower lives only in the mmproj, so +# without this file the endpoint is text-only by construction). Public repo — +# no auth header, and never a token on the command line (it is world-readable +# in /proc//cmdline). # # Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized file # on a 31 GiB payload); -f/--fail-with-body so an HTTP 401 is not a success, @@ -33,6 +36,7 @@ DEST="${DEST:-/home/dank/models/MiMo-Q5KM}" # name bytes sha256 FILES=( "MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf 6876124704 04cf46f2b5584ef922697d6c063f94879434fb86c467f1a09e52daf0cddaf35f" + "mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf 918166048 ff348f3180a63188aa7285db85f550fe38acb61dd013c599eb8bad08d2cc2576" ) mkdir -p "$DEST" diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 70c3272..872504d 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -1,39 +1,51 @@ # frank-small — llama.cpp (master, CUDA sm_86) serving # bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M on the RTX 3090, :7771. # -# 2026-10-01 night (Frank): one quant step down from the Q6_K file this -# endpoint ran for a few hours, on the same runtime and the same shape — the -# llama.cpp GGUF path is kept for the ASYMMETRIC KV contract (K unquantized -# f16, V q8_0) that no other runtime on this host exposes per-tensor, and the -# freed 0.86 GiB of weights is what keeps the two full windows bootable with -# margin for CUDA graphs. +# 2026-10-02 (Frank): THREE contexts, vision on, and a hard instruction to +# leave 2-3 GiB of the card free. Both new demands cost memory, so the per-slot +# window shrinks — see the budget below. # -# KV arithmetic: attention KV exists only in the 8 full-attention layers -# (4 KV heads x 256 head_dim). K f16 = 16 KiB/token; V q8_0 = 8.5 KiB/token -# (34 B per 32-element block) => 24.5 KiB/token. Two full native windows -# (2 x 262,144 = 524,288 tokens) cost 12.5 GiB; weights 6.40 + KV 12.5 + the -# fixed Gated DeltaNet state (~25 MiB/slot) + compute buffers and CUDA graphs -# ~1.5-2 => ~20.5 GiB against the card's 24 GiB. Q6_K measured 20.9/24.6 here, -# which is the same arithmetic with 7.26 GiB of weights. +# Measured on this card, this runtime, 2 x 262,144 (2026-10-02, steady state): +# 20,190 MiB used of 24,576. KV at that pool was 524,288 tokens x 24.5 KiB = +# 12,800 MiB, so everything that is NOT KV — weights 6,557 + Gated DeltaNet +# state ~25/slot + compute buffers and CUDA graphs ~800 — is 7,390 MiB. # -# Why V is not f16: full-precision V costs 16 KiB/token instead of 8.5, i.e. -# +3.96 GiB on a 524,288-token pool — 24.5 GiB total, past the card. If the -# graph capture OOMs, lower CTX_SIZE (262144 = two 131,072 windows) rather -# than quantizing K. +# KV arithmetic (unchanged): attention KV exists only in the 8 full-attention +# layers (4 KV heads x 256 head_dim). K f16 = 16 KiB/token, V q8_0 = +# 8.5 KiB/token (34 B per 32-element block) => 24.5 KiB/token. Three slots at +# 196,608 = 589,824 tokens cost 14,113 MiB. Add the non-KV 7,390, a third +# DeltaNet state (~25) and the vision tower + its graph (~1,030: 876 MiB of +# mmproj weights plus buffer) => ~22.5 GiB used, ~2.0 GiB free — the bottom of +# the requested band. Step CTX_SIZE down by 12,288 (=4096/slot) if measurement +# says tighter, up if it says looser; the native 262,144 per slot is NOT +# reachable three times (3 x 262,144 x 24.5 KiB = 18.4 GiB of KV alone). +# +# Why V stays q8_0 and K stays f16: full-precision V would add 8 KiB/token — +# +4.5 GiB on this pool — which is what pushes the card past its 24 GiB. K is +# the tensor the asymmetric-KV contract exists to protect. LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server MODEL_FILE=/home/dank/models/MiMo-Q5KM/MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf +# Vision: the Q5_K_M GGUF ships no `v.*` tensors, so the projector file IS the +# vision capability (sha256 ff348f31..., 918,166,048 B, fetched by +# scripts/download-frank-small.sh). Empty string = text-only on purpose. +MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf + # Fleet endpoint: same port and alias as every frank-small before it, so the # harness models.yml id and the homepage probe do not move. ALIAS=frank-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# Two contexts at the model's native window each. llama.cpp divides -# --ctx-size across --parallel slots (n_ctx_slot = CTX_SIZE/PARALLEL), and -# without unified KV each slot gets its own cache. -CTX_SIZE=524288 -PARALLEL=2 +# Three independent conversations. llama.cpp divides --ctx-size across +# --parallel slots (n_ctx_slot = CTX_SIZE/PARALLEL), and a mamba-hybrid model +# cannot use unified KV, so each slot gets its own cache and its own window. +# 196,608 per slot is BELOW the native 262,144: a single request longer than +# ~196k tokens now needs compaction instead of one shot (the 2026-10-01 +# 2 x 262,144 shape was chosen to hold a 216,269-token compaction request; +# three contexts is the trade Frank asked for). +CTX_SIZE=589824 +PARALLEL=3 # The KV contract (Frank, 2026-10-01 night): unquantized K, q8_0 V. # q8_0 on V is only served by the flash-attention path; the serve script diff --git a/scripts/frank-small.service b/scripts/frank-small.service index af9062e..5834475 100644 --- a/scripts/frank-small.service +++ b/scripts/frank-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 2x262144, f16 K + q8_0 V) on the RTX 3090 (dankclaw, frank-small) +Description=llama.cpp MiMo-V2.6-Distill-Qwen-9B Q5_K_M GGUF (qwen35 hybrid, 3x196608, f16 K + q8_0 V, mmproj vision) on the RTX 3090 (dankclaw, frank-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index ee13c79..637b232 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -11,9 +11,10 @@ # # Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated # DeltaNet, native 262144 window. KV lives only in the 8 attention layers; -# the DeltaNet state is per-slot fixed size. GGUF path is TEXT-ONLY: no -# mmproj, the in-checkpoint vision tower does not exist here (models.yml -# declares frank-small text-only since this swap). +# the DeltaNet state is per-slot fixed size. Vision comes from the repo's +# separate projector (the Q5_K_M GGUF carries no `v.*` tensors at all), so +# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented +# text-only opt-out. # # Sampling: llama.cpp takes it per request (the fleet harness sends its # recipe from models.yml); greedy is the harness default. @@ -36,20 +37,30 @@ fi : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -# ctx-size is the POOL llama.cpp divides across slots: 524288 / 2 = TWO -# full native 262144 windows. -: "${CTX_SIZE:=524288}" -: "${PARALLEL:=2}" +# ctx-size is the POOL llama.cpp divides across slots: 589824 / 3 gives +# 196608 per slot — three independent conversations, each with a 192k window +# (the native 262144 no longer fits three times on a 24 GiB card; see +# frank-small.env for the arithmetic that lands ~2.5 GiB of card free). +: "${CTX_SIZE:=589824}" +: "${PARALLEL:=3}" : "${CACHE_TYPE_K:=f16}" : "${CACHE_TYPE_V:=q8_0}" : "${NGL:=99}" +# Vision projector. No colon on purpose: an explicitly empty value means +# "text-only, on purpose" and must NOT be replaced by the default path. +: "${MMPROJ_FILE=/home/dank/models/MiMo-Q5KM/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}" # Fail loudly rather than half-loading: a missing file is a download that -# has not finished, and Restart=on-failure would turn it into a crash loop -# that reads like a runtime fault. +# has not finished, and Restart=always would turn it into a crash loop that +# reads like a runtime fault. for path in "$LLAMA_BIN" "$MODEL_FILE"; do [[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; } done +EXTRA_ARGS=() +if [[ -n "$MMPROJ_FILE" ]]; then + [[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; } + EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE") +fi # The UVM device node is created lazily by nvidia-modprobe on the first CUDA # open after boot; make sure it exists before the server needs it. @@ -58,17 +69,19 @@ nvidia-modprobe -u -c=0 2>/dev/null || true # q8_0 on V REQUIRES flash attention (the quantized-value kernels only exist # on the FA path); an FA-less build silently falls back and the KV contract # in frank-small.env becomes a lie, so pin the flag pair here. -exec "$LLAMA_BIN" \ - --model "$MODEL_FILE" \ - --alias "$ALIAS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - --ctx-size "$CTX_SIZE" \ - --parallel "$PARALLEL" \ - --n-gpu-layers "$NGL" \ - --flash-attn on \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --jinja \ - --reasoning auto \ +ARGS=( + --model "$MODEL_FILE" + --alias "$ALIAS" + --host "$BIND_ADDR" + --port "$LISTEN_PORT" + --ctx-size "$CTX_SIZE" + --parallel "$PARALLEL" + --n-gpu-layers "$NGL" + --flash-attn on + --cache-type-k "$CACHE_TYPE_K" + --cache-type-v "$CACHE_TYPE_V" + --jinja + --reasoning auto --reasoning-format deepseek +) +exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 2276085..b2c95fd 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -3,9 +3,11 @@ # frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves # bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q5_K_M via a CUDA (sm_86) build of # llama.cpp master, chosen for the ASYMMETRIC KV Frank asked for: K f16 -# (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. This -# seam runs the serve script against a stub engine and asserts on the argv -# it actually execs, so what is pinned is the command the operator gets. +# (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. Since +# 2026-10-02 it runs THREE contexts and carries a vision projector, and the +# card must keep 2-3 GiB free. This seam runs the serve script against a stub +# engine and asserts on the argv it actually execs, so what is pinned is the +# command the operator gets. load helpers @@ -15,6 +17,8 @@ ENV_FILE="$REPO_ROOT/scripts/frank-small.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q5_K_M.gguf" +# Same basename as the shipped projector: the test env repoints it into $STAGE. +MM_STUB="mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf" setup() { STAGE="$(mktemp -d)" @@ -26,7 +30,7 @@ setup() { printf '%s\n' "$@" STUB chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server" - touch "$MODEL_DIR/$TARGET_STUB" + touch "$MODEL_DIR/$TARGET_STUB" "$MODEL_DIR/$MM_STUB" } teardown() { @@ -37,9 +41,12 @@ teardown() { # argv, one token per line, in $output. serve_argv() { local envf="${1:-$STAGE/test.env}" + local mmproj + mmproj="$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")" cat > "$envf" < 8. + # Two asserts, both about output the operator sees: the pool must still + # fit with 2 GiB spare, and it must be large enough that 5 GiB spare is + # NOT reachable — an undersized pool is as wrong as an oversized one. + # Arithmetic, not a boot log: change CTX_SIZE and this bites before the + # deploy does. The same pool with f16 V (34 KiB/token) lands past the + # card, which is why the contract is asymmetric. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 25 / 1048576)) - [ $((kv_gib + 7 + 2)) -le 24 ] + [ $((kv_gib + 8 + 2)) -le 24 ] + [ $((kv_gib + 8 + 5)) -gt 24 ] f16v_gib=$((ctx * 34 / 1048576)) - [ $((f16v_gib + 7 + 2)) -gt 24 ] + [ $((f16v_gib + 8)) -gt 24 ] } @test "no speculative draft ships by default" { @@ -125,30 +152,49 @@ argv_value() { ! grep -q '^DRAFT' "$ENV_FILE" } -@test "frank-small stays text-only on the GGUF path and speaks the fleet reasoning field" { - # The GGUF carries no vision projector; the image path left with the - # SGLang checkpoint. models.yml declares frank-small text-only since - # the 2026-10-01 night swap. - ! grep -q -- '--mmproj' "$SERVE" - # deepseek format = thinking lands in reasoning_content, the field the - # harness has always read on this endpoint (SGLang parity). +@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" { + # The Q5_K_M GGUF carries no `v.*` tensors, so the projector argument IS + # the vision capability: no flag, no eyes. serve_argv + [ "$status" -eq 0 ] + [ "$(argv_value --mmproj)" = "$MODEL_DIR/$MM_STUB" ] + + # The documented opt-out must reach the engine as ABSENCE of the flag, + # not as an argument pointing at an empty string. + sed -i 's|^MMPROJ_FILE=.*|MMPROJ_FILE=|' "$STAGE/test.env" + run env FRANK_SMALL_ENV_FILE="$STAGE/test.env" nvidia-modprobe=true bash "$SERVE" + [ "$status" -eq 0 ] + ! grep -qxF -- '--mmproj' <<<"$output" +} + +@test "the fleet reasoning contract survives the reslice to three contexts" { + serve_argv + [ "$status" -eq 0 ] + # deepseek format = thinking lands in reasoning_content, the field the + # harness reads on this endpoint; --jinja is what makes + # chat_template_kwargs reach the template at all. [ "$(argv_value --reasoning-format)" = "deepseek" ] argv_value --jinja >/dev/null grep -qxF -- '--jinja' <<<"$output" } -@test "the serve script refuses a missing runtime or payload" { +@test "the serve script refuses a missing runtime, GGUF or projector" { + rm -f "$MODEL_DIR/$MM_STUB" + serve_argv + [ "$status" -ne 0 ] + [[ "$output" == *"frank-small: missing"* ]] + rm -f "$RUNTIME_DIR/bin/llama-server" serve_argv [ "$status" -ne 0 ] [[ "$output" == *"frank-small: missing"* ]] } -@test "the deploy handler gates on the binary and the GGUF, then MainPID-hardens health" { +@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" { grep -q 'deploy_llm_small' "$REMOTE_DEPLOY" grep -q 'LLM_SMALL_RUNTIME' "$REMOTE_DEPLOY" grep -q 'LLM_SMALL_MODEL_FILE' "$REMOTE_DEPLOY" + grep -q 'LLM_SMALL_MMPROJ' "$REMOTE_DEPLOY" grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY" grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY" grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"