diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 699fbef..0b7483b 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -27,15 +27,20 @@ # context-backed drafters cannot share one process, so draft-mtp,ngram-mod # is the only legal shape that carries both. # -# SIZING (RTX 3090, 24576 MiB): not modelled by hand -- CTX_SIZE is left -# unset and --fit sizes the KV pool (q8_0 K and V) against what the 6.87 -# GiB of weights, the 0.16 GiB projector, the MTP sidecar's draft context -# and the compute buffers leave on the card. The engine prints the chosen -# context at startup; read it from the log, do not hard-code it back here. +# SIZING (RTX 3090, 24576 MiB): CTX_SIZE = PARALLEL x 262144 -- four +# FULL-window slots, measured to fit: the 4-slot boot idles at 22,346 +# MiB and each extra full-context slot costs 3,450 MiB (measured 1-slot +# 11,996 -> 4-slot 22,346). A fifth slot projects to ~25.8 GiB, past the +# card, and the KV pool allocates upfront -- so 4 is the max. Explicit +# CTX_SIZE bypasses --fit on purpose: --fit maximises the TOTAL pool, +# which would shrink each slot below the full 262144 window once +# parallel > 1. # -# Vision: --mmproj loads the BF16 projector (11 tensors, 52M params; it -# also carries an audio encoder, unused by the fleet). The fleet's image -# path lives here, which is why halogen no longer sets HALOGEN_VISION_TOWER. +# Vision + audio: --mmproj loads the BF16 projector (11 tensors, 52M +# params; it carries an audio encoder too). The fleet's image path lives +# here, which is why halogen no longer sets HALOGEN_VISION_TOWER. Audio +# WORKS: input_audio content parts transcribe verbatim (verified +# 2026-09-17 with a 16k mono WAV). # # Sampling: greedy by default (the fleet contract); a client that sends a # temperature gets its own, and the harness applies per-model overrides @@ -62,9 +67,11 @@ ALIAS=frank-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# Context: intentionally unset -- --fit decides it (see the sizing note). -# One slot; a second concurrent request queues behind the first. -PARALLEL=1 +# Four FULL 262144-token contexts, one per slot (CTX_SIZE is the total +# pool; --parallel divides it). See the SIZING note above for the fit +# arithmetic; a fifth slot does not fit. +CTX_SIZE=1048576 +PARALLEL=4 CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 7dd1710..a97f7e4 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -15,13 +15,16 @@ # context-backed drafters cannot share one process, so this chain is the # only shape that carries both. # -# Context: CTX_SIZE is left unset and --fit sizes the KV pool (q8_0 K and -# V) against what the weights, projector, draft context and compute -# buffers leave on the card; the chosen value lands in the startup log. +# Context: CTX_SIZE is set explicitly to PARALLEL x 262144 so every slot +# gets the model's FULL window; --fit (the CTX_SIZE-unset branch below) +# maximises the TOTAL pool instead, which would divide a smaller window +# across slots. The 4-slot layout is measured to fit the card (see the +# SIZING note in frank-small.env); more slots would fail the KV allocation. # -# Vision: --mmproj loads the BF16 projector (it also carries an audio -# encoder; vision is the fleet contract). The fleet's image path lives -# here, which is why halogen no longer sets HALOGEN_VISION_TOWER. +# Vision + audio: --mmproj loads the BF16 projector, which also carries +# an audio encoder — input_audio content parts transcribe verbatim +# (verified 2026-09-17). The fleet's image path lives here, which is why +# halogen no longer sets HALOGEN_VISION_TOWER. # # Sampling: greedy by default (the fleet contract). Clients that send their own # temperature get their own values, and the harness overrides per model from diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index d952d8e..01163b1 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -61,22 +61,28 @@ QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE" } -@test "frank-small quantizes KV to q8 and lets --fit size the context" { - local parallel cache_k cache_v ctx +@test "frank-small's slot grid is full-window and fits the 24 GiB card" { + local parallel cache_k cache_v ctx fixed per_slot parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")" cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" - [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] - [ "$parallel" -ge 1 ] - # q8_0 KV is what fits the 24 GiB card beside 7.4 GiB of Q4_K_M weights + - # mmproj + sidecar: Gemma4's 16 kv heads x 512 dim cost 16 KiB per token - # per layer at q8, and the model's sliding-window pattern keeps the pool - # far below full-ctx x all-layers. The engine itself sizes the context: - # CTX_SIZE is unset, so the serve script passes --fit on. - [ -z "$ctx" ] + [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$ctx" ] + # Every slot must be the model's FULL trained window (262144), not a + # divided one -- CTX_SIZE is the total pool and --parallel divides it. + [ $((ctx / parallel)) -eq 262144 ] + [ $((ctx % parallel)) -eq 0 ] [ "$cache_k" = "q8_0" ] && [ "$cache_v" = "q8_0" ] - grep -q -- '--fit on' "$SERVE" + # Fit budget from the measured boot arithmetic on the 3090 (24576 + # MiB): one full-context slot idles at 11,996 MiB, four at 22,346, + # so each slot costs (22346-11996)/3 = 3450 MiB and the fixed part + # (weights, projector, sidecar draft, compute buffers, CUDA context) + # is 11996-3450 = 8546 MiB. If the config asks for more slots than + # the card holds, this inequality fails before deploy instead of the + # KV allocation failing at boot. + fixed=8546 + per_slot=3450 + [ $((fixed + parallel * per_slot)) -le 24576 ] # Quantized KV on CUDA needs the GGML_CUDA_FA_ALL_QUANTS build option, # without which prefill collapses about 12x (see the CUDA lesson in # AGENTS.md). The build-cuda tree must carry it.