diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index d0d8e6c..d6feae9 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1459,7 +1459,7 @@ LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism-b10 # image path — a deploy that starts without it silently ships a text-only # endpoint. Checked unconditionally, like frank-dragoon's. Q8_0 is MANDATORY, # not a preference: the card says the BF16 tower can crash llama-server when -# several image requests run at once, and this endpoint runs --parallel 3. Its +# several image requests run at once, and this endpoint runs --parallel 2. Its # 868.32 MiB worst-case VRAM (595.29 model + 248.10 compute in the boot ledger) # lands AFTER the engine's fit check, so it is not inside the q8_0 budget. LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}" diff --git a/scripts/frank-zealot.env b/scripts/frank-zealot.env index e5e69cd..aeead2d 100644 --- a/scripts/frank-zealot.env +++ b/scripts/frank-zealot.env @@ -2,12 +2,14 @@ # prism-ml/Ternary-Bonsai-2-9B-gguf at PQ2_0 **with the MTP head in the file** # on the RTX 3090, :7771. # -# 2026-10-18 (Frank): swap from the Bonsai-2-**27B** PQ2_0 to the 9B MTP export -# at THREE 262,144-token conversations (Frank: "if you can possibly fit a third -# context, do it"). The 9B bills HALF the KV of the 27B per token, which is what -# funds 3 x the native window on one 24,123 MiB card. Speculative decoding ships -# OFF — measured, not assumed; see the MTP law below, which is the whole story -# and the reason the publisher's "depth 2 works best" line does not transfer. +# 2026-10-18 (Frank): swap from the Bonsai-2-**27B** PQ2_0 to the 9B MTP export, +# and the same evening the shape moved to TWO 262,144-token conversations over an +# UNQUANTIZED (f16) KV cache — "shift frank-zealot to 2 parallels of unquantized +# kv". Exact K and V is the retrieval-fidelity ask; the price is one concurrent +# conversation, because f16 owns the card. Speculative decoding ships OFF — +# measured, not assumed; see the MTP law below, which is the whole story and the +# reason the publisher's "depth 2 works best" line does not transfer (and why +# exact KV makes the drafter illegal outright). # # THE RUNTIME IS A LOCK, NOT A PREFERENCE. The file is general.file_type 141 # carrying custom GGML type 142 tensors plus prism.hadamard.* metadata @@ -27,7 +29,7 @@ MODEL_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.g # 624,229,664 B, Hub LFS sha256 e0cb6f69… — Q8_0 IS MANDATORY, not a preference: # the publisher's card says the BF16 mmproj "can crash llama-server when several # image requests run at once; Q8_0 is not affected", and this endpoint runs -# --parallel 3, i.e. exactly that shape. The projector's worst case lands AFTER +# --parallel 2, i.e. exactly that shape. The projector's worst case lands AFTER # the engine's own fit check, so it is never inside a projection number. MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf # Base alignment: Bonsai-2 is the censored BASE lineage. The 27B CRACK re-export @@ -42,16 +44,18 @@ MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.ggu ALIAS=frank-zealot BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# CTX_SIZE is the POOL llama.cpp divides across slots: 786,432 / 3 = 262,144 per -# conversation — each slot at the model's TRAINED window, which is the point of -# the swap (the 27B era had to cut each conversation to 200,704 to fit two). -# MUST stay divisible by PARALLEL (the seam enforces it) and the harness -# contextWindow quotes the PER-SLOT number — a longer prompt is answered with a -# 400, never truncated. The ceiling cannot go past 3 x 262,144: that is the -# native wall (no YaRN in this path) and the arithmetic agrees — a 4th slot -# would push each below the compaction contract. -CTX_SIZE=786432 -PARALLEL=3 +# CTX_SIZE is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144 per +# conversation — each slot STILL at the model's TRAINED window; the per-slot +# contract did not move. MUST stay divisible by PARALLEL (the seam enforces it) +# and the harness contextWindow quotes the PER-SLOT number — a longer prompt is +# answered with a 400, never truncated. THE POOL CANNOT STAY AT 786,432 UNDER +# EXACT KV: f16 bills 32,768 B/token, so that pool is 24,576 MiB — more than the +# 24,123 MiB the driver leaves addressable, before a single weight loads. +# Halving the POOL to 524,288 (16,384 MiB) is what funds exact KV. The window +# cannot go past 262,144 a slot either: that is the native wall (no YaRN in this +# path), so a 3rd slot would be a KV-dtype cut, not more context. +CTX_SIZE=524288 +PARALLEL=2 BATCH_SIZE=4096 # Ubatch is a prefill knob (decode never moves) with NO interior optimum on this # CUDA card up to 2048 — 2048 is the measured optimum (+4.6 % prefill). It has a @@ -60,38 +64,47 @@ BATCH_SIZE=4096 UBATCH_SIZE=2048 # KV law (header + boot ledger, both confirmed): qwen35 hybrid, 32 layers with # qwen35.full_attention_interval = 4 ⇒ 8 full-attention + 24 Gated DeltaNet; -# 4 KV heads × 256 dim = 8,192 elements per token per cache. q8_0 bills 8,704 B -# per cache ⇒ 17,408 B/token = 17 KiB, so the 786,432 pool is 13,056.00 MiB — -# the ledger said exactly that. HALF the 27B's 34,816 B/token, which is the -# whole reason three native windows fit. f16 would bill 26,112 MiB at this pool -# (against a 24,123 MiB card) and dies at cudaMalloc; the 24 DeltaNet layers -# bill a fixed recurrent state per SLOT instead of per token — the PRODUCTION -# ledger prints "RS buffer size = 150.75 MiB (3 cells, 32 layers, 3 seqs)" -# (~50 MiB a cell), while the SPEC-ARMED arms print 452.25 MiB, because MTP -# depth 2 bills three seqs per slot (9 cells) — which is why pool arithmetic -# alone under-prices the shape. -CACHE_TYPE_K=q8_0 -CACHE_TYPE_V=q8_0 +# 4 KV heads × 256 dim (n_embd_k_gqa = n_embd_v_gqa = 1,024 per layer) = 8,192 +# elements per token per cache. EXACT KV: f16 = 2 B/element ⇒ 16,384 B per cache +# ⇒ 32,768 B/token = 32 KiB, so the 524,288 pool is 16,384.00 MiB. q8_0 bills +# 1.0625 B/element (32 codes + one 2-byte scale per 32-block) ⇒ 17,408 B/token, +# which is the 13,056.00 MiB the 3-slot ledger printed at the 786,432 pool. +# THE RATIO IS 1.882×, NOT 2×: the old note here said "f16 would bill 26,112 +# MiB" because it doubled the q8_0 figure with its scale overhead included; +# exact arithmetic says 24,576 MiB at that pool. The wall stands, only its +# height moves. The 24 DeltaNet layers bill a fixed recurrent state per SLOT +# instead of per token — the 3-slot ledger printed "RS buffer size = 150.75 MiB +# (3 cells, 32 layers, 3 seqs)", ~50.25 MiB a cell so two slots bill 100.50 — +# while the SPEC-ARMED arms print 452.25 MiB, because MTP depth 2 bills three +# seqs per slot (9 cells). Pool arithmetic alone under-prices the shape. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 # SPECULATIVE DECODING: OFF, MEASURED. The head is in the file and the drafter # is GOOD (acceptance 0.798/0.617 per position, mean accepted length 2.41) — it # still loses, because the draft context bills its OWN f16 KV over the WHOLE -# pool: 4,096 B/position × 786,432 = 3,072.00 MiB, plus its compute buffers, -# 4,830 MiB all told. Booted arms, same card, same traffic (2026-10-18): +# pool at its default dtype. That dtype IS a knob (--spec-draft-type-k / +# --spec-draft-type-v, both default f16 — verified against this build's --help), +# but SPEC_TYPE=none never sends them. Under the +# 3-slot q8_0 shape that bill was 4,096 B/position × 786,432 = 3,072.00 MiB plus +# ~1,758 MiB of buffers. Booted arms, same card, same traffic (2026-10-18, the +# WHOLE table is q8_0 K + q8_0 V — it predates exact KV): # 3 × 262,144 spec OFF ub 2048 → 17,876 used / 6,249 free, decode 167.2 t/s # 3 × 262,144 spec ON ub 2048 → 22,706 used / 1,419 free, decode 109.2 t/s # 3 × 262,144 spec ON ub 1024 → 21,400 used / 2,725 free, decode 213.5 t/s # 2 × 262,144 spec ON ub 2048 → 17,178 used / 6,947 free, decode 213.2 t/s # 1 × 262,144 spec ON ub 2048 → 11,652 used / 12,473 free, decode 214.2 t/s -# i.e. MTP is worth +28 % decode whenever the card has room, and at the 3-slot -# pool it costs 1.3 GiB of compute buffers that ubatch 2048 refuses to give up — -# the squeeze, not the drafter, is what halves decode. The arm that WOULD run -# MTP at three windows is SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 (decode 213.5, -# 3-stream 250.8 aggregate) and pays prefill −17 % and 2.7 GiB of headroom on a -# card whose projector worst-case lands after the fit check. This endpoint's -# load is the fleet's COMPACTION HUB (prefill-bound, ~170k in) plus 2-3 -# concurrent subagents, so the deployed arm takes the +17 % prefill -# (2,877 vs 2,395 t/s at 106,418 tokens) and the +12 % three-stream aggregate -# (280.2 vs 250.8) over single-stream speed, and keeps 6.2 GiB free. +# MTP is worth +28 % decode whenever the card has room. At the SHIPPED exact-KV +# shape an f16-draft arm has none: the draft KV alone is 4,096 × 524,288 = +# 2,048.00 MiB plus its buffers, on a trunk that already projects ~21,200 MiB of +# the 24,123 the driver leaves addressable. DOORS BACK TO THE DRAFTER: one +# window per conversation; KV back to q8_0 with UBATCH_SIZE=1024 (213.5 t/s); +# or the UNMEASURED arm --spec-draft-type-k q8_0 --spec-draft-type-v q8_0, which +# halves the draft bill to 1,088 MiB and is legal on this build (>= b10776). +# Nobody has booted that beside exact KV — measure before promising it. +# This endpoint's load is still the fleet's COMPACTION HUB (prefill-bound, +# ~170k in): prefill is weight-bandwidth bound and does not read the KV cache +# per generated token, which is why exact KV is affordable here at all; what it +# buys back is concurrency — TWO conversations, not three. # SPEC_TYPE=none strips the WHOLE spec set (--spec-type, every --spec-draft-*): # flags move as a set or the arm measures a crash instead of an absence. SPEC_TYPE=none @@ -105,27 +118,33 @@ SPEC_DRAFT_P_MIN=0 # PRESENCE_PENALTY="" is the documented opt-out. PRESENCE_PENALTY=1.0 NGL=99 -# Boot ledger (2026-10-18 PRODUCTION boot under the unit, verbosity 4): -# 17,922 MiB used / 6,203 MiB free of 24,123 (nvidia-smi settled) +# Boot ledger REFERENCE — the previous production boot (2026-10-18 13:19, +# 3 × 262,144, q8_0 K + q8_0 V, verbosity 4), i.e. the shape this file replaced: +# 17,956 MiB used / 6,169 MiB free of 24,123 addressable (nvidia-smi settled; +# its total column reads 24,576 — the driver reserves the difference, and +# older notes quoting "a 24,123 MiB card" mean this addressable figure) # CUDA0 model 2,036.71 MiB + CPU_Mapped 257.66 MiB # CUDA0 KV 13,056.00 MiB (q8_0 K + q8_0 V, 786,432 positions) # CUDA0 RS 150.75 MiB (3 cells, 32 layers, 3 seqs) # compute 1,409.02 MiB CUDA + 1,088.87 MiB CUDA_Host (+2.84 MiB output buf) -# mmproj worst-case estimate 868.32 MiB (248.10 CUDA + 24.93 CPU compute -# meta) — charged AFTER the engine's own fit check, as always +# mmproj 595 MiB weights + 248.10 MiB CUDA compute meta; estimated worst case +# 868.32 MiB — charged AFTER the engine's own fit check, as always # no clamp line, no cudaMalloc error -# Measured on this boot (/completion timings, greedy, ignore_eos, 512 tokens): -# 168.7 t/s decode (three runs 168.4-168.7) and ~300 t/s aggregate across -# THREE concurrent streams (102.8 each). The trial arm's record (167.2 decode, -# 280.2 aggregate, 2,877 t/s prefill at 106,418) stands; the 27B it replaces -# gave 76.6 / 109.4 / 691.7 at 166k. The FIRST request after a boot is a -# warmup artifact (37.7 t/s class) and a natural generation that stops early -# makes a wall-clock aggregate read 4x low - measure with ignore_eos and read -# the engine's own timings. -# Fallback rung if the margin is ever eaten (desktop, co-tenant): q4_0 KV on the -# 27B era measured 16,392 used / 7,733 free at 2 × 200,704; the same drop here -# frees ~6.5 GiB at three windows. UBATCH_SIZE=1024 is the cheaper lever -# (−1,306 MiB measured on this shape). +# Measured on THAT boot (/completion timings, greedy, ignore_eos, 512 tokens): +# 168.7 t/s decode (three runs 168.4-168.7), ~300 t/s aggregate across THREE +# concurrent streams (102.8 each), 2,877 t/s prefill at 106,418. The 27B it +# replaced gave 76.6 / 109.4 / 691.7 at 166k. These are the numbers the exact-KV +# arm is judged against: decode reads 1.882× the KV bytes per token, so a decode +# drop is EXPECTED and must be measured, not assumed, while prefill should hold. +# Every non-KV term of that ledger survives this reslice except RS (−50.25 MiB, +# one fewer cell) and one slot's worth of scheduler state, so the f16 arm +# PROJECTS 17,956 − 13,056 + 16,384 − 50 = 21,234 MiB used / 2,889 free, and +# ~2.0 GiB free through the projector's worst case. REPLACE THIS PROJECTION WITH +# THE MEASURED PAIR after the boot: read the new ledger AND settled nvidia-smi. +# Fallback rungs if that margin gets eaten (desktop, co-tenant), cheapest first: +# CACHE_TYPE_V=q8_0 (exact K, quantized V) frees 8,192 MiB at this pool; +# CTX_SIZE=401408 (two 200,704 conversations) puts exact KV at 12,544.00 MiB and +# buys ~6.7 GiB back; UBATCH_SIZE=1024 hands 1,306 MiB of compute buffers back. # Log verbosity threshold (llama.cpp -lv). 4 = debug: KV layout, buffer sizes # and graph capture land in the journal, which is what the VRAM-fit forensics # read. Levels above 4 flood the journal. diff --git a/scripts/frank-zealot.service b/scripts/frank-zealot.service index 36fb78a..907d16c 100644 --- a/scripts/frank-zealot.service +++ b/scripts/frank-zealot.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp (PrismML build 10777) Ternary-Bonsai-2-9B PQ2_0-MTP GGUF (qwen35 hybrid, 3x262144 slots over one 786432 pool, q8_0 K+V, MTP off, mmproj Q8_0 vision) on the RTX 3090 (dankclaw, frank-zealot) +Description=llama.cpp (PrismML build 10777) Ternary-Bonsai-2-9B PQ2_0-MTP GGUF (qwen35 hybrid, 2x262144 slots over one 524288 pool, unquantized f16 K+V, MTP off, mmproj Q8_0 vision) on the RTX 3090 (dankclaw, frank-zealot) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-zealot.sh b/scripts/serve-frank-zealot.sh index 9ef001b..0302aca 100755 --- a/scripts/serve-frank-zealot.sh +++ b/scripts/serve-frank-zealot.sh @@ -3,33 +3,35 @@ # in the file) as frank-zealot on Dankclaw's RTX 3090 via llama.cpp (PrismML # vendor build 10777, CUDA sm_86). # -# 2026-10-18 (Frank): Bonsai-2 27B PQ2_0 → the 9B MTP export at THREE -# 262,144-token conversations (--ctx-size 786432 --parallel 3), q8_0 K + q8_0 V. -# The 9B bills 17,408 B/token against the 27B's 34,816 (8 full-attention layers -# instead of 16), and that halved rate is what funds three NATIVE windows where -# the 27B could only afford two cut to 200,704. Verbosity 4 ships so the -# VRAM-fit forensics (KV layout, buffer sizes, graph capture) land in the -# journal. +# 2026-10-18 (Frank): Bonsai-2 27B PQ2_0 → the 9B MTP export, then the same +# evening TWO 262,144-token conversations (--ctx-size 524288 --parallel 2) over +# an UNQUANTIZED f16 KV cache. The 9B bills 8,192 elements per token per cache +# (8 full-attention layers instead of the 27B's 16), so exact K and V costs +# 32,768 B/token — 16,384.00 MiB at the 524,288 pool, which is what a 24,123 MiB +# addressable card can carry beside the trunk. Verbosity 4 ships so the VRAM-fit +# forensics (KV layout, buffer sizes, graph capture) land in the journal. # # Architecture (engine ledger, not the marketing page): qwen35 hybrid, 32 # layers with qwen35.full_attention_interval = 4 ⇒ 8 full-attention + 24 Gated # DeltaNet, 4 KV heads × 256 dim, native 262,144. The KV pool scales with the 8 -# full layers (13,056.00 MiB at the 786,432 pool, ledger-exact); the DeltaNet -# layers cost a fixed recurrent state per SLOT — 452.25 MiB for 3 cells, ~151 -# MiB each. The runtime is a hard lock, not a preference: custom GGML type 142 -# plus prism.hadamard.* metadata, which only the vendor build registers, and -# MTP beside quantized V needs build ≥ b10776 — so the prebuilt moved 10754 → -# 10777 with the payload. Vision comes from the publisher's Q8_0 projector (the -# BF16 one can crash the server on concurrent images, and this endpoint runs -# three slots), so --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is -# the documented text-only opt-out. +# full layers (16,384.00 MiB of f16 at the 524,288 pool); the DeltaNet layers +# cost a fixed recurrent state per SLOT — ~50.25 MiB a cell, so 100.50 MiB for +# two. The runtime is a hard lock, not a preference: custom GGML type 142 plus +# prism.hadamard.* metadata, which only the vendor build registers (the b10776 +# gate that moved the prebuilt 10754 → 10777 was MTP beside quantized V, which +# exact KV now makes moot — the type-142 lock alone keeps the build pinned). +# Vision comes from the publisher's Q8_0 projector (the BF16 one can crash the +# server on concurrent images, and this endpoint runs two slots), so --mmproj is +# what gives this endpoint eyes; MMPROJ_FILE="" is the documented text-only +# opt-out. # # Speculative decoding ships OFF, measured: the head loads and accepts well # (0.798/0.617 per position, mean length 2.41) but the draft context bills its -# own f16 KV over the WHOLE pool — 4,096 B/position × 786,432 = 3,072 MiB plus -# buffers — and at ubatch 2048 that squeeze HALVES decode (167.2 → 109.2 t/s). -# frank-zealot.env carries the six-arm table and the arm that turns it back on -# (--ubatch-size 1024, decode 213.5 t/s) with its price. +# own f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus +# buffers — and the trunk already projects 21,234 MiB with exact KV. Exact KV +# and MTP cannot share this card; frank-zealot.env carries the q8_0-era arm table +# and the two doors back to the drafter (one window, or KV back to q8_0 at +# --ubatch-size 1024 for 213.5 t/s). # # Sampling: the checkpoint carries the publisher's thinking recipe as GGUF # metadata (general.sampling.* = temp 1.0 / top_p 0.95 / top_k 20 / min_p 0 / @@ -56,40 +58,46 @@ fi : "${ALIAS:=frank-zealot}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -# ctx-size is the POOL llama.cpp divides across slots: 786,432 / 3 = 262,144 -# per conversation — every slot at the model's trained window. The harness +# ctx-size is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144 +# per conversation — every slot still at the model's trained window. The harness # contextWindow MUST quote the PER-SLOT number — a longer prompt is answered -# with a 400, never truncated. -: "${CTX_SIZE:=786432}" -: "${PARALLEL:=3}" +# with a 400, never truncated. Under exact KV the pool cannot stay at 786,432: +# 32,768 B/token × 786,432 = 24,576 MiB, more than the card's whole addressable +# 24,123 MiB before a single weight loads. +: "${CTX_SIZE:=524288}" +: "${PARALLEL:=2}" # Compute-buffer geometry. Ubatch is a prefill knob (decode never moves). # 2048 is the measured CUDA optimum (MiMo era, same arch/shape: +4.6 % prefill # at 8.9k depth, decode flat, 616 MiB). The logical batch is VRAM- and # throughput-neutral at a fixed ubatch. It ALSO sets the compute-buffer size, -# which is the term that decides whether an MTP draft context fits beside a -# 786,432-position pool: 1024 hands back 1,306 MiB and is the arm that runs the -# drafter at three slots (frank-zealot.env carries the table). +# which is the term that decides whether an MTP draft context fits beside a KV +# pool: 1024 hands back 1,306 MiB and was the arm that ran the drafter at three +# q8_0 windows (frank-zealot.env carries the table). : "${BATCH_SIZE:=4096}" : "${UBATCH_SIZE:=2048}" -# Quantized KV: q8_0 K + q8_0 V — the only pair that funds 3 × 262,144 on this -# card. 17 KiB/token at this geometry (8 full-attention layers × 4 KV heads × -# 256 dim); 13,056.00 MiB at the 786,432 pool, ledger-exact. f16 would bill -# 26,112 MiB against a 24,123 MiB card and dies at cudaMalloc; q4_0 is the -# fallback rung. Explicit either way so the contract is visible in the argv, -# and --flash-attn stays required for quantized V (no true fp8 KV dtype exists -# in this build). -: "${CACHE_TYPE_K:=q8_0}" -: "${CACHE_TYPE_V:=q8_0}" +# Exact KV: f16 K + f16 V — no quantization error anywhere in the attention +# state. 32 KiB/token at this geometry (8 full-attention layers × 4 KV heads × +# 256 dim × 2 B × 2 caches); 16,384.00 MiB at the 524,288 pool. That is the whole +# reason the slot count is 2: the same dtype at the old 786,432 pool is 24,576 +# MiB and dies at cudaMalloc. q8_0 K + q8_0 V (17,408 B/token, 13,056.00 MiB at +# 786,432) is the documented fallback rung and funds three windows; CACHE_TYPE_V +# alone is the halfway door. Explicit either way so the contract is visible in +# the argv, and --flash-attn stays pinned because quantized V — the fallback — +# is illegal without it (no true fp8 KV dtype exists in this build). +: "${CACHE_TYPE_K:=f16}" +: "${CACHE_TYPE_V:=f16}" # Speculative decoding defaults OFF, measured on this card and shape: the head # IS in the checkpoint (qwen35.nextn_predict_layers = 1 — MTP is a property of # the FILE, so the plain PQ2_0 sibling could not run it at any flag value), it # loads, and it accepts well (mean accepted length 2.41) — but the draft context -# bills its own f16 KV over the WHOLE pool (4,096 B/position, 3,072 MiB here) -# and at ubatch 2048 that squeeze halves decode: 167.2 → 109.2 t/s. There is no -# --spec-draft-cache-type knob in this build to make the draft cheap, and no -# --spec-draft-model either: the draft rides the main model's weights. Card gate: -# quantized V alongside MTP needs fork build >= prism-b10776, which is why -# LLAMA_BIN moved to the 10777 prebuilt; on an older build the pair is illegal. +# bills its own f16 KV over the WHOLE pool (4,096 B/position, 2,048 MiB at the +# 524,288 pool) and at ubatch 2048 that squeeze halves decode: 167.2 → 109.2 t/s +# on the q8_0 3-slot shape. The draft cache dtype IS a knob (--spec-draft-type-k +# / --spec-draft-type-v, both default f16 — verified in this build's --help), so +# an armed drafter can halve its own bill; there is no --spec-draft-model because +# the draft rides the main model's weights. Card gate: quantized V alongside MTP +# needs fork build >= prism-b10776, which is why LLAMA_BIN moved to the 10777 +# prebuilt; exact KV removes that gate but NOT the type-142 loader lock. # SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 is the documented MTP arm (213.5 t/s); # SPEC_TYPE=none strips the whole set (flags move together). : "${SPEC_TYPE:=none}" @@ -104,7 +112,7 @@ fi : "${NGL:=99}" # Log verbosity threshold (llama.cpp -lv). 3 is the engine default (info); # 4 adds debug detail (KV layout, buffer sizes, graph capture) which is what -# the VRAM-fit forensics need — the q8_0 KV contract and the draft-KV bill are +# the VRAM-fit forensics need — the exact-KV size and the draft-KV bill are # only provable from this log. Levels above 4 flood the journal. : "${VERBOSITY:=4}" # /metrics needs the flag: llama.cpp returns 501 unless the server starts with @@ -169,11 +177,11 @@ fi # open after boot; make sure it exists before the server needs it. nvidia-modprobe -u -c=0 2>/dev/null || true -# Flash attention stays pinned on: it is REQUIRED for quantized V (q8_0), -# and the build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17), so -# the q8_0 KV contract in frank-zealot.env is a real path — without this -# flag or that build option the engine silently falls back and the -# contract becomes a lie. +# Flash attention stays pinned on. Exact f16 KV runs fine with or without it, +# but the documented fallback rung (quantized V) is ILLEGAL without it, and the +# build carries GGML_CUDA_FA_ALL_QUANTS=ON (since 2026-09-17) so that rung is a +# real path — dropping the flag here would leave the retreat route silently +# falling back to something the env file does not describe. ARGS=( --model "$MODEL_FILE" --alias "$ALIAS" diff --git a/tests/frank_zealot_unit.bats b/tests/frank_zealot_unit.bats index da63189..1212a4f 100755 --- a/tests/frank_zealot_unit.bats +++ b/tests/frank_zealot_unit.bats @@ -3,15 +3,16 @@ # prism-ml/Ternary-Bonsai-2-9B-gguf at PQ2_0 — the ternary Bonsai-2-9B, a # qwen35 hybrid with 32 layers (8 full-attention + 24 Gated DeltaNet) and the # MTP draft head INSIDE the file — via the PrismML fork build of llama.cpp -# (vendor build 10777, the minimum that pairs MTP with a quantized V cache), -# the ONLY loader that registers the payload's custom tensor type 142 and -# honours its prism.hadamard.* metadata: THREE 262,144-token conversations -# (--ctx-size 786432 --parallel 3), q8_0 K + q8_0 V under flash attention (f16 -# KV would bill 26,112 MiB of a 24,123 MiB card), the publisher's own Q8_0 -# vision projector (the BF16 one can crash on concurrent images), and MTP OFF -# — measured, not because the head is missing: the draft context bills f16 KV -# over the whole pool and halves decode at this shape. Log verbosity 4 ships so -# the VRAM-fit forensics land in the journal. +# (vendor build 10777), the ONLY loader that registers the payload's custom +# tensor type 142 and honours its prism.hadamard.* metadata: TWO 262,144-token +# conversations (--ctx-size 524288 --parallel 2) over an UNQUANTIZED f16 KV +# cache. Exact KV bills 32,768 B/token here, so the pool is what pays for it — +# the same dtype at the old 786,432 pool is 24,576 MiB against the 24,123 MiB +# the driver leaves addressable, and the slot count is the term that moved. The +# publisher's own Q8_0 vision projector rides along (the BF16 one can crash on +# concurrent images), and MTP stays OFF: the head is present, but exact KV +# leaves no room for a draft context that bills f16 KV over the whole pool. Log +# verbosity 4 ships so the VRAM-fit forensics land in the journal. # This seam runs the serve script against a stub engine and asserts on the # argv it actually execs, so what is pinned is the command the operator gets. @@ -76,6 +77,22 @@ argv_value() { grep -A1 -xF -- "$1" <<<"$output" | tail -1 } +# kv_bytes_per_token DTYPE — bytes ONE cache costs per token at this payload's +# geometry: 8 full-attention layers × 4 KV heads × 256 dim = 8,192 elements +# (measured from the GGUF header: n_embd_k_gqa = n_embd_v_gqa = 1024). f16 is +# 2 B/element; q8_0 is 1.0625 (32 codes plus one 2-byte scale per 32-block); +# q4_0 is 0.625 (codes + scale + min per 32-block). Unknown dtype is a FAILURE, +# never a silent zero — the fit gate below is only honest if it cannot be +# satisfied by a rung it does not understand. +kv_bytes_per_token() { + case "$1" in + f16|bf16) echo 16384 ;; + q8_0) echo 8704 ;; + q4_0) echo 5120 ;; + *) return 1 ;; + esac +} + # run_stage — exec the CURRENT $STAGE/test.env as-is. serve_argv REWRITES # that file from the shipped env, so any test that mutates it (drop a line, # append a drafter) must exec through this, not serve_argv — which silently @@ -126,7 +143,7 @@ run_stage() { done } -@test "three conversations share the pool at a 262144 window each" { +@test "two conversations share the pool at a 262144 window each" { serve_argv [ "$status" -eq 0 ] [ "$(argv_value --alias)" = "frank-zealot" ] @@ -134,15 +151,13 @@ run_stage() { [ "$(argv_value --host)" = "0.0.0.0" ] [ "$(argv_value --model)" = "$MODEL_DIR/$MODEL_STUB" ] [ "$(argv_value --n-gpu-layers)" -gt 0 ] - # llama.cpp divides --ctx-size across slots, so the observable contract - # is the per-slot window. The 9B swap exists precisely so every - # conversation gets the model's TRAINED 262,144 (the 27B had to be cut to - # 200,704 to fund two), and the harness contextWindow must quote 262144 — - # the pool and the slot count are neither, and a longer prompt is a 400. - # Three is also the ceiling: a fourth slot would push each window under the - # fleet's worst-case compaction input, which is this endpoint's job. - [ "$(argv_value --parallel)" -eq 3 ] - [ "$(argv_value --ctx-size)" -eq 786432 ] + # llama.cpp divides --ctx-size across slots, so the observable contract is + # the per-slot window. Exact KV costs the SLOT COUNT, not the window: every + # conversation still gets the model's TRAINED 262,144, and the harness + # contextWindow must quote that number — the pool and the slot count are + # neither, and a longer prompt is a 400, never a truncation. + [ "$(argv_value --parallel)" -eq 2 ] + [ "$(argv_value --ctx-size)" -eq 524288 ] local pool slots pool="$(argv_value --ctx-size)"; slots="$(argv_value --parallel)" [ $((pool % slots)) -eq 0 ] @@ -151,15 +166,16 @@ run_stage() { @test "the KV dtype pair follows the env file and flash-attn stays pinned on" { # Asserted against the env rather than a literal so a retune moves one - # file, not two. The shipped pair is q8_0/q8_0: this payload bills - # 8 full-attention layers × 4 KV heads × 256 dim = 8,192 elements per token - # per cache, so f16/f16 would need 26,112 MiB of the 24,123 MiB card while - # q8_0 bills 17,408 B/token = 13,056.00 MiB at the 786,432 pool — the - # ledger said exactly that, and it is the referee. The 24 DeltaNet layers - # bill a fixed recurrent state per slot (452.25 MiB for three cells), which - # is why pool arithmetic alone under-prices the shape. flash-attn is not - # decoration: quantized V is only legal with it, and it must stay explicit - # so a future quant swap cannot silently fall back. + # file, not two. The shipped pair is f16/f16: exact attention state, and the + # slot count is what makes it affordable — this payload bills 8,192 elements + # per token per cache (8 full-attention layers × 4 KV heads × 256 dim), so + # exact KV is 32,768 B/token = 16,384.00 MiB at the 524,288 pool, where the + # q8_0 pair this file used to ship billed 17,408 B/token = 13,056.00 MiB at + # the old 786,432 pool (ledger-exact both times). The 24 DeltaNet layers + # bill a fixed recurrent state per slot (~50.25 MiB a cell), which is why + # pool arithmetic alone under-prices the shape. flash-attn is not + # decoration: the quantized-V fallback is only legal with it, so it stays + # explicit and a dtype retune cannot silently fall back. local env_k env_v env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" @@ -170,6 +186,38 @@ run_stage() { [ "$(argv_value --flash-attn)" = "on" ] } +@test "the shipped KV dtype and pool actually fit the card this boots on" { + # No health gate can see this mistake: llama.cpp sizes the KV pool as + # rate × pool-positions and cudaMalloc fails at boot ON THE LIVE ENDPOINT. + # The fit is arithmetic off the payload geometry plus the non-KV terms + # measured off the boot ledger (weights 2,036.71 + compute buffers 1,409.02 + # + projector 595 + 248.10 + CUDA context ≈ 460 = 4,749 MiB, dtype- and + # slot-independent at this ubatch), plus one ~51 MiB recurrent-state cell + # per slot. 24,123 MiB is what the driver leaves addressable (nvidia-smi + # reports it as used + free; its total column reads 24,576). + local env_k env_v pool slots rate_k rate_v kv total + env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" + env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" + pool="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + slots="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")" + rate_k="$(kv_bytes_per_token "$env_k")" + rate_v="$(kv_bytes_per_token "$env_v")" + kv="$(( pool * (rate_k + rate_v) / 1048576 ))" + total="$(( kv + 4749 + 51 * slots ))" + # The shipped arm has to leave real room: the projector's worst case is + # charged AFTER the engine's own fit check, so a projection that lands on + # the ceiling is a crash waiting for the first image request. + [ "$total" -le $(( 24123 - 1024 )) ] + + # And the gate must be able to REFUSE: the same exact KV at the pool this + # shape left behind is 24,576 MiB of cache alone — a boot that dies before + # /health ever answers. Proving the gate bites is what makes passing one + # meaningful. + local over + over="$(( 786432 * (rate_k + rate_v) / 1048576 + 4749 + 51 * 3 ))" + [ "$over" -gt 24123 ] +} + @test "log verbosity follows the env file" { serve_argv [ "$status" -eq 0 ] @@ -233,10 +281,11 @@ run_stage() { # Frank, 2026-10-18: this file DOES carry the MTP head (the boot arms prove # it loads and drafts, acceptance 0.798/0.617 per position, mean accepted # length 2.41) and it is still off, because the draft context bills its own - # f16 KV over the WHOLE pool — 4,096 B/position × 786,432 = 3,072 MiB plus - # buffers — and at --ubatch-size 2048 that squeeze halves decode: 167.2 → - # 109.2 t/s. The MTP arm is SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 (213.5 - # t/s), documented in frank-zealot.env. The flags move as a set: + # f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus + # buffers — on a trunk that already projects 21,234 MiB with exact KV. The + # MTP arms in frank-zealot.env are all q8_0-era measurements; the doors back + # to the drafter are one window per conversation, or KV back to q8_0 with + # UBATCH_SIZE=1024 (213.5 t/s). The flags move as a set: # SPEC_TYPE=none strips --spec-type, --spec-draft-n-max AND # --spec-draft-model together, so a no-spec boot cannot carry half a spec # pair (the ROCmFPX-era baseline arm segfaulted at load doing that). @@ -276,7 +325,7 @@ run_stage() { # The Bonsai-2 text file carries no vision tensors — the publisher's Q8_0 # projector argument IS the vision capability: no flag, no eyes. Q8_0 (not # BF16) is the publisher's own requirement for concurrent image requests, - # which is what --parallel 3 means in practice. + # which is what --parallel 2 means in practice. serve_argv [ "$status" -eq 0 ] [ "$(argv_value --mmproj)" = "$MODEL_DIR/$MM_STUB" ]