feat(frank-zealot): 3 x 200,704, q8_0 V, MTP armed #25
No files matched your search
@@ -1459,7 +1459,7 @@ LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism-b10
|
||||
# image path — a deploy that starts without it silently ships a text-only
|
||||
# endpoint. Checked unconditionally, like frank-dragoon's. Q8_0 is MANDATORY,
|
||||
# not a preference: the card says the BF16 tower can crash llama-server when
|
||||
# several image requests run at once, and this endpoint runs --parallel 2. Its
|
||||
# several image requests run at once, and this endpoint runs --parallel 3. Its
|
||||
# 868.32 MiB worst-case VRAM (595.29 model + 248.10 compute in the boot ledger)
|
||||
# lands AFTER the engine's fit check, so it is not inside the q8_0 budget.
|
||||
LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}"
|
||||
|
||||
+34
-21
@@ -11,35 +11,48 @@ LLAMA_BIN=/home/dank/GitPackages/llama-prism-b10777/llama-server
|
||||
# metadata dequantizes wrong, silently.
|
||||
MODEL_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf
|
||||
# Q8_0 IS MANDATORY: the publisher states the BF16 tower can crash llama-server
|
||||
# under concurrent image requests, which is exactly --parallel 2.
|
||||
# under concurrent image requests, which is exactly --parallel 3.
|
||||
MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf
|
||||
# Served id — the harness/family name, not a fact about the payload.
|
||||
# Id-keyed consumers: OpenWebUI (model row, access_grant, api_configs), models.yml.
|
||||
ALIAS=frank-zealot
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
# CTX_SIZE is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144 a
|
||||
# conversation, each still at the trained window. MUST stay divisible by
|
||||
# PARALLEL (the seam enforces it) and models.yml contextWindow quotes the
|
||||
# PER-SLOT number — a longer prompt is a 400, never a truncation. The pool
|
||||
# cannot go back to 786,432 under exact f16 KV: 24,576 MiB of cache against the
|
||||
# 24,123 MiB the driver leaves addressable, before a weight loads.
|
||||
CTX_SIZE=524288
|
||||
PARALLEL=2
|
||||
BATCH_SIZE=4096
|
||||
# CTX_SIZE is the POOL llama.cpp divides across slots: 602,112 / 3 = 200,704 a
|
||||
# conversation. MUST stay divisible by PARALLEL (the seam enforces it) and
|
||||
# models.yml contextWindow quotes the PER-SLOT number — a longer prompt is a
|
||||
# 400, never a truncation. The pool is bought, not granted: K+V at 25,088
|
||||
# B/token is 14,406.00 MiB here and the MTP draft context bills its own f16 KV
|
||||
# over the WHOLE pool (4,096 B/position ⇒ 2,352.00 MiB), so the old 786,432
|
||||
# pool cannot carry a drafter at any KV dtype — 18,816 + 3,072 of cache alone
|
||||
# against the 24,123 MiB the driver leaves addressable.
|
||||
CTX_SIZE=602112
|
||||
PARALLEL=3
|
||||
# The logical batch is VRAM- and throughput-neutral at a fixed ubatch (the law
|
||||
# on [[frank-zealot]]); 1024×6 keeps it a clean multiple of the ubatch.
|
||||
BATCH_SIZE=6144
|
||||
# Ubatch is a prefill knob (decode never moves with it) and it sets the
|
||||
# compute-buffer size — which is what decides whether a drafter fits.
|
||||
# UBATCH_SIZE=1024 is the documented MTP arm.
|
||||
UBATCH_SIZE=2048
|
||||
# Exact K and V: the retrieval-fidelity ask, paid with one conversation of
|
||||
# concurrency. Rate law and the arithmetic: [[frank-zealot]] → Laws.
|
||||
# compute-buffer size — which is what decides whether a drafter fits: 1024
|
||||
# hands back ~1,306 MiB beside the draft context, and it is the rung the
|
||||
# measured MTP arm ran (213.5 t/s at 1024 against 109.2 at 2048).
|
||||
UBATCH_SIZE=1024
|
||||
# K exact, V in the 8-bit class: 16,384 + 8,704 = 25,088 B/token (8,192
|
||||
# elements per token per cache × 2 B for f16, × 1.0625 for q8_0 — there is no
|
||||
# true fp8 KV dtype in this build). That 7,680 B/token saved against exact V is
|
||||
# what funds BOTH the third conversation and the drafter. Quantized V is
|
||||
# illegal without --flash-attn, which the serve script pins. Rate law and the
|
||||
# arithmetic: [[frank-zealot]] → Laws.
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
# Speculation OFF — measured, not assumed: the draft context bills its own f16
|
||||
# KV over the WHOLE pool, and there is no room beside exact KV (the arm census
|
||||
# is on the page). SPEC_TYPE=none strips the WHOLE set (--spec-type and every
|
||||
# --spec-draft-*); half a set has segfaulted at load.
|
||||
SPEC_TYPE=none
|
||||
CACHE_TYPE_V=q8_0
|
||||
# Speculation ON (Frank, 2026-10-19): the head is in the file
|
||||
# (qwen35.nextn_predict_layers = 1, blk.32.nextn.*) and rides the MAIN model's
|
||||
# weights, so there is no --spec-draft-model. Its bill is pool × 4,096 B of
|
||||
# draft KV plus ~0.45 GiB of buffers and a tripled recurrent state — paid for
|
||||
# by CACHE_TYPE_V=q8_0 and UBATCH_SIZE=1024 above. Build gate: quantized V
|
||||
# beside MTP needs fork build >= b10776; LLAMA_BIN is 10777.
|
||||
# SPEC_TYPE=none is the rollback and strips the WHOLE set (--spec-type and
|
||||
# every --spec-draft-*); half a set has segfaulted at load.
|
||||
SPEC_TYPE=draft-mtp
|
||||
SPEC_DRAFT_N_MAX=2
|
||||
SPEC_DRAFT_N_MIN=0
|
||||
SPEC_DRAFT_P_MIN=0
|
||||
|
||||
@@ -5,33 +5,43 @@
|
||||
#
|
||||
# 2026-10-18 (Frank): Bonsai-2 27B PQ2_0 → the 9B MTP export, then the same
|
||||
# evening TWO 262,144-token conversations (--ctx-size 524288 --parallel 2) over
|
||||
# an UNQUANTIZED f16 KV cache. The 9B bills 8,192 elements per token per cache
|
||||
# (8 full-attention layers instead of the 27B's 16), so exact K and V costs
|
||||
# 32,768 B/token — 16,384.00 MiB at the 524,288 pool, which is what a 24,123 MiB
|
||||
# addressable card can carry beside the trunk. Verbosity 4 ships so the VRAM-fit
|
||||
# forensics (KV layout, buffer sizes, graph capture) land in the journal.
|
||||
# an UNQUANTIZED f16 KV cache.
|
||||
#
|
||||
# 2026-10-19 (Frank): THREE 200,704-token conversations over a 602,112 pool
|
||||
# (--ctx-size 602112 --parallel 3), K left at the engine's own default (f16),
|
||||
# V dropped to q8_0, batch 1024×6 / ubatch 1024, and the MTP drafter ARMED
|
||||
# (--spec-type draft-mtp). The 9B bills 8,192 elements per token per cache (8
|
||||
# full-attention layers instead of the 27B's 16), so f16 K is 16,384 B/token and
|
||||
# q8_0 V is 8,704: 25,088 B/token = 14,406.00 MiB at the 602,112 pool. That
|
||||
# 7,680 B/token against exact V is exactly what funds the third conversation
|
||||
# AND a draft context that bills f16 KV over the whole pool. Verbosity 4 ships
|
||||
# so the VRAM-fit forensics (KV layout, buffer sizes, graph capture) land in
|
||||
# the journal.
|
||||
#
|
||||
# Architecture (engine ledger, not the marketing page): qwen35 hybrid, 32
|
||||
# layers with qwen35.full_attention_interval = 4 ⇒ 8 full-attention + 24 Gated
|
||||
# DeltaNet, 4 KV heads × 256 dim, native 262,144. The KV pool scales with the 8
|
||||
# full layers (16,384.00 MiB of f16 at the 524,288 pool); the DeltaNet layers
|
||||
# cost a fixed recurrent state per SLOT — ~50.25 MiB a cell, so 100.50 MiB for
|
||||
# two. The runtime is a hard lock, not a preference: custom GGML type 142 plus
|
||||
# prism.hadamard.* metadata, which only the vendor build registers (the b10776
|
||||
# gate that moved the prebuilt 10754 → 10777 was MTP beside quantized V, which
|
||||
# exact KV now makes moot — the type-142 lock alone keeps the build pinned).
|
||||
# full layers (14,406.00 MiB of f16 K + q8_0 V at the 602,112 pool); the
|
||||
# DeltaNet layers cost a fixed recurrent state per SLOT — ~50.25 MiB a cell, so
|
||||
# 150.75 MiB for three, and an armed depth-2 drafter triples that again. The
|
||||
# runtime is a hard lock, not a preference: custom GGML type 142 plus
|
||||
# prism.hadamard.* metadata, which only the vendor build registers — and the
|
||||
# b10776 gate that moved the prebuilt 10754 → 10777 (MTP beside a quantized V)
|
||||
# is LOAD-BEARING again on this shape, not moot as it was under exact KV.
|
||||
# Vision comes from the publisher's Q8_0 projector (the BF16 one can crash the
|
||||
# server on concurrent images, and this endpoint runs two slots), so --mmproj is
|
||||
# server on concurrent images, and this endpoint runs three slots), so --mmproj is
|
||||
# what gives this endpoint eyes; MMPROJ_FILE="" is the documented text-only
|
||||
# opt-out.
|
||||
#
|
||||
# Speculative decoding ships OFF, measured: the head loads and accepts well
|
||||
# (0.798/0.617 per position, mean length 2.41) but the draft context bills its
|
||||
# own f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus
|
||||
# buffers — and the trunk already projects 21,234 MiB with exact KV. Exact KV
|
||||
# and MTP cannot share this card; frank-zealot.env carries the q8_0-era arm table
|
||||
# and the two doors back to the drafter (one window, or KV back to q8_0 at
|
||||
# --ubatch-size 1024 for 213.5 t/s).
|
||||
# Speculative decoding ships ON (Frank, 2026-10-19). The head is IN the file and
|
||||
# accepts well (0.798/0.617 per position, mean length 2.41, measured 2026-10-18);
|
||||
# what made it lose that evening was the bill, not the drafts — the draft context
|
||||
# is built against the target's WHOLE pool, so blk.32 adds 4,096 B/position
|
||||
# (2,352.00 MiB at 602,112) plus ~0.45 GiB of buffers at this ubatch, and the
|
||||
# recurrent state triples. Dropping V to q8_0 (−3,840 MiB at this pool) and the
|
||||
# ubatch to 1024 (−~1,306 MiB of compute buffers) is how those two bills got
|
||||
# paid; the arm census and the rates are on [[frank-zealot]]. SPEC_TYPE=none is
|
||||
# the rollback and strips the whole flag set.
|
||||
#
|
||||
# Sampling: the checkpoint carries the publisher's thinking recipe as GGUF
|
||||
# metadata (general.sampling.* = temp 1.0 / top_p 0.95 / top_k 20 / min_p 0 /
|
||||
@@ -58,49 +68,46 @@ fi
|
||||
: "${ALIAS:=frank-zealot}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
# ctx-size is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144
|
||||
# per conversation — every slot still at the model's trained window. The harness
|
||||
# contextWindow MUST quote the PER-SLOT number — a longer prompt is answered
|
||||
# with a 400, never truncated. Under exact KV the pool cannot stay at 786,432:
|
||||
# 32,768 B/token × 786,432 = 24,576 MiB, more than the card's whole addressable
|
||||
# 24,123 MiB before a single weight loads.
|
||||
: "${CTX_SIZE:=524288}"
|
||||
: "${PARALLEL:=2}"
|
||||
# ctx-size is the POOL llama.cpp divides across slots: 602,112 / 3 = 200,704 a
|
||||
# conversation. The harness contextWindow MUST quote the PER-SLOT number — a
|
||||
# longer prompt is answered with a 400, never truncated. The pool is priced by
|
||||
# the KV bill, not by the trained window: at 25,088 B/token (f16 K + q8_0 V) the
|
||||
# 602,112 pool costs 14,406.00 MiB and the armed drafter adds its own f16 KV over
|
||||
# the whole pool (2,352.00 MiB). Exact V at this pool (18,816 MiB) or the old
|
||||
# 786,432 pool at any dtype cannot boot inside the 24,123 MiB the driver leaves
|
||||
# addressable.
|
||||
: "${CTX_SIZE:=602112}"
|
||||
: "${PARALLEL:=3}"
|
||||
# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves).
|
||||
# 2048 is the measured CUDA optimum (MiMo era, same arch/shape: +4.6 % prefill
|
||||
# at 8.9k depth, decode flat, 616 MiB). The logical batch is VRAM- and
|
||||
# throughput-neutral at a fixed ubatch. It ALSO sets the compute-buffer size,
|
||||
# which is the term that decides whether an MTP draft context fits beside a KV
|
||||
# pool: 1024 hands back 1,306 MiB and was the arm that ran the drafter at three
|
||||
# q8_0 windows (frank-zealot.env carries the table).
|
||||
: "${BATCH_SIZE:=4096}"
|
||||
: "${UBATCH_SIZE:=2048}"
|
||||
# Exact KV: f16 K + f16 V — no quantization error anywhere in the attention
|
||||
# state. 32 KiB/token at this geometry (8 full-attention layers × 4 KV heads ×
|
||||
# 256 dim × 2 B × 2 caches); 16,384.00 MiB at the 524,288 pool. That is the whole
|
||||
# reason the slot count is 2: the same dtype at the old 786,432 pool is 24,576
|
||||
# MiB and dies at cudaMalloc. q8_0 K + q8_0 V (17,408 B/token, 13,056.00 MiB at
|
||||
# 786,432) is the documented fallback rung and funds three windows; CACHE_TYPE_V
|
||||
# alone is the halfway door. Explicit either way so the contract is visible in
|
||||
# the argv, and --flash-attn stays pinned because quantized V — the fallback —
|
||||
# is illegal without it (no true fp8 KV dtype exists in this build).
|
||||
# The logical batch is VRAM- and throughput-neutral at a fixed ubatch, so 6144
|
||||
# (1024×6) is free; the ubatch is what sets the compute-buffer size, and 1024 is
|
||||
# the rung that leaves room for the draft context (−~1,306 MiB against 2048, and
|
||||
# the measured MTP arm ran here: 213.5 t/s at 1024 against 109.2 at 2048).
|
||||
: "${BATCH_SIZE:=6144}"
|
||||
: "${UBATCH_SIZE:=1024}"
|
||||
# Asymmetric KV: f16 K (the engine's own default, spelled out so the contract is
|
||||
# visible in argv) + q8_0 V. 8,192 elements per token per cache (8 full-attention
|
||||
# layers × 4 KV heads × 256 dim) ⇒ 16,384 B/token for K and 8,704 for V (q8_0 is
|
||||
# 1.0625 B/element — llama.cpp has no true fp8 KV dtype), 25,088 B/token and
|
||||
# 14,406.00 MiB at the 602,112 pool. That is the halfway door between exact KV
|
||||
# (32,768 B/token, which costs the third conversation and the drafter both) and
|
||||
# the fully quantized pair (17,408 B/token, which costs retrieval fidelity in the
|
||||
# keys); --flash-attn stays pinned because quantized V is illegal without it.
|
||||
: "${CACHE_TYPE_K:=f16}"
|
||||
: "${CACHE_TYPE_V:=f16}"
|
||||
# Speculative decoding defaults OFF, measured on this card and shape: the head
|
||||
# IS in the checkpoint (qwen35.nextn_predict_layers = 1 — MTP is a property of
|
||||
# the FILE, so the plain PQ2_0 sibling could not run it at any flag value), it
|
||||
# loads, and it accepts well (mean accepted length 2.41) — but the draft context
|
||||
# bills its own f16 KV over the WHOLE pool (4,096 B/position, 2,048 MiB at the
|
||||
# 524,288 pool) and at ubatch 2048 that squeeze halves decode: 167.2 → 109.2 t/s
|
||||
# on the q8_0 3-slot shape. The draft cache dtype IS a knob (--spec-draft-type-k
|
||||
# / --spec-draft-type-v, both default f16 — verified in this build's --help), so
|
||||
# an armed drafter can halve its own bill; there is no --spec-draft-model because
|
||||
# the draft rides the main model's weights. Card gate: quantized V alongside MTP
|
||||
# needs fork build >= prism-b10776, which is why LLAMA_BIN moved to the 10777
|
||||
# prebuilt; exact KV removes that gate but NOT the type-142 loader lock.
|
||||
# SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 is the documented MTP arm (213.5 t/s);
|
||||
# SPEC_TYPE=none strips the whole set (flags move together).
|
||||
: "${SPEC_TYPE:=none}"
|
||||
: "${CACHE_TYPE_V:=q8_0}"
|
||||
# Speculative decoding defaults ON: the head IS in the checkpoint
|
||||
# (qwen35.nextn_predict_layers = 1 — MTP is a property of the FILE, so the plain
|
||||
# PQ2_0 sibling could not run it at any flag value) and the draft context rides
|
||||
# the MAIN model's weights, so there is no --spec-draft-model. Its bill is the
|
||||
# target's whole pool at 4,096 B/position plus ~0.45 GiB of buffers at this
|
||||
# ubatch and a tripled recurrent state; CACHE_TYPE_V=q8_0 and UBATCH_SIZE=1024
|
||||
# above are what pay for it. The draft cache dtype IS a further knob
|
||||
# (--spec-draft-type-k / --spec-draft-type-v, both default f16 — verified in this
|
||||
# build's --help) and stays at its default here. Card gate: quantized V alongside
|
||||
# MTP needs fork build >= prism-b10776, which is why LLAMA_BIN is the 10777
|
||||
# prebuilt; the type-142 loader lock keeps it pinned regardless.
|
||||
# SPEC_TYPE=none is the rollback and strips the whole set (flags move together).
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${SPEC_DRAFT_N_MAX:=2}"
|
||||
: "${SPEC_DRAFT_N_MIN:=0}"
|
||||
: "${SPEC_DRAFT_P_MIN:=0}"
|
||||
|
||||
@@ -4,15 +4,15 @@
|
||||
# qwen35 hybrid with 32 layers (8 full-attention + 24 Gated DeltaNet) and the
|
||||
# MTP draft head INSIDE the file — via the PrismML fork build of llama.cpp
|
||||
# (vendor build 10777), the ONLY loader that registers the payload's custom
|
||||
# tensor type 142 and honours its prism.hadamard.* metadata: TWO 262,144-token
|
||||
# conversations (--ctx-size 524288 --parallel 2) over an UNQUANTIZED f16 KV
|
||||
# cache. Exact KV bills 32,768 B/token here, so the pool is what pays for it —
|
||||
# the same dtype at the old 786,432 pool is 24,576 MiB against the 24,123 MiB
|
||||
# the driver leaves addressable, and the slot count is the term that moved. The
|
||||
# publisher's own Q8_0 vision projector rides along (the BF16 one can crash on
|
||||
# concurrent images), and MTP stays OFF: the head is present, but exact KV
|
||||
# leaves no room for a draft context that bills f16 KV over the whole pool. Log
|
||||
# verbosity 4 ships so the VRAM-fit forensics land in the journal.
|
||||
# tensor type 142 and honours its prism.hadamard.* metadata: THREE 200,704-token
|
||||
# conversations (--ctx-size 602112 --parallel 3) over an f16 K + q8_0 V cache —
|
||||
# 25,088 B/token, 14,406.00 MiB at this pool — with the MTP drafter ARMED
|
||||
# (--spec-type draft-mtp). The asymmetry IS the shape: exact V (32,768 B/token)
|
||||
# leaves nothing for a draft context that bills its own f16 KV over the WHOLE
|
||||
# pool (4,096 B/position, 2,352.00 MiB here), and the 786,432 pool this shape
|
||||
# replaced cannot carry a drafter at any dtype inside the 24,123 MiB the driver
|
||||
# leaves addressable. The publisher's own Q8_0 vision projector rides along (the
|
||||
# BF16 one can crash on concurrent images). Log
|
||||
# This seam runs the serve script against a stub engine and asserts on the
|
||||
# argv it actually execs, so what is pinned is the command the operator gets.
|
||||
|
||||
@@ -143,7 +143,7 @@ run_stage() {
|
||||
done
|
||||
}
|
||||
|
||||
@test "two conversations share the pool at a 262144 window each" {
|
||||
@test "three conversations share the pool at a 200704 window each" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --alias)" = "frank-zealot" ]
|
||||
@@ -152,30 +152,36 @@ run_stage() {
|
||||
[ "$(argv_value --model)" = "$MODEL_DIR/$MODEL_STUB" ]
|
||||
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
||||
# llama.cpp divides --ctx-size across slots, so the observable contract is
|
||||
# the per-slot window. Exact KV costs the SLOT COUNT, not the window: every
|
||||
# conversation still gets the model's TRAINED 262,144, and the harness
|
||||
# contextWindow must quote that number — the pool and the slot count are
|
||||
# neither, and a longer prompt is a 400, never a truncation.
|
||||
[ "$(argv_value --parallel)" -eq 2 ]
|
||||
[ "$(argv_value --ctx-size)" -eq 524288 ]
|
||||
local pool slots
|
||||
# the per-slot window: harness contextWindow, Prometheus dashboards and every
|
||||
# consumer's compaction trigger read THAT number, and a longer prompt is a
|
||||
# 400, never a truncation. Two invariants the pool/slot split has to keep:
|
||||
# it divides evenly (llama.cpp floors it otherwise) and a slot never exceeds
|
||||
# the model's TRAINED window (there is no YaRN here, so the engine would be
|
||||
# quoting a window it cannot fill).
|
||||
[ "$(argv_value --parallel)" -eq 3 ]
|
||||
[ "$(argv_value --ctx-size)" -eq 602112 ]
|
||||
local pool slots window
|
||||
pool="$(argv_value --ctx-size)"; slots="$(argv_value --parallel)"
|
||||
[ $((pool % slots)) -eq 0 ]
|
||||
[ $((pool / slots)) -eq 262144 ]
|
||||
window=$((pool / slots))
|
||||
[ "$window" -eq 200704 ]
|
||||
[ "$window" -le 262144 ]
|
||||
}
|
||||
|
||||
@test "the KV dtype pair follows the env file and flash-attn stays pinned on" {
|
||||
# Asserted against the env rather than a literal so a retune moves one
|
||||
# file, not two. The shipped pair is f16/f16: exact attention state, and the
|
||||
# slot count is what makes it affordable — this payload bills 8,192 elements
|
||||
# per token per cache (8 full-attention layers × 4 KV heads × 256 dim), so
|
||||
# exact KV is 32,768 B/token = 16,384.00 MiB at the 524,288 pool, where the
|
||||
# q8_0 pair this file used to ship billed 17,408 B/token = 13,056.00 MiB at
|
||||
# the old 786,432 pool (ledger-exact both times). The 24 DeltaNet layers
|
||||
# file, not two. The shipped pair is ASYMMETRIC — f16 K (the engine's own
|
||||
# default) plus q8_0 V: this payload bills 8,192 elements per token per cache
|
||||
# (8 full-attention layers × 4 KV heads × 256 dim), so K costs 16,384 B/token
|
||||
# and V 8,704 (q8_0 is 1.0625 B/element; there is no true fp8 KV dtype in
|
||||
# this build) = 25,088 B/token = 14,406.00 MiB at the 602,112 pool. Exact V
|
||||
# would bill 32,768 B/token and the all-q8_0 pair this file used to ship
|
||||
# billed 17,408 — the halfway rung is what pays for the third slot AND the
|
||||
# armed drafter (the fit gate below prices both). The 24 DeltaNet layers
|
||||
# bill a fixed recurrent state per slot (~50.25 MiB a cell), which is why
|
||||
# pool arithmetic alone under-prices the shape. flash-attn is not
|
||||
# decoration: the quantized-V fallback is only legal with it, so it stays
|
||||
# explicit and a dtype retune cannot silently fall back.
|
||||
# decoration: quantized V is only legal with it, so it stays explicit and a
|
||||
# dtype retune cannot silently fall back.
|
||||
local env_k env_v
|
||||
env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
||||
env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
||||
@@ -192,30 +198,47 @@ run_stage() {
|
||||
# The fit is arithmetic off the payload geometry plus the non-KV terms
|
||||
# measured off the boot ledger (weights 2,036.71 + compute buffers 1,409.02
|
||||
# + projector 595 + 248.10 + CUDA context ≈ 460 = 4,749 MiB, dtype- and
|
||||
# slot-independent at this ubatch), plus one ~51 MiB recurrent-state cell
|
||||
# per slot. 24,123 MiB is what the driver leaves addressable (nvidia-smi
|
||||
# reports it as used + free; its total column reads 24,576).
|
||||
local env_k env_v pool slots rate_k rate_v kv total
|
||||
# slot-independent at ubatch 2048), plus one ~51 MiB recurrent-state cell per
|
||||
# slot. That constant is deliberately NOT credited with the ~1,306 MiB the
|
||||
# shipped ubatch 1024 hands back: the gate stays conservative.
|
||||
# An ARMED drafter is billed, because it is the term this shape pays
|
||||
# CACHE_TYPE_V=q8_0 to afford: the draft context is built against the
|
||||
# TARGET's whole pool at its own f16 rate (measured 4,096 B/position at
|
||||
# blk.32) plus buffers and its own recurrent cells — the measured spec-ON
|
||||
# against spec-OFF delta at the 786,432 / ubatch-1024 arm was 3,524 MiB
|
||||
# beside 3,072 MiB of draft KV, i.e. ~452 MiB, doubled here for the third
|
||||
# slot's extra cells. 24,123 MiB is what the driver leaves addressable
|
||||
# (nvidia-smi reports it as used + free; its total column reads 24,576).
|
||||
local env_k env_v pool slots rate_k rate_v kv total spec f16 exact_pool exact_v
|
||||
env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
||||
env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
||||
pool="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
slots="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
spec="$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
rate_k="$(kv_bytes_per_token "$env_k")"
|
||||
rate_v="$(kv_bytes_per_token "$env_v")"
|
||||
f16="$(kv_bytes_per_token f16)"
|
||||
kv="$(( pool * (rate_k + rate_v) / 1048576 ))"
|
||||
total="$(( kv + 4749 + 51 * slots ))"
|
||||
if [ "$spec" != "none" ]; then
|
||||
total="$(( total + pool * 4096 / 1048576 + 768 ))"
|
||||
fi
|
||||
# The shipped arm has to leave real room: the projector's worst case is
|
||||
# charged AFTER the engine's own fit check, so a projection that lands on
|
||||
# the ceiling is a crash waiting for the first image request.
|
||||
[ "$total" -le $(( 24123 - 1024 )) ]
|
||||
|
||||
# And the gate must be able to REFUSE: the same exact KV at the pool this
|
||||
# shape left behind is 24,576 MiB of cache alone — a boot that dies before
|
||||
# /health ever answers. Proving the gate bites is what makes passing one
|
||||
# meaningful.
|
||||
local over
|
||||
over="$(( 786432 * (rate_k + rate_v) / 1048576 + 4749 + 51 * 3 ))"
|
||||
[ "$over" -gt 24123 ]
|
||||
# And the gate must be able to REFUSE, or passing above means nothing. Two
|
||||
# boots this shape can walk straight into: the retired dtype pair at the
|
||||
# retired pool (exact f16 at 786,432 = 24,576 MiB of cache alone, which died
|
||||
# at cudaMalloc), and the shipped pool with V moved back to f16 while the
|
||||
# drafter stays armed — 18,816 MiB of cache plus the draft's 2,352. Both die
|
||||
# before /health ever answers, and no health gate can see either.
|
||||
exact_pool="$(( 786432 * (rate_k + f16) / 1048576 + 4749 + 51 * 3 ))"
|
||||
[ "$exact_pool" -gt 24123 ]
|
||||
exact_v="$(( pool * (rate_k + f16) / 1048576 + 4749 + 51 * slots
|
||||
+ pool * 4096 / 1048576 + 768 ))"
|
||||
[ "$exact_v" -gt 24123 ]
|
||||
}
|
||||
|
||||
@test "log verbosity follows the env file" {
|
||||
@@ -277,48 +300,54 @@ run_stage() {
|
||||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||||
}
|
||||
|
||||
@test "speculation is off: no drafter flag reaches the argv" {
|
||||
# Frank, 2026-10-18: this file DOES carry the MTP head (the boot arms prove
|
||||
# it loads and drafts, acceptance 0.798/0.617 per position, mean accepted
|
||||
# length 2.41) and it is still off, because the draft context bills its own
|
||||
# f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus
|
||||
# buffers — on a trunk that already projects 21,234 MiB with exact KV. The
|
||||
# MTP arms in frank-zealot.env are all q8_0-era measurements; the doors back
|
||||
# to the drafter are one window per conversation, or KV back to q8_0 with
|
||||
# UBATCH_SIZE=1024 (213.5 t/s). The flags move as a set:
|
||||
# SPEC_TYPE=none strips --spec-type, --spec-draft-n-max AND
|
||||
# --spec-draft-model together, so a no-spec boot cannot carry half a spec
|
||||
# pair (the ROCmFPX-era baseline arm segfaulted at load doing that).
|
||||
@test "the armed drafter ships the whole MTP set and no second copy of the trunk" {
|
||||
# Frank, 2026-10-19: the drafter is ON. The head is IN this file
|
||||
# (qwen35.nextn_predict_layers = 1, blk.32.nextn.*) and it accepted well
|
||||
# when it was measured off (0.798/0.617 per position, mean accepted length
|
||||
# 2.41); what it now pays for is CACHE_TYPE_V=q8_0 plus UBATCH_SIZE=1024,
|
||||
# which is why the dtype pair and the batch pair are pinned next door.
|
||||
# The observable contract is the SET: --spec-type plus every --spec-draft-*
|
||||
# the env names, and NO --spec-draft-model — the MTP draft rides the MAIN
|
||||
# model's weights, so a draft file would load the trunk a second time.
|
||||
local n_max
|
||||
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||||
[ "$(argv_value --spec-draft-n-max)" = "$n_max" ]
|
||||
[ "$(argv_value --spec-draft-n-min)" = "0" ]
|
||||
[ "$(argv_value --spec-draft-p-min)" = "0" ]
|
||||
! grep -qxF -- '--spec-draft-model' <<<"$output"
|
||||
}
|
||||
|
||||
@test "the rollback knob strips the entire spec set" {
|
||||
# SPEC_TYPE=none is the documented retreat (and the shape this endpoint
|
||||
# shipped before 2026-10-19). The law it exists to obey: the flags move as a
|
||||
# SET — a no-spec boot that keeps --spec-draft-n-max, or a baseline arm that
|
||||
# hands over --model-draft with no --spec-type, has segfaulted at load with
|
||||
# no diagnostic. So the rollback is proven by ABSENCE of all of them.
|
||||
serve_argv
|
||||
sed -i 's/^SPEC_TYPE=.*/SPEC_TYPE=none/' "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
! grep -qxF -- '--spec-type' <<<"$output"
|
||||
! grep -qxF -- '--spec-draft-n-max' <<<"$output"
|
||||
! grep -qxF -- '--spec-draft-n-min' <<<"$output"
|
||||
! grep -qxF -- '--spec-draft-p-min' <<<"$output"
|
||||
! grep -qxF -- '--spec-draft-model' <<<"$output"
|
||||
grep -qE '^SPEC_TYPE=none$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "naming a drafter ships the MTP set, with or without a separate draft file" {
|
||||
# The documented MTP arm (frank-zealot.env): SPEC_TYPE=draft-mtp ALONE is
|
||||
# legal here because the head is IN the checkpoint and the draft context
|
||||
# rides the main model's weights — so --spec-draft-model must NOT appear,
|
||||
# or the trunk would be loaded twice.
|
||||
@test "an external drafter still reaches the engine as a complete set" {
|
||||
# The other arm the ROCmFPX-era crash was about: a named draft FILE must
|
||||
# arrive alongside --spec-type and the depth recipe, never alone. This
|
||||
# endpoint does not use it (the head is in the trunk), so the knob ships
|
||||
# empty and only this test sets it.
|
||||
serve_argv
|
||||
{
|
||||
echo 'SPEC_TYPE=draft-mtp'
|
||||
echo 'SPEC_DRAFT_N_MAX=2'
|
||||
} >> "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||||
[ "$(argv_value --spec-draft-n-max)" = "2" ]
|
||||
! grep -qxF -- '--spec-draft-model' <<<"$output"
|
||||
|
||||
# And an external drafter still reaches the engine as a complete set — the
|
||||
# pair the ROCmFPX-era crash was about.
|
||||
echo "SPEC_DRAFT_FILE=$MODEL_DIR/$DRAFT_STUB" >> "$STAGE/test.env"
|
||||
run_stage
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --spec-draft-model)" = "$MODEL_DIR/$DRAFT_STUB" ]
|
||||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||||
}
|
||||
|
||||
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
||||
|
||||
Reference in new issue
Block a user