diff --git a/AGENTS.md b/AGENTS.md index 98e80aa..aa2d72f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -384,10 +384,11 @@ the publisher's own **Q8_0** mmproj passed as `--mmproj` — Q8_0 is the publisher's requirement for concurrent image requests (the card states the BF16 tower can crash `llama-server` when several image requests run at once, and this endpoint runs several slots). `frank-zealot`'s `contextWindow` is -**262144** — the PER-SLOT window (`--ctx-size 524288 --parallel 2` since the -2026-10-18 reslice to exact f16 KV), which EQUALS the model's -native window; llama.cpp answers a longer prompt with a 400 naming the limit, -never a truncation, so the harness must compact before it. +**200704** — the PER-SLOT window (`--ctx-size 602112 --parallel 3` since the +2026-10-19 reslice that funded a third slot AND an armed MTP drafter), +deliberately below the model's native window; llama.cpp answers a longer prompt +with a 400 naming the limit, never a truncation, so the harness must compact +before it. Note also that the reasoning channel differs by runtime: llama.cpp AND SGLang (qwen3 @@ -435,10 +436,11 @@ disagrees with it is the bug. loader ignoring that metadata dequantizes wrong); payload `Bonsai2-9B-MTP/…PQ2_0.gguf` + the publisher's **Q8_0** mmproj (the BF16 tower can crash the server with concurrent image requests). -- Shape: `--ctx-size 524288 --parallel 2` = **two conversations at the - model's native 262,144**, `CACHE_TYPE_K/V=f16`, batch 4096 / ubatch 2048, - `SPEC_TYPE=none`, `--flash-attn on`, `--jinja`, `--metrics`. -- Harness/gate contract: `contextWindow: 262144` is the **per-slot** window +- Shape: `--ctx-size 602112 --parallel 3` = **three conversations at + 200,704 each**, `CACHE_TYPE_K=f16` + `CACHE_TYPE_V=q8_0` (asymmetric: 25,088 + B/token, 14,406 MiB of pool), batch 6144 / ubatch 1024, + `SPEC_TYPE=draft-mtp` (depth 2), `--flash-attn on`, `--jinja`, `--metrics`. +- Harness/gate contract: `contextWindow: 200704` is the **per-slot** window (`n_ctx_slot = CTX_SIZE / PARALLEL`), and llama.cpp answers a longer prompt with a 400 rather than truncating. `--metrics` is mandatory or Prometheus reads 501 = DOWN beside a healthy service. Presence penalty has no GGUF diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index d6feae9..d0d8e6c 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1459,7 +1459,7 @@ LLM_ZEALOT_RUNTIME="${LLM_ZEALOT_RUNTIME:-/home/dank/GitPackages/llama-prism-b10 # image path — a deploy that starts without it silently ships a text-only # endpoint. Checked unconditionally, like frank-dragoon's. Q8_0 is MANDATORY, # not a preference: the card says the BF16 tower can crash llama-server when -# several image requests run at once, and this endpoint runs --parallel 2. Its +# several image requests run at once, and this endpoint runs --parallel 3. Its # 868.32 MiB worst-case VRAM (595.29 model + 248.10 compute in the boot ledger) # lands AFTER the engine's fit check, so it is not inside the q8_0 budget. LLM_ZEALOT_MMPROJ="${LLM_ZEALOT_MMPROJ:-/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf}" diff --git a/scripts/frank-zealot.env b/scripts/frank-zealot.env index 48608ab..be1037f 100644 --- a/scripts/frank-zealot.env +++ b/scripts/frank-zealot.env @@ -11,35 +11,48 @@ LLAMA_BIN=/home/dank/GitPackages/llama-prism-b10777/llama-server # metadata dequantizes wrong, silently. MODEL_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-PQ2_0-MTP-Q8_0.gguf # Q8_0 IS MANDATORY: the publisher states the BF16 tower can crash llama-server -# under concurrent image requests, which is exactly --parallel 2. +# under concurrent image requests, which is exactly --parallel 3. MMPROJ_FILE=/home/dank/models/Bonsai2-9B-MTP/Ternary-Bonsai-2-9B-mmproj-Q8_0.gguf # Served id — the harness/family name, not a fact about the payload. # Id-keyed consumers: OpenWebUI (model row, access_grant, api_configs), models.yml. ALIAS=frank-zealot BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# CTX_SIZE is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144 a -# conversation, each still at the trained window. MUST stay divisible by -# PARALLEL (the seam enforces it) and models.yml contextWindow quotes the -# PER-SLOT number — a longer prompt is a 400, never a truncation. The pool -# cannot go back to 786,432 under exact f16 KV: 24,576 MiB of cache against the -# 24,123 MiB the driver leaves addressable, before a weight loads. -CTX_SIZE=524288 -PARALLEL=2 -BATCH_SIZE=4096 +# CTX_SIZE is the POOL llama.cpp divides across slots: 602,112 / 3 = 200,704 a +# conversation. MUST stay divisible by PARALLEL (the seam enforces it) and +# models.yml contextWindow quotes the PER-SLOT number — a longer prompt is a +# 400, never a truncation. The pool is bought, not granted: K+V at 25,088 +# B/token is 14,406.00 MiB here and the MTP draft context bills its own f16 KV +# over the WHOLE pool (4,096 B/position ⇒ 2,352.00 MiB), so the old 786,432 +# pool cannot carry a drafter at any KV dtype — 18,816 + 3,072 of cache alone +# against the 24,123 MiB the driver leaves addressable. +CTX_SIZE=602112 +PARALLEL=3 +# The logical batch is VRAM- and throughput-neutral at a fixed ubatch (the law +# on [[frank-zealot]]); 1024×6 keeps it a clean multiple of the ubatch. +BATCH_SIZE=6144 # Ubatch is a prefill knob (decode never moves with it) and it sets the -# compute-buffer size — which is what decides whether a drafter fits. -# UBATCH_SIZE=1024 is the documented MTP arm. -UBATCH_SIZE=2048 -# Exact K and V: the retrieval-fidelity ask, paid with one conversation of -# concurrency. Rate law and the arithmetic: [[frank-zealot]] → Laws. +# compute-buffer size — which is what decides whether a drafter fits: 1024 +# hands back ~1,306 MiB beside the draft context, and it is the rung the +# measured MTP arm ran (213.5 t/s at 1024 against 109.2 at 2048). +UBATCH_SIZE=1024 +# K exact, V in the 8-bit class: 16,384 + 8,704 = 25,088 B/token (8,192 +# elements per token per cache × 2 B for f16, × 1.0625 for q8_0 — there is no +# true fp8 KV dtype in this build). That 7,680 B/token saved against exact V is +# what funds BOTH the third conversation and the drafter. Quantized V is +# illegal without --flash-attn, which the serve script pins. Rate law and the +# arithmetic: [[frank-zealot]] → Laws. CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 -# Speculation OFF — measured, not assumed: the draft context bills its own f16 -# KV over the WHOLE pool, and there is no room beside exact KV (the arm census -# is on the page). SPEC_TYPE=none strips the WHOLE set (--spec-type and every -# --spec-draft-*); half a set has segfaulted at load. -SPEC_TYPE=none +CACHE_TYPE_V=q8_0 +# Speculation ON (Frank, 2026-10-19): the head is in the file +# (qwen35.nextn_predict_layers = 1, blk.32.nextn.*) and rides the MAIN model's +# weights, so there is no --spec-draft-model. Its bill is pool × 4,096 B of +# draft KV plus ~0.45 GiB of buffers and a tripled recurrent state — paid for +# by CACHE_TYPE_V=q8_0 and UBATCH_SIZE=1024 above. Build gate: quantized V +# beside MTP needs fork build >= b10776; LLAMA_BIN is 10777. +# SPEC_TYPE=none is the rollback and strips the WHOLE set (--spec-type and +# every --spec-draft-*); half a set has segfaulted at load. +SPEC_TYPE=draft-mtp SPEC_DRAFT_N_MAX=2 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0 diff --git a/scripts/serve-frank-zealot.sh b/scripts/serve-frank-zealot.sh index 0302aca..29e3348 100755 --- a/scripts/serve-frank-zealot.sh +++ b/scripts/serve-frank-zealot.sh @@ -5,33 +5,43 @@ # # 2026-10-18 (Frank): Bonsai-2 27B PQ2_0 → the 9B MTP export, then the same # evening TWO 262,144-token conversations (--ctx-size 524288 --parallel 2) over -# an UNQUANTIZED f16 KV cache. The 9B bills 8,192 elements per token per cache -# (8 full-attention layers instead of the 27B's 16), so exact K and V costs -# 32,768 B/token — 16,384.00 MiB at the 524,288 pool, which is what a 24,123 MiB -# addressable card can carry beside the trunk. Verbosity 4 ships so the VRAM-fit -# forensics (KV layout, buffer sizes, graph capture) land in the journal. +# an UNQUANTIZED f16 KV cache. +# +# 2026-10-19 (Frank): THREE 200,704-token conversations over a 602,112 pool +# (--ctx-size 602112 --parallel 3), K left at the engine's own default (f16), +# V dropped to q8_0, batch 1024×6 / ubatch 1024, and the MTP drafter ARMED +# (--spec-type draft-mtp). The 9B bills 8,192 elements per token per cache (8 +# full-attention layers instead of the 27B's 16), so f16 K is 16,384 B/token and +# q8_0 V is 8,704: 25,088 B/token = 14,406.00 MiB at the 602,112 pool. That +# 7,680 B/token against exact V is exactly what funds the third conversation +# AND a draft context that bills f16 KV over the whole pool. Verbosity 4 ships +# so the VRAM-fit forensics (KV layout, buffer sizes, graph capture) land in +# the journal. # # Architecture (engine ledger, not the marketing page): qwen35 hybrid, 32 # layers with qwen35.full_attention_interval = 4 ⇒ 8 full-attention + 24 Gated # DeltaNet, 4 KV heads × 256 dim, native 262,144. The KV pool scales with the 8 -# full layers (16,384.00 MiB of f16 at the 524,288 pool); the DeltaNet layers -# cost a fixed recurrent state per SLOT — ~50.25 MiB a cell, so 100.50 MiB for -# two. The runtime is a hard lock, not a preference: custom GGML type 142 plus -# prism.hadamard.* metadata, which only the vendor build registers (the b10776 -# gate that moved the prebuilt 10754 → 10777 was MTP beside quantized V, which -# exact KV now makes moot — the type-142 lock alone keeps the build pinned). +# full layers (14,406.00 MiB of f16 K + q8_0 V at the 602,112 pool); the +# DeltaNet layers cost a fixed recurrent state per SLOT — ~50.25 MiB a cell, so +# 150.75 MiB for three, and an armed depth-2 drafter triples that again. The +# runtime is a hard lock, not a preference: custom GGML type 142 plus +# prism.hadamard.* metadata, which only the vendor build registers — and the +# b10776 gate that moved the prebuilt 10754 → 10777 (MTP beside a quantized V) +# is LOAD-BEARING again on this shape, not moot as it was under exact KV. # Vision comes from the publisher's Q8_0 projector (the BF16 one can crash the -# server on concurrent images, and this endpoint runs two slots), so --mmproj is +# server on concurrent images, and this endpoint runs three slots), so --mmproj is # what gives this endpoint eyes; MMPROJ_FILE="" is the documented text-only # opt-out. # -# Speculative decoding ships OFF, measured: the head loads and accepts well -# (0.798/0.617 per position, mean length 2.41) but the draft context bills its -# own f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus -# buffers — and the trunk already projects 21,234 MiB with exact KV. Exact KV -# and MTP cannot share this card; frank-zealot.env carries the q8_0-era arm table -# and the two doors back to the drafter (one window, or KV back to q8_0 at -# --ubatch-size 1024 for 213.5 t/s). +# Speculative decoding ships ON (Frank, 2026-10-19). The head is IN the file and +# accepts well (0.798/0.617 per position, mean length 2.41, measured 2026-10-18); +# what made it lose that evening was the bill, not the drafts — the draft context +# is built against the target's WHOLE pool, so blk.32 adds 4,096 B/position +# (2,352.00 MiB at 602,112) plus ~0.45 GiB of buffers at this ubatch, and the +# recurrent state triples. Dropping V to q8_0 (−3,840 MiB at this pool) and the +# ubatch to 1024 (−~1,306 MiB of compute buffers) is how those two bills got +# paid; the arm census and the rates are on [[frank-zealot]]. SPEC_TYPE=none is +# the rollback and strips the whole flag set. # # Sampling: the checkpoint carries the publisher's thinking recipe as GGUF # metadata (general.sampling.* = temp 1.0 / top_p 0.95 / top_k 20 / min_p 0 / @@ -58,49 +68,46 @@ fi : "${ALIAS:=frank-zealot}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -# ctx-size is the POOL llama.cpp divides across slots: 524,288 / 2 = 262,144 -# per conversation — every slot still at the model's trained window. The harness -# contextWindow MUST quote the PER-SLOT number — a longer prompt is answered -# with a 400, never truncated. Under exact KV the pool cannot stay at 786,432: -# 32,768 B/token × 786,432 = 24,576 MiB, more than the card's whole addressable -# 24,123 MiB before a single weight loads. -: "${CTX_SIZE:=524288}" -: "${PARALLEL:=2}" +# ctx-size is the POOL llama.cpp divides across slots: 602,112 / 3 = 200,704 a +# conversation. The harness contextWindow MUST quote the PER-SLOT number — a +# longer prompt is answered with a 400, never truncated. The pool is priced by +# the KV bill, not by the trained window: at 25,088 B/token (f16 K + q8_0 V) the +# 602,112 pool costs 14,406.00 MiB and the armed drafter adds its own f16 KV over +# the whole pool (2,352.00 MiB). Exact V at this pool (18,816 MiB) or the old +# 786,432 pool at any dtype cannot boot inside the 24,123 MiB the driver leaves +# addressable. +: "${CTX_SIZE:=602112}" +: "${PARALLEL:=3}" # Compute-buffer geometry. Ubatch is a prefill knob (decode never moves). -# 2048 is the measured CUDA optimum (MiMo era, same arch/shape: +4.6 % prefill -# at 8.9k depth, decode flat, 616 MiB). The logical batch is VRAM- and -# throughput-neutral at a fixed ubatch. It ALSO sets the compute-buffer size, -# which is the term that decides whether an MTP draft context fits beside a KV -# pool: 1024 hands back 1,306 MiB and was the arm that ran the drafter at three -# q8_0 windows (frank-zealot.env carries the table). -: "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=2048}" -# Exact KV: f16 K + f16 V — no quantization error anywhere in the attention -# state. 32 KiB/token at this geometry (8 full-attention layers × 4 KV heads × -# 256 dim × 2 B × 2 caches); 16,384.00 MiB at the 524,288 pool. That is the whole -# reason the slot count is 2: the same dtype at the old 786,432 pool is 24,576 -# MiB and dies at cudaMalloc. q8_0 K + q8_0 V (17,408 B/token, 13,056.00 MiB at -# 786,432) is the documented fallback rung and funds three windows; CACHE_TYPE_V -# alone is the halfway door. Explicit either way so the contract is visible in -# the argv, and --flash-attn stays pinned because quantized V — the fallback — -# is illegal without it (no true fp8 KV dtype exists in this build). +# The logical batch is VRAM- and throughput-neutral at a fixed ubatch, so 6144 +# (1024×6) is free; the ubatch is what sets the compute-buffer size, and 1024 is +# the rung that leaves room for the draft context (−~1,306 MiB against 2048, and +# the measured MTP arm ran here: 213.5 t/s at 1024 against 109.2 at 2048). +: "${BATCH_SIZE:=6144}" +: "${UBATCH_SIZE:=1024}" +# Asymmetric KV: f16 K (the engine's own default, spelled out so the contract is +# visible in argv) + q8_0 V. 8,192 elements per token per cache (8 full-attention +# layers × 4 KV heads × 256 dim) ⇒ 16,384 B/token for K and 8,704 for V (q8_0 is +# 1.0625 B/element — llama.cpp has no true fp8 KV dtype), 25,088 B/token and +# 14,406.00 MiB at the 602,112 pool. That is the halfway door between exact KV +# (32,768 B/token, which costs the third conversation and the drafter both) and +# the fully quantized pair (17,408 B/token, which costs retrieval fidelity in the +# keys); --flash-attn stays pinned because quantized V is illegal without it. : "${CACHE_TYPE_K:=f16}" -: "${CACHE_TYPE_V:=f16}" -# Speculative decoding defaults OFF, measured on this card and shape: the head -# IS in the checkpoint (qwen35.nextn_predict_layers = 1 — MTP is a property of -# the FILE, so the plain PQ2_0 sibling could not run it at any flag value), it -# loads, and it accepts well (mean accepted length 2.41) — but the draft context -# bills its own f16 KV over the WHOLE pool (4,096 B/position, 2,048 MiB at the -# 524,288 pool) and at ubatch 2048 that squeeze halves decode: 167.2 → 109.2 t/s -# on the q8_0 3-slot shape. The draft cache dtype IS a knob (--spec-draft-type-k -# / --spec-draft-type-v, both default f16 — verified in this build's --help), so -# an armed drafter can halve its own bill; there is no --spec-draft-model because -# the draft rides the main model's weights. Card gate: quantized V alongside MTP -# needs fork build >= prism-b10776, which is why LLAMA_BIN moved to the 10777 -# prebuilt; exact KV removes that gate but NOT the type-142 loader lock. -# SPEC_TYPE=draft-mtp + UBATCH_SIZE=1024 is the documented MTP arm (213.5 t/s); -# SPEC_TYPE=none strips the whole set (flags move together). -: "${SPEC_TYPE:=none}" +: "${CACHE_TYPE_V:=q8_0}" +# Speculative decoding defaults ON: the head IS in the checkpoint +# (qwen35.nextn_predict_layers = 1 — MTP is a property of the FILE, so the plain +# PQ2_0 sibling could not run it at any flag value) and the draft context rides +# the MAIN model's weights, so there is no --spec-draft-model. Its bill is the +# target's whole pool at 4,096 B/position plus ~0.45 GiB of buffers at this +# ubatch and a tripled recurrent state; CACHE_TYPE_V=q8_0 and UBATCH_SIZE=1024 +# above are what pay for it. The draft cache dtype IS a further knob +# (--spec-draft-type-k / --spec-draft-type-v, both default f16 — verified in this +# build's --help) and stays at its default here. Card gate: quantized V alongside +# MTP needs fork build >= prism-b10776, which is why LLAMA_BIN is the 10777 +# prebuilt; the type-142 loader lock keeps it pinned regardless. +# SPEC_TYPE=none is the rollback and strips the whole set (flags move together). +: "${SPEC_TYPE:=draft-mtp}" : "${SPEC_DRAFT_N_MAX:=2}" : "${SPEC_DRAFT_N_MIN:=0}" : "${SPEC_DRAFT_P_MIN:=0}" diff --git a/serverAppConfigs-docs/LLM/frank-zealot.md b/serverAppConfigs-docs/LLM/frank-zealot.md index 5028636..5d2221f 100644 --- a/serverAppConfigs-docs/LLM/frank-zealot.md +++ b/serverAppConfigs-docs/LLM/frank-zealot.md @@ -1,6 +1,6 @@ --- type: reference -title: "frank-zealot on Dankclaw — llama.cpp (PrismML fork 10777), Ternary-Bonsai-2-9B PQ2_0-MTP, 2 × 262144, exact f16 KV" +title: "frank-zealot on Dankclaw — llama.cpp (PrismML fork 10777), Ternary-Bonsai-2-9B PQ2_0-MTP, 3 × 200704, f16 K + q8_0 V, MTP armed" description: "The one frank-zealot reference: the live shape on Dankclaw's RTX 3090 (:7771) exactly as scripts/frank-zealot.env ships it, every era with its rollback path, the deduplicated laws of this card, the wire contract, and the owning files." tags: - llm @@ -12,10 +12,11 @@ tags: timestamp: 2026-10-18 --- -# frank-zealot — llama.cpp (PrismML) Bonsai-2-9B PQ2_0-MTP, 2 × 262,144, exact f16 KV +# frank-zealot — llama.cpp (PrismML) Bonsai-2-9B PQ2_0-MTP, 3 × 200,704, f16 K + q8_0 V, MTP armed Dankclaw's RTX 3090 endpoint — `http://192.168.0.143:7771`, wire id `frank-zealot`, the fleet's **compaction hub** -(widest window it has). One page per endpoint: eras are rows below, not pages. Siblings: [[frank-archon]], +(three conversations at 200,704 each, the window the other two endpoints also cap at). One page per +endpoint: eras are rows below, not pages. Siblings: [[frank-archon]], [[frank-dragoon]]. Deploy with `./scripts/deploy.sh dankclaw --service frank-zealot`; swap mechanics in [[Model Swap and Rollback]], sizing math in [[Llama Cpp Serving Sizing]]. `scripts/frank-zealot.env` and `scripts/serve-frank-zealot.sh` are the authority for the shape; where any prose disagrees with them, the prose @@ -31,72 +32,62 @@ is the bug. | Runtime | `/home/dank/GitPackages/llama-prism-b10777/llama-server` — PrismML **fork**, `0.2.0-dev (build 10777, commit e3de71522)`; the only loader for this payload | | Trio | `scripts/frank-zealot.service` (root, relinked, `User=dank`, `Restart=on-failure`, `RestartSec=10`, `LimitMEMLOCK=infinity`, `TimeoutStopSec=180`) · `scripts/frank-zealot.env` · `scripts/serve-frank-zealot.sh` | | Address / alias | `BIND_ADDR=0.0.0.0`, `LISTEN_PORT=7771`, `ALIAS=frank-zealot` (a flag, not a fact) | -| Flags | `--ctx-size 524288 --parallel 2` ⇒ 262,144 per slot · `--cache-type-k f16 --cache-type-v f16` (exact, **16,384 MiB**) · `--flash-attn on` · `--batch-size 4096 --ubatch-size 2048` · `--n-gpu-layers 99` · `--jinja --reasoning auto --reasoning-format deepseek` · `--metrics` · `--mmproj` · `--log-verbosity 4` · `--presence-penalty 1.0` · **`SPEC_TYPE=none`** | +| Flags | `--ctx-size 602112 --parallel 3` ⇒ 200,704 per slot · `--cache-type-k f16 --cache-type-v q8_0` (asymmetric, **14,406.00 MiB**) · `--flash-attn on` · `--batch-size 6144 --ubatch-size 1024` · `--n-gpu-layers 99` · `--jinja --reasoning auto --reasoning-format deepseek` · `--metrics` · `--mmproj` · `--log-verbosity 4` · `--presence-penalty 1.0` · **`--spec-type draft-mtp`** with `--spec-draft-n-max 2 --spec-draft-n-min 0 --spec-draft-p-min 0` | **Why this shape.** `general.architecture qwen35`: 33 blocks (32 trunk + `blk.32` = the MTP block), `full_attention_interval 4` ⇒ 8 full-attention + 24 Gated DeltaNet, 4 KV heads × 256 dim, `nextn_predict_layers 1` (the head **is in the file**), trained window 262,144, `general.sampling.*` 1.0/0.95/20. Only the 8 full layers bill -per token (32,768 B at exact f16 ⇒ 16,384 MiB over the pool) and the DeltaNet layers add ~50 MiB of recurrent state -**per slot**: exact KV costs the third conversation and nothing else. +per token and the DeltaNet layers add ~50 MiB of recurrent state **per slot**, so the shape is a single budget +equation at 25,088 B/token (f16 K 16,384 + q8_0 V 8,704): **14,406.00 MiB** of cache over a 602,112 pool + +**2,352.00 MiB** of draft cache (blk.32 at its own f16 rate, billed over the WHOLE pool) + ~7.7 GiB of weights, +projector, buffers and states, inside the 24,123 MiB the driver leaves addressable. Exact V (32,768 B/token ⇒ +18,816 MiB here) buys neither the third conversation nor the drafter; the fully quantized pair (17,408 B/token) +buys both and costs the keys their precision. -**MTP is OFF by measurement, not by absence of a head.** The drafter is *good* — 2026-10-18 measured -acceptance 0.798 / 0.617 per position, mean accepted length 2.41, and the publisher's card says depth 2 works -best — and it still loses here, because the draft context is built against the *target's* pool: `blk.32` bills -4,096 B/position at its default f16 ⇒ 2,048 MiB at 524,288 positions on a trunk that projects 21,234 MiB of the -24,123 MiB addressable (that projection predicts 2,889 MiB free; replace it with the measured pair). Exact KV and a -drafter cannot share this card. `--spec-draft-type-k`/`--spec-draft-type-v` (default f16, verified in this build's -`--help`) would cut the draft to 1,088 MiB — legal on b10777, **never booted beside exact KV**. There is no -`--spec-draft-model`: the MTP draft rides the main model's weights. The env carries the recipe -(`SPEC_DRAFT_N_MAX=2`, `SPEC_DRAFT_N_MIN=0`, `SPEC_DRAFT_P_MIN=0`); `SPEC_TYPE=none` never sends it. +**MTP is ON (Frank, 2026-10-19).** The drafter was always good — 2026-10-18 measured acceptance 0.798 / 0.617 per +position, mean accepted length 2.41, and the publisher's card says depth 2 works best — and arming it is a +budget decision, not a flag: the draft context is built against the *target's* pool, so `blk.32` bills 4,096 +B/position (2,352.00 MiB at 602,112) plus ~0.45 GiB of buffers at this ubatch and its own recurrent cells. +`CACHE_TYPE_V=q8_0` (−3,840 MiB against exact V at this pool) and `UBATCH_SIZE=1024` (−~1,306 MiB of compute +buffers, and the rung every measured MTP arm here ran on) are what pay for it; `UBATCH_SIZE=2048` beside an armed +drafter is the shape that halved decode (167.2 → 109.2 t/s). `--spec-draft-type-k`/`--spec-draft-type-v` (default +f16, verified in this build's `--help`) would cut the draft's bill again and stay at their default; there is no +`--spec-draft-model` — the draft rides the main model's weights, so a draft file would load the trunk twice. +`SPEC_TYPE=none` is the rollback and strips the whole set. **Harness contract** (`~/.omp/agent/models.yml`, provider `llamacpp (dankclaw-3090)`, id `frank-zealot`): -`contextWindow: 262144` — **the per-slot window**, because llama.cpp divides `--ctx-size` across slots and answers a +`contextWindow: 200704` — **the per-slot window**, because llama.cpp divides `--ctx-size` across slots and answers a longer prompt with a 400, never a truncation — plus `maxTokens: 16384`, `input: [text, image]`, `compat.reasoningContentField: reasoning_content`, `compat.reasoningEffortMap` (**load-bearing**: the template takes only `xhigh|medium|low` and raises on anything else), `extraBody` 1.0 / 0.95 / 20 / `min_p 0`, and the `compactionModel` of [[frank-archon]], [[frank-dragoon]] **and** frank-zealot all pointing here. -**The exact-KV boot, measured** (2026-10-18 evening, under the unit, verbosity 4). KV **16,384.00 MiB** = -32,768 B/token × 524,288 positions, to the MiB — rate derivation and ledger agree with no slop, which is what -buries the doc's old "f16 needs 26,112 MiB and cannot boot" figure for good: that was the 27B's 34,816 B/token -applied to the 9B. RS **100.50 MiB** (2 cells, one 50.25 MiB cell lighter than the q8_0 era), compute -1,408.06 + 1,088.07 CUDA_Host, mmproj worst case **868.32 MiB** charged *after* the engine's own fit check, -settled `nvidia-smi` **21,176 used / 2,949 free** of 24,123, `n_slots = 2`, `n_ctx_slot = 262144`, -`DropInPaths` empty, no clamp line, no `cudaMalloc` error — within 24 MiB of the pre-flight projection. That is -a desktop margin: the ~2 GiB of slack is what an image storm has to live inside. - -**Rates on this shape are FLOORS, not the shape's rates** — measured the same evening greedy with `ignore_eos` -at 512 tokens, first request discarded, while another consumer drove the other slot (56 new prompts in three -minutes, 96 % GPU-util): the exact condition the bench window exists to exclude, and one contended deep arm -even came back truncated (a 512 KB body accepted as `prompt_n = 4`). - -| Arm | q8_0 3-slot record | f16 2-slot, contended | -|---|---|---| -| single-stream decode | 168.7 | **134.1 – 141.5** (145.8 and 163.2 in a lighter minute) | -| concurrent aggregate | ≈300 across 3 (102.8 each) | **256.6 / 262.4 / 265.1** across 2 (≈128–133 each) | -| prefill at ~106k | 2,877 at 106,418 | **3,159.5 at 106,282** | -| decode at 106k depth | never measured | **71.5** | - -What the floors already support: per-conversation concurrency **improved** (two streams hold ~130 t/s each where -three held 102.8), endpoint aggregate sits ~12 % under the 3-slot record, prefill beat the record while -contended, and single-stream decode is between parity and −20 %. The 1.882× KV byte bill is real; its true size -on this card is still unmeasured, so quoting 134 or 163 t/s as "the" number is a guess in either direction — -re-run the same arms with the box quiet before a figure is carried anywhere else. +**This shape ships unbenchmarked, by law.** Frank ordered the reslice, not a +number, so the evidence a swap needs is the seam's fit gate (it prices cache + +draft cache + recurrent states against the 24,123 MiB the driver leaves +addressable, and refuses the two boots this shape can walk into: exact V with the +drafter armed, and the retired dtype pair at the retired pool), the deployer's +health gate, and the first verbosity-4 ledger of the new dtype pair — which is +recorded on this page because projections drift, and nowhere else. The previous +era's exact-KV boot and its rate floors live in that era's row below; those floors +were measured while another consumer drove the other slot (56 prompts in three +minutes, 96 % GPU-util), the exact condition a bench window exists to exclude. **Fallback rungs**, cheapest first — one line of `frank-zealot.env` each, and each must keep the seam's fit gate green: -| Knob | Effect | +| Knob | Effect from the shipped shape | |---|---| -| `UBATCH_SIZE=1024` | hands back 1,306 MiB of compute buffers; prefill −17 %; it is also the arm that ran the drafter at three windows (213.5 t/s) | -| `CACHE_TYPE_V=q8_0` | exact K + quantized V: KV 16,384 → **12,544 MiB**, freeing 3,840 (llama.cpp has no true fp8 KV dtype — q8_0 is the 8-bit class) | -| `CTX_SIZE=401408` | two 200,704 conversations, exact KV at 12,544 MiB (same 3,840 freed) — `contextWindow` MUST follow the per-slot number | -| `CTX_SIZE=786432 PARALLEL=3 CACHE_TYPE_K/V=q8_0` | the era below this one: 13,056 MiB of cache, the third conversation back | -| `CACHE_TYPE_K/V=q4_0` | 10,240 B/token ⇒ 5,120 MiB (the 27B era measured q4_0 at 16,392 used / 7,733 free at 2 × 200,704) | +| `SPEC_TYPE=none` | the drafter off: hands back 2,352 MiB of draft cache plus its buffers and recurrent cells, at the cost of the ~+28 % decode the 2026-10-18 census measured whenever the card has room | +| `UBATCH_SIZE=2048` | +~1,306 MiB of compute buffers and +4.6 % prefill — and the rung that HALVED decode beside an armed drafter (167.2 → 109.2 t/s) | +| `CTX_SIZE=401408` | two 200,704 conversations: frees 4,802 MiB of cache — `contextWindow` MUST follow the per-slot number | +| `CACHE_TYPE_V=f16` | exact V: +4,410 MiB at this pool with the drafter armed, which is a `cudaMalloc` death the seam refuses — legal only once the drafter or a slot is gone | +| `CACHE_TYPE_K=q8_0` | the fully quantized pair (17,408 B/token): frees 4,410 MiB and moves the quantization error into the keys | ## Eras | Window | Payload | Runtime | Shape | Replaced because | Rollback | |---|---|---|---|---|---| +| 2026-10-18 21:06 → 2026-10-19 | the same 9B MTP file and projector | PrismML fork 10777 | 2 × 262,144 (pool 524,288), exact f16 K+V = 16,384.00 MiB, `SPEC_TYPE=none`, batch 4096 / ubatch 2048; boot ledger KV 16,384.00 / RS 100.50 / compute 1,408.06 + 1,088.07 CUDA_Host / mmproj worst case 868.32 charged after the fit check, settled **21,176 used / 2,949 free**, `n_ctx_slot = 262144`, within 24 MiB of the projection; floors 134.1–141.5 t/s single-stream, 256.6–265.1 aggregate over two, 3,159.5 t/s prefill at 106,282, 71.5 t/s decode at 106k depth (contended — floors, not the shape's rates) | Frank ordered the third conversation back AND the drafter armed, and exact V funds neither: it bills 18,816 MiB at the pool that pays for the third slot, leaving nothing for the draft's 2,352 | one line each — `CTX_SIZE=524288 PARALLEL=2 CACHE_TYPE_V=f16 BATCH_SIZE=4096 UBATCH_SIZE=2048 SPEC_TYPE=none` — and redeploy; no payload move | | 2026-10-18 13:19 → same evening (the boot `68afc6d` replaced) | the same 9B MTP file and projector | PrismML fork 10777 | 3 × 262,144 (pool 786,432), q8_0 K+V = 13,056 MiB, settled 17,956 / 6,169 MiB; 168.7 t/s decode (three runs 168.4–168.7), ~300 t/s over three streams (102.8 each), 2,877 t/s prefill at 106,418 | Frank ordered unquantized KV, and that pool cannot survive it: 24,576 MiB of f16 cache > 24,123 addressable | one env line — `CTX_SIZE=786432 PARALLEL=3 CACHE_TYPE_K/V=q8_0` — and redeploy; no payload move | | 2026-10-18, 24 min (arm under the unit) | `Bonsai2-PTQ1_0/Ternary-Bonsai-2-27B-PTQ1_0.gguf` 5,946,648,928 B + hardlinked Q8 mmproj | PrismML fork 10777 | 2 × 200,704, batch/ubatch 3072/1536 | Frank scratched the 27B plan — ~48 t/s decode vs 76.6 for PQ2_0 on the same weights | `/home/dank/models/Bonsai2-PTQ1_0/` + `scripts/download-frank-zealot-bonsai2-ptq1.sh` | | 2026-10-17 → 10-18 | Ternary-Bonsai-2-**27B** PQ2_0 7,206,168,928 B `3907dc16…` + Q8_0 mmproj 629,246,976 B | PrismML fork **10754** (`~/GitPackages/llama-prism/`) | 2 × 200,704 (pool 401,408), q8_0 K+V = 13,328 MiB, settled 22,664 / 1,461; 76.6 t/s decode, 109.4 aggregate over two streams, 691.7 t/s prefill at 166,027; MTP impossible — no head in the file | the 9B bills half the KV per token (8 full layers, not 16), which funded three slots at the native window and beat it on every rate | `/home/dank/models/Bonsai2-PQ2_0/` + the 10754 fork tree, `scripts/download-frank-zealot-bonsai2.sh`, revert the swap commit | @@ -160,7 +151,7 @@ re-run the same arms with the box quiet before a figure is carried anywhere else ## Contract -- `/health` → `{"status":"ok"}`; `/props` reports `n_ctx` and `presence_penalty`; `/slots` shows occupancy; a third +- `/health` → `{"status":"ok"}`; `/props` reports `n_ctx` and `presence_penalty`; `/slots` shows occupancy; a fourth concurrent conversation **waits** for a slot — the slot count is the concurrency contract. - `/metrics` answers 200 with 15 `llamacpp:*` families **only with `--metrics`**; without it llama.cpp answers **501**, which Prometheus reads as DOWN while `/health` stays 200. The scrape target is `host.containers.internal:7771/metrics` and the oldest retained sample of `up{job="frank-zealot"}` (2026-10-11) is already 0 — this target was never successfully scraped, across every payload era. `METRICS=0` is the documented opt-out and re-breaks it; the counters cost nothing on the request path. - Reasoning arrives in **`reasoning_content`** (`--reasoning-format deepseek`) and **counts against `max_tokens`**: a short budget yields empty `content` with `finish_reason: "length"`, which reads as a broken model, not a mid-thought one. The slot prefix cache is live (`cached_tokens` in `usage`). @@ -168,9 +159,9 @@ re-run the same arms with the box quiet before a figure is carried anywhere else (1.0 / 0.95 / 20). `presence_penalty` is the one recipe value with no GGUF metadata key, so it must reach argv; the serve script uses `${PRESENCE_PENALTY=1.0}`, deliberately **not** `:=`, because `""` is the documented opt-out. - Vision: only the OpenAI **`image_url`** content part reaches the model — the legacy top-level `images` field bumps `usage.prompt_tokens` while the model invents an answer. Probe with a **≥ 256 px** solid color (a postage stamp answers "video") and budget **≥ 512** tokens; a 512 px image bills ~280 prompt tokens. -- **Q8_0 mmproj, not BF16**: the publisher warns the BF16 tower "can crash llama-server when several image requests run at once", which is exactly the `--parallel 2` shape. `MMPROJ_FILE=` (explicitly empty) is the text-only opt-out and returns ~1.4 GiB; a path at a missing file is a boot **refusal**, never silent blindness. +- **Q8_0 mmproj, not BF16**: the publisher warns the BF16 tower "can crash llama-server when several image requests run at once", which is exactly the `--parallel 3` shape. `MMPROJ_FILE=` (explicitly empty) is the text-only opt-out and returns ~1.4 GiB; a path at a missing file is a boot **refusal**, never silent blindness. - Tools: native via `--jinja`; `finish_reason: tool_calls` with valid JSON arguments. -- Two non-faults: a `400` naming 262,144 is one conversation outgrowing its **slot**, not pool exhaustion; and +- Two non-faults: a `400` naming 200,704 is one conversation outgrowing its **slot**, not pool exhaustion; and the raw `/completion` path is a **dead instrument on this fork build** — a greedy `ignore_eos` run answers **500 "The model produced output that does not match the expected Content-only format"**, because build 10777 guards the raw path the moment reasoning markup leaks (older builds only logged `failed to parse reasoning diff --git a/tests/frank_zealot_unit.bats b/tests/frank_zealot_unit.bats index 1212a4f..8c012b3 100755 --- a/tests/frank_zealot_unit.bats +++ b/tests/frank_zealot_unit.bats @@ -4,15 +4,15 @@ # qwen35 hybrid with 32 layers (8 full-attention + 24 Gated DeltaNet) and the # MTP draft head INSIDE the file — via the PrismML fork build of llama.cpp # (vendor build 10777), the ONLY loader that registers the payload's custom -# tensor type 142 and honours its prism.hadamard.* metadata: TWO 262,144-token -# conversations (--ctx-size 524288 --parallel 2) over an UNQUANTIZED f16 KV -# cache. Exact KV bills 32,768 B/token here, so the pool is what pays for it — -# the same dtype at the old 786,432 pool is 24,576 MiB against the 24,123 MiB -# the driver leaves addressable, and the slot count is the term that moved. The -# publisher's own Q8_0 vision projector rides along (the BF16 one can crash on -# concurrent images), and MTP stays OFF: the head is present, but exact KV -# leaves no room for a draft context that bills f16 KV over the whole pool. Log -# verbosity 4 ships so the VRAM-fit forensics land in the journal. +# tensor type 142 and honours its prism.hadamard.* metadata: THREE 200,704-token +# conversations (--ctx-size 602112 --parallel 3) over an f16 K + q8_0 V cache — +# 25,088 B/token, 14,406.00 MiB at this pool — with the MTP drafter ARMED +# (--spec-type draft-mtp). The asymmetry IS the shape: exact V (32,768 B/token) +# leaves nothing for a draft context that bills its own f16 KV over the WHOLE +# pool (4,096 B/position, 2,352.00 MiB here), and the 786,432 pool this shape +# replaced cannot carry a drafter at any dtype inside the 24,123 MiB the driver +# leaves addressable. The publisher's own Q8_0 vision projector rides along (the +# BF16 one can crash on concurrent images). Log # This seam runs the serve script against a stub engine and asserts on the # argv it actually execs, so what is pinned is the command the operator gets. @@ -143,7 +143,7 @@ run_stage() { done } -@test "two conversations share the pool at a 262144 window each" { +@test "three conversations share the pool at a 200704 window each" { serve_argv [ "$status" -eq 0 ] [ "$(argv_value --alias)" = "frank-zealot" ] @@ -152,30 +152,36 @@ run_stage() { [ "$(argv_value --model)" = "$MODEL_DIR/$MODEL_STUB" ] [ "$(argv_value --n-gpu-layers)" -gt 0 ] # llama.cpp divides --ctx-size across slots, so the observable contract is - # the per-slot window. Exact KV costs the SLOT COUNT, not the window: every - # conversation still gets the model's TRAINED 262,144, and the harness - # contextWindow must quote that number — the pool and the slot count are - # neither, and a longer prompt is a 400, never a truncation. - [ "$(argv_value --parallel)" -eq 2 ] - [ "$(argv_value --ctx-size)" -eq 524288 ] - local pool slots + # the per-slot window: harness contextWindow, Prometheus dashboards and every + # consumer's compaction trigger read THAT number, and a longer prompt is a + # 400, never a truncation. Two invariants the pool/slot split has to keep: + # it divides evenly (llama.cpp floors it otherwise) and a slot never exceeds + # the model's TRAINED window (there is no YaRN here, so the engine would be + # quoting a window it cannot fill). + [ "$(argv_value --parallel)" -eq 3 ] + [ "$(argv_value --ctx-size)" -eq 602112 ] + local pool slots window pool="$(argv_value --ctx-size)"; slots="$(argv_value --parallel)" [ $((pool % slots)) -eq 0 ] - [ $((pool / slots)) -eq 262144 ] + window=$((pool / slots)) + [ "$window" -eq 200704 ] + [ "$window" -le 262144 ] } @test "the KV dtype pair follows the env file and flash-attn stays pinned on" { # Asserted against the env rather than a literal so a retune moves one - # file, not two. The shipped pair is f16/f16: exact attention state, and the - # slot count is what makes it affordable — this payload bills 8,192 elements - # per token per cache (8 full-attention layers × 4 KV heads × 256 dim), so - # exact KV is 32,768 B/token = 16,384.00 MiB at the 524,288 pool, where the - # q8_0 pair this file used to ship billed 17,408 B/token = 13,056.00 MiB at - # the old 786,432 pool (ledger-exact both times). The 24 DeltaNet layers + # file, not two. The shipped pair is ASYMMETRIC — f16 K (the engine's own + # default) plus q8_0 V: this payload bills 8,192 elements per token per cache + # (8 full-attention layers × 4 KV heads × 256 dim), so K costs 16,384 B/token + # and V 8,704 (q8_0 is 1.0625 B/element; there is no true fp8 KV dtype in + # this build) = 25,088 B/token = 14,406.00 MiB at the 602,112 pool. Exact V + # would bill 32,768 B/token and the all-q8_0 pair this file used to ship + # billed 17,408 — the halfway rung is what pays for the third slot AND the + # armed drafter (the fit gate below prices both). The 24 DeltaNet layers # bill a fixed recurrent state per slot (~50.25 MiB a cell), which is why # pool arithmetic alone under-prices the shape. flash-attn is not - # decoration: the quantized-V fallback is only legal with it, so it stays - # explicit and a dtype retune cannot silently fall back. + # decoration: quantized V is only legal with it, so it stays explicit and a + # dtype retune cannot silently fall back. local env_k env_v env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" @@ -192,30 +198,47 @@ run_stage() { # The fit is arithmetic off the payload geometry plus the non-KV terms # measured off the boot ledger (weights 2,036.71 + compute buffers 1,409.02 # + projector 595 + 248.10 + CUDA context ≈ 460 = 4,749 MiB, dtype- and - # slot-independent at this ubatch), plus one ~51 MiB recurrent-state cell - # per slot. 24,123 MiB is what the driver leaves addressable (nvidia-smi - # reports it as used + free; its total column reads 24,576). - local env_k env_v pool slots rate_k rate_v kv total + # slot-independent at ubatch 2048), plus one ~51 MiB recurrent-state cell per + # slot. That constant is deliberately NOT credited with the ~1,306 MiB the + # shipped ubatch 1024 hands back: the gate stays conservative. + # An ARMED drafter is billed, because it is the term this shape pays + # CACHE_TYPE_V=q8_0 to afford: the draft context is built against the + # TARGET's whole pool at its own f16 rate (measured 4,096 B/position at + # blk.32) plus buffers and its own recurrent cells — the measured spec-ON + # against spec-OFF delta at the 786,432 / ubatch-1024 arm was 3,524 MiB + # beside 3,072 MiB of draft KV, i.e. ~452 MiB, doubled here for the third + # slot's extra cells. 24,123 MiB is what the driver leaves addressable + # (nvidia-smi reports it as used + free; its total column reads 24,576). + local env_k env_v pool slots rate_k rate_v kv total spec f16 exact_pool exact_v env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")" env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")" pool="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" slots="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")" + spec="$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")" rate_k="$(kv_bytes_per_token "$env_k")" rate_v="$(kv_bytes_per_token "$env_v")" + f16="$(kv_bytes_per_token f16)" kv="$(( pool * (rate_k + rate_v) / 1048576 ))" total="$(( kv + 4749 + 51 * slots ))" + if [ "$spec" != "none" ]; then + total="$(( total + pool * 4096 / 1048576 + 768 ))" + fi # The shipped arm has to leave real room: the projector's worst case is # charged AFTER the engine's own fit check, so a projection that lands on # the ceiling is a crash waiting for the first image request. [ "$total" -le $(( 24123 - 1024 )) ] - # And the gate must be able to REFUSE: the same exact KV at the pool this - # shape left behind is 24,576 MiB of cache alone — a boot that dies before - # /health ever answers. Proving the gate bites is what makes passing one - # meaningful. - local over - over="$(( 786432 * (rate_k + rate_v) / 1048576 + 4749 + 51 * 3 ))" - [ "$over" -gt 24123 ] + # And the gate must be able to REFUSE, or passing above means nothing. Two + # boots this shape can walk straight into: the retired dtype pair at the + # retired pool (exact f16 at 786,432 = 24,576 MiB of cache alone, which died + # at cudaMalloc), and the shipped pool with V moved back to f16 while the + # drafter stays armed — 18,816 MiB of cache plus the draft's 2,352. Both die + # before /health ever answers, and no health gate can see either. + exact_pool="$(( 786432 * (rate_k + f16) / 1048576 + 4749 + 51 * 3 ))" + [ "$exact_pool" -gt 24123 ] + exact_v="$(( pool * (rate_k + f16) / 1048576 + 4749 + 51 * slots + + pool * 4096 / 1048576 + 768 ))" + [ "$exact_v" -gt 24123 ] } @test "log verbosity follows the env file" { @@ -277,48 +300,54 @@ run_stage() { argv_value --ubatch-size | grep -qE '^[0-9]+$' } -@test "speculation is off: no drafter flag reaches the argv" { - # Frank, 2026-10-18: this file DOES carry the MTP head (the boot arms prove - # it loads and drafts, acceptance 0.798/0.617 per position, mean accepted - # length 2.41) and it is still off, because the draft context bills its own - # f16 KV over the WHOLE pool — 4,096 B/position × 524,288 = 2,048 MiB plus - # buffers — on a trunk that already projects 21,234 MiB with exact KV. The - # MTP arms in frank-zealot.env are all q8_0-era measurements; the doors back - # to the drafter are one window per conversation, or KV back to q8_0 with - # UBATCH_SIZE=1024 (213.5 t/s). The flags move as a set: - # SPEC_TYPE=none strips --spec-type, --spec-draft-n-max AND - # --spec-draft-model together, so a no-spec boot cannot carry half a spec - # pair (the ROCmFPX-era baseline arm segfaulted at load doing that). +@test "the armed drafter ships the whole MTP set and no second copy of the trunk" { + # Frank, 2026-10-19: the drafter is ON. The head is IN this file + # (qwen35.nextn_predict_layers = 1, blk.32.nextn.*) and it accepted well + # when it was measured off (0.798/0.617 per position, mean accepted length + # 2.41); what it now pays for is CACHE_TYPE_V=q8_0 plus UBATCH_SIZE=1024, + # which is why the dtype pair and the batch pair are pinned next door. + # The observable contract is the SET: --spec-type plus every --spec-draft-* + # the env names, and NO --spec-draft-model — the MTP draft rides the MAIN + # model's weights, so a draft file would load the trunk a second time. + local n_max + n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")" serve_argv [ "$status" -eq 0 ] + [ "$(argv_value --spec-type)" = "draft-mtp" ] + [ "$(argv_value --spec-draft-n-max)" = "$n_max" ] + [ "$(argv_value --spec-draft-n-min)" = "0" ] + [ "$(argv_value --spec-draft-p-min)" = "0" ] + ! grep -qxF -- '--spec-draft-model' <<<"$output" +} + +@test "the rollback knob strips the entire spec set" { + # SPEC_TYPE=none is the documented retreat (and the shape this endpoint + # shipped before 2026-10-19). The law it exists to obey: the flags move as a + # SET — a no-spec boot that keeps --spec-draft-n-max, or a baseline arm that + # hands over --model-draft with no --spec-type, has segfaulted at load with + # no diagnostic. So the rollback is proven by ABSENCE of all of them. + serve_argv + sed -i 's/^SPEC_TYPE=.*/SPEC_TYPE=none/' "$STAGE/test.env" + run_stage + [ "$status" -eq 0 ] ! grep -qxF -- '--spec-type' <<<"$output" ! grep -qxF -- '--spec-draft-n-max' <<<"$output" + ! grep -qxF -- '--spec-draft-n-min' <<<"$output" + ! grep -qxF -- '--spec-draft-p-min' <<<"$output" ! grep -qxF -- '--spec-draft-model' <<<"$output" - grep -qE '^SPEC_TYPE=none$' "$ENV_FILE" } -@test "naming a drafter ships the MTP set, with or without a separate draft file" { - # The documented MTP arm (frank-zealot.env): SPEC_TYPE=draft-mtp ALONE is - # legal here because the head is IN the checkpoint and the draft context - # rides the main model's weights — so --spec-draft-model must NOT appear, - # or the trunk would be loaded twice. +@test "an external drafter still reaches the engine as a complete set" { + # The other arm the ROCmFPX-era crash was about: a named draft FILE must + # arrive alongside --spec-type and the depth recipe, never alone. This + # endpoint does not use it (the head is in the trunk), so the knob ships + # empty and only this test sets it. serve_argv - { - echo 'SPEC_TYPE=draft-mtp' - echo 'SPEC_DRAFT_N_MAX=2' - } >> "$STAGE/test.env" - run_stage - [ "$status" -eq 0 ] - [ "$(argv_value --spec-type)" = "draft-mtp" ] - [ "$(argv_value --spec-draft-n-max)" = "2" ] - ! grep -qxF -- '--spec-draft-model' <<<"$output" - - # And an external drafter still reaches the engine as a complete set — the - # pair the ROCmFPX-era crash was about. echo "SPEC_DRAFT_FILE=$MODEL_DIR/$DRAFT_STUB" >> "$STAGE/test.env" run_stage [ "$status" -eq 0 ] [ "$(argv_value --spec-draft-model)" = "$MODEL_DIR/$DRAFT_STUB" ] + [ "$(argv_value --spec-type)" = "draft-mtp" ] } @test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {