From 812d9ebff5e8bf13f91deefe10f0a408f0bc914d Mon Sep 17 00:00:00 2001 From: Frank Date: Tue, 6 Oct 2026 00:07:03 -0700 Subject: [PATCH] frank-small: two 128k slots, no spec, ubatch 512 (Frank's tight-fit order, 2026-10-06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CTX 262144 / PARALLEL 2 = two 131,072 conversations; SPEC_TYPE=none strips both speculative flags in the serve script (ngram-mod skipped on purpose — on a qwen35 hybrid it disables recurrent-state rollback); ubatch 1024->512 claws back ~0.3 GiB. Budget: non-KV ~15.5 GiB + two q8_0 slots 8,704 MiB lands AT the card — the 2-3 GiB floor is knowingly violated by explicit order; nvidia-smi free at boot is the measured truth. Seam retargeted: shape (2 slots, 131072/slot), budget (two fit, three don't), spec-off (absence of both flags), unit Description. --- scripts/frank-small.env | 81 ++++++++++++++++++++---------------- scripts/frank-small.service | 2 +- scripts/serve-frank-small.sh | 58 ++++++++++++++------------ tests/frank_small_unit.bats | 69 +++++++++++++++--------------- 4 files changed, 113 insertions(+), 97 deletions(-) diff --git a/scripts/frank-small.env b/scripts/frank-small.env index 71b7d0e..be4f8a8 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -23,22 +23,29 @@ # => 34 KiB/token (f16 would be 64 KiB/token — halogen's 27B card quotes # the same f16 number; at f16 the 131,072 slot alone would cost 8.0 GiB). # -# Why ONE slot at 131,072 is the maximum (Frank's cap per conversation): -# card 24,576 MiB. Weights 11,561 MiB (12,120,016,896 B, the IQ3_S-mtp -# file; the GSQ "IQ3_S" tier is a per-tensor ALLOCATION mix — histogram -# IQ2_XS 144 / IQ3_XXS 96 / Q6_K 78 / F32 360 / BF16 96 / Q3_K_M 39 ... -# all stock types, stock llama.cpp loads it). Projector ~1,350 MiB -# (927,607,488 B F16 file at the measured ~1.45x card law). Compute -# buffers at ubatch 1024 ~1.6 GiB. CUDA ctx ~500 MiB. DeltaNet state -# ~150 MiB/slot (48 linear layers x 48 value heads x 128x128 x 4 B). -# MTP draft context ~0.3-0.5 GiB (one nextn layer over the slot window; -# the boot ledger is the truth). Non-KV ~= 15.5 GiB. One q8_0 slot = -# 4,352 MiB => free ~= 4.5 GiB. A SECOND slot wants +4,352 MiB => free -# ~0.2 GiB, far below the 2-3 GiB floor. "As many 128k contexts as fit" -# = 1. MEASURE nvidia-smi free at boot (quote the free field): under -# ~1.5 GiB or a boot failure, step the pool down in 12,288-token -# increments (~410 MiB each at this rate). Compute buffers allocate at -# graph capture during BOOT, so a failure is loud and fast. +# 2026-10-06 (Frank): drop the MTP drafter (skip ngram too — on a qwen35 +# hybrid, ngram speculation disables recurrent-state rollback: fleet lesson +# from the frank-large chain measurement), drop ubatch to 512, and take TWO +# conversations at 131,072 — "should be a very tight fit", accepted +# explicitly, i.e. the 2-3 GiB floor is knowingly violated. Measured at the +# 1x131072 MTP boot (2026-10-05): 19,404 used / 4,721 MiB free. +# +# Why TWO slots at 131,072 fits, by arithmetic (pool = 2 x 131,072 = +# 262,144; the per-conversation cap stays Frank's 128k): +# +# card 24,576 MiB. Weights 11,561 MiB (the IQ3_S-mtp file; the GSQ +# "IQ3_S" tier is a per-tensor ALLOCATION mix — histogram IQ2_XS 144 / +# IQ3_XXS 96 / Q6_K 78 / F32 360 / BF16 96 / Q3_K_M 39 ... all stock +# types, stock llama.cpp loads it). Projector ~1,350 MiB (927,607,488 B +# F16 at the measured ~1.45x card law). Compute buffers at ubatch 512 +# ~1.3 GiB (inferred from the measured 616 MiB/doubling rung on the 9B). +# CUDA ctx ~500 MiB. DeltaNet state ~150 MiB/slot x 2. NO draft context +# (the MTP draft ctx cost ~0.3-0.5 GiB and is gone with speculation). +# Non-KV ~15.5 GiB. Two q8_0 slots = 8,704 MiB (34 KiB/token x 131,072 +# each). Predicted free at boot ~0.3-1.3 GiB. Compute buffers allocate +# at graph capture during BOOT, so a miss is loud and fast; MEASURE +# nvidia-smi free at boot (quote the free field) and let the runbook +# carry the number. # # Payload history (for the rollback trail): the Ornith-1.5-9B Q8_0 swap of # earlier on 2026-10-05 (commits 77decb5..380aafa) was committed but never @@ -66,22 +73,23 @@ ALIAS=frank-small BIND_ADDR=0.0.0.0 LISTEN_PORT=7771 -# ONE conversation at Frank's 128k cap: --ctx-size is the POOL llama.cpp -# divides across slots; 131072 / 1 = 131,072 per conversation. The cap is a -# product decision (Swift's own benchmark protocol ran at 131,072), and a -# second 131,072 slot does not fit the 2-3 GiB floor (arithmetic above). -CTX_SIZE=131072 -PARALLEL=1 +# TWO conversations at Frank's 128k cap (2026-10-06): --ctx-size is the +# POOL llama.cpp divides across slots; 262144 / 2 = 131,072 per +# conversation. Frank accepted the tight fit explicitly (arithmetic above); +# the per-slot window is the product contract, so models.yml stays at +# 131,072. +CTX_SIZE=262144 +PARALLEL=2 # Compute-buffer geometry. Ubatch is a prefill knob (decode never moves); -# on this CUDA 3090 the curve climbs monotonically to 2048, but Frank's -# standing law for a resize is LOWER UBATCH FIRST (the measured 616 MiB rung -# on the 9B; the 27B's buffers are larger, so the claw-back is worth more). -# The pair is a measurement, not a law: if boot lands well above the floor, -# ubatch 2048 is the first thing to A/B for prefill (loud failure at graph -# capture if it does not fit). +# on the 9B the 1024->2048 rung measured +4.6 % prefill for 616 MiB, and +# Frank's 2026-10-06 order spends the other direction: ubatch 512 claws +# back ~0.3 GiB toward the second slot at some prefill cost. The pair is a +# measurement, not a law: ubatch 1024 is the first A/B back if a future +# boot lands with headroom (loud failure at graph capture if it does not +# fit). BATCH_SIZE=4096 -UBATCH_SIZE=1024 +UBATCH_SIZE=512 # The KV contract for this swap (Frank, 2026-10-05): q8_0 K + q8_0 V — # llama.cpp's FP8 class. The flags stay explicit so the choice is visible in @@ -92,12 +100,13 @@ UBATCH_SIZE=1024 CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Speculative decoding: the MTP head is IN the -mtp file (blk.64.nextn.*, -# nextn_predict_layers=1), so draft-mtp needs no second model and no second -# KV pool — the head shares the main model's weights. Depth 3 matches the -# fleet's radiance MTP depth; measure draft acceptance from the response -# timings object and retune here if the head earns more. -SPEC_TYPE=draft-mtp -SPEC_DRAFT_N_MAX=3 +# Speculative decoding is OFF (Frank, 2026-10-06): no draft-mtp, and +# ngram-mod is skipped too — on a qwen35 hybrid ngram disables +# recurrent-state rollback, so a rejected draft would corrupt the DeltaNet +# state. SPEC_TYPE=none strips BOTH speculative flags in the serve script +# (flags move as a set). The -mtp file still carries the head; it is simply +# not drafted with. Measured MTP reference (2026-10-05): 44.8 % acceptance +# at the recipe temperature => 58.9 t/s, 52.1 % at temp 0.8 => 66.1 t/s. +SPEC_TYPE=none NGL=99 diff --git a/scripts/frank-small.service b/scripts/frank-small.service index 7140cbb..2dcef60 100644 --- a/scripts/frank-small.service +++ b/scripts/frank-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S-mtp GGUF (qwen35 hybrid, 1x131072, q8_0 K+V, in-file MTP head, mmproj vision) on the RTX 3090 (dankclaw, frank-small) +Description=llama.cpp Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S-mtp GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, no speculative decoding, mmproj vision) on the RTX 3090 (dankclaw, frank-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 892f29f..39753e0 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -3,11 +3,13 @@ # frank-small on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA # sm_86). # -# 2026-10-05 evening (Frank): a 27B on the 24 GiB card — ONE 131,072-token -# conversation (Frank's cap), KV at FP8 class (q8_0 both halves), the MTP -# head that lives IN the -mtp file drafting 3, vision on. The q8_0 KV -# contract supersedes the same-day f16/f16 rule, which was written for the -# Ornith 9B fit; the per-tensor flags stay explicit either way. +# 2026-10-06 (Frank): the 128k cap keeps the per-conversation window but +# the pool doubles to TWO 131,072 conversations (262144 / 2), speculative +# decoding is OFF (no draft-mtp; ngram skipped — on a qwen35 hybrid it +# disables recurrent-state rollback), and ubatch drops to 512. Frank +# accepted the tight fit explicitly (the 2-3 GiB floor is knowingly +# violated; frank-small.env carries the arithmetic). The q8_0 KV contract +# (FP8 class) is unchanged; the per-tensor flags stay explicit either way. # # Architecture: qwen35 hybrid — 64 real layers, 16 full-attention + 48 # Gated DeltaNet (full_attention_interval 4), native 262144 window, plus @@ -40,29 +42,30 @@ fi : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" -# ctx-size is the POOL llama.cpp divides across slots: 131072 / 1 gives -# 131,072 per conversation — Frank's 128k cap. A second 131,072 slot does -# not fit the 2-3 GiB floor on the 24 GiB card with the 27B payload -# (frank-small.env carries the arithmetic); "as many 128k contexts as fit" -# = 1. -: "${CTX_SIZE:=131072}" -: "${PARALLEL:=1}" -# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves), -# and Frank's standing law for a resize is LOWER UBATCH FIRST; the shipped -# pair is 4096/1024. If boot lands well above the 2-3 GiB floor, ubatch -# 2048 is the first A/B for prefill (a miss fails loud at graph capture). -# The logical batch is VRAM- and throughput-neutral at a fixed ubatch. +# ctx-size is the POOL llama.cpp divides across slots: 262144 / 2 gives +# TWO 131,072 conversations — Frank's 128k cap each. 2026-10-06 (Frank): +# drop MTP, drop ubatch to 512, take two slots; the fit is tight by his +# explicit order (frank-small.env carries the arithmetic and the floor +# violation). +: "${CTX_SIZE:=262144}" +: "${PARALLEL:=2}" +# Compute-buffer geometry. Ubatch is a prefill knob (decode never moves). +# Frank's 2026-10-06 tight-fit order ships 4096/512 (claws back ~0.3 GiB +# toward the second slot); ubatch 1024 is the first A/B back if a future +# boot lands with headroom (a miss fails loud at graph capture). The +# logical batch is VRAM- and throughput-neutral at a fixed ubatch. : "${BATCH_SIZE:=4096}" -: "${UBATCH_SIZE:=1024}" +: "${UBATCH_SIZE:=512}" # KV at FP8 class = q8_0 on llama.cpp (1.0625 B/elem => 8.5 effective bits; # there is no true fp8 KV dtype). Explicit either way so the contract is # visible in the argv. : "${CACHE_TYPE_K:=q8_0}" : "${CACHE_TYPE_V:=q8_0}" -# Speculative decoding: draft-mtp loads the MTP head that is IN the -mtp -# GGUF; the flags move as a set. -: "${SPEC_TYPE:=draft-mtp}" -: "${SPEC_DRAFT_N_MAX:=3}" +# Speculative decoding is OFF by default (Frank, 2026-10-06): SPEC_TYPE +# none strips BOTH speculative flags (flags move as a set). Setting a +# drafter here re-enables the pair — draft-mtp needs the head that is IN +# the -mtp GGUF. +: "${SPEC_TYPE:=none}" : "${NGL:=99}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. @@ -80,11 +83,12 @@ if [[ -n "$MMPROJ_FILE" ]]; then EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE") fi -# The MTP head ships IN the -mtp GGUF (blk.64.nextn.*, -# nextn_predict_layers=1): draft-mtp loads it against the main model's own -# weights — no --model-draft and no second KV pool. Speculative flags move -# as a set: a no-spec baseline must strip BOTH of these, not just one. -EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX") +# Speculative flags move as a set: they reach the argv ONLY when a drafter +# is named. SPEC_TYPE=none (the shipped contract) strips both — a no-spec +# baseline must not carry half a spec pair. +if [[ -n "$SPEC_TYPE" && "$SPEC_TYPE" != "none" ]]; then + EXTRA_ARGS+=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX") +fi # The UVM device node is created lazily by nvidia-modprobe on the first CUDA # open after boot; make sure it exists before the server needs it. diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 3284fd5..55d3b4d 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -1,13 +1,15 @@ #!/usr/bin/env bats # Seam: the frank-small llama.cpp root unit on dankclaw. -# frank-small is the RTX 3090 endpoint; since 2026-10-05 evening it serves -# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B, MTP head in the -# file) via a CUDA (sm_86) build of llama.cpp master: ONE 131,072-token -# conversation at Frank's 128k cap, q8_0 K + q8_0 V (FP8 class), the -# in-file MTP head drafting 3, and a vision projector. The card must keep -# 2-3 GiB free. This seam runs the serve script against a stub engine and -# asserts on the argv it actually execs, so what is pinned is the -# command the operator gets. +# frank-small is the RTX 3090 endpoint; since 2026-10-06 it serves +# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B; the -mtp file's +# head is present but NOT drafted — speculation is off) via a CUDA (sm_86) +# build of llama.cpp master: TWO 131,072-token conversations at Frank's +# 128k cap (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), and a +# vision projector. Frank accepted the tight fit explicitly — the card may +# land under the old 2-3 GiB floor; the seam pins the arithmetic, the boot +# log carries the measured free. This seam runs the serve script against a +# stub engine and asserts on the argv it actually execs, so what is pinned +# is the command the operator gets. load helpers @@ -110,10 +112,9 @@ argv_value() { [ "$(argv_value --n-gpu-layers)" -gt 0 ] # llama.cpp divides --ctx-size across slots and a mamba hybrid cannot # share one cache, so the observable contract is the per-slot window. - # 2026-10-05 evening (Frank): the 27B Swift-1.5 swap caps each - # conversation at 131,072, and ONE slot is the maximum — a second - # 131,072 slot does not fit the 2-3 GiB floor (KV budget test below). - [ "$(argv_value --parallel)" -eq 1 ] + # 2026-10-06 (Frank): drop MTP, drop ubatch, and take TWO conversations + # at 131,072 — pool 262144, parallel 2, per-slot 131072 (the cap). + [ "$(argv_value --parallel)" -eq 2 ] ctx="$(argv_value --ctx-size)" par="$(argv_value --parallel)" [ $((ctx % par)) -eq 0 ] @@ -134,7 +135,7 @@ argv_value() { [ "$(argv_value --flash-attn)" = "on" ] } -@test "the KV budget fits the 2-3 GiB floor and forbids a second 128k slot" { +@test "the KV budget fits TWO 128k slots (Frank's tight-fit order) and forbids a third" { # Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub # file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64 # real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv @@ -142,18 +143,19 @@ argv_value() { # => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64). # The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) + # vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured - # ~1.45x cost law) + DeltaNet states + compute buffers at ubatch 1024 + - # CUDA graphs + the MTP draft context => 17 GiB budgeted. Two asserts, - # both about the output the operator sees: the pool must still fit with - # 2 GiB spare, and a SECOND 131,072 slot must not fit — Frank's "as - # many 128k contexts as fit" has the answer 1 on this card, and the - # pool itself is capped by the 128k law, not by the card. - # Arithmetic, not a boot log: change CTX_SIZE and this bites before the - # deploy does. + # ~1.45x cost law) + compute buffers at ubatch 512 + CUDA graphs + TWO + # DeltaNet states (~150 MiB each) and NO draft context (speculation is + # off) => 15 GiB budgeted. 2026-10-06 (Frank): two 131,072 slots are + # the order — the budget lands AT the card (tight fit, accepted + # explicitly; the old 2-3 GiB floor is knowingly violated) and a THIRD + # slot must not fit. Arithmetic, not a boot log: change CTX_SIZE and + # this bites before the deploy does; the boot's nvidia-smi free is the + # measured truth. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 37 / 1048576)) - [ $((kv_gib + 17 + 2)) -le 24 ] - [ $((2 * kv_gib + 17)) -gt 24 ] + third_slot_kv=$(( (ctx + 131072) * 37 / 1048576 )) + [ $((kv_gib + 15)) -le 24 ] + [ $((third_slot_kv + 15)) -gt 24 ] } @test "the compute-buffer pair follows the env file and stays self-consistent" { @@ -183,18 +185,19 @@ argv_value() { argv_value --ubatch-size | grep -qE '^[0-9]+$' } -@test "the in-file MTP head ships as draft-mtp at depth 3" { - # The -mtp GGUF carries the head (blk.64.nextn.*, nextn_predict_layers - # =1): draft-mtp loads it against the main model's own weights — no - # --model-draft and no second draft KV pool. Speculative flags move as - # a set; both must appear in the argv the supervisor runs, and the env - # must carry the same contract the argv executes. +@test "speculation is off: no half spec pair reaches the argv" { + # Frank, 2026-10-06: drop MTP, skip ngram-mod (on a qwen35 hybrid + # ngram disables recurrent-state rollback — a rejected draft corrupts + # the DeltaNet state). The flags move as a set, so the no-spec contract + # is ABSENCE of both in the argv the supervisor runs, and the env must + # carry the same contract. The -mtp file's head stays; it is not + # drafted with. Reference numbers stay in frank-small.env for the day + # MTP returns (44.8 % acceptance at the recipe temp => 58.9 t/s). serve_argv [ "$status" -eq 0 ] - [ "$(argv_value --spec-type)" = "draft-mtp" ] - [ "$(argv_value --spec-draft-n-max)" = "3" ] - grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" - grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE" + ! grep -qxF -- '--spec-type' <<<"$output" + ! grep -qxF -- '--spec-draft-n-max' <<<"$output" + grep -qE '^SPEC_TYPE=none$' "$ENV_FILE" } @test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {