diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 0c26a41..c26d95e 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1207,14 +1207,13 @@ deploy_llm_large() { # # It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is # not shared: frank-large's engine runs on the gfx1151 iGPU. -LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}" +LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}" LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -# The projector IS the vision capability here: no Swift repo ships one (the -# GGUF carries no vision tensors), so the file is unsloth's mmproj built -# from the shared Qwen3.8-27B base — proven byte-identical to Swift's own -# vision stack. A deploy that starts without this file silently ships a -# text-only endpoint. Checked unconditionally, like frank-large's. -LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf}" +# The projector IS the vision capability here: the GGUF carries no vision +# tensors, and the publisher ships mmproj-BF16.gguf in the same repo, so a +# deploy that starts without this file silently ships a text-only +# endpoint. Checked unconditionally, like frank-large's. +LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}" deploy_llm_small() { # Refuse rather than restart-loop: a unit whose runtime or payload is diff --git a/scripts/download-frank-small.sh b/scripts/download-frank-small.sh index d35cf33..1447b89 100755 --- a/scripts/download-frank-small.sh +++ b/scripts/download-frank-small.sh @@ -1,31 +1,27 @@ #!/usr/bin/env bash # Download the frank-small payload for llama.cpp on the RTX 3090: -# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S **with the MTP head in the -# file** (the -mtp variant), plus a vision projector. No Swift repo ships a -# projector — ukisai's release is language-model-only ("A Swift 27B vision -# projector has not been verified for this release") — so this script fetches -# unsloth's mmproj-F16.gguf built from the shared base Qwen/Qwen3.8-27B. -# That substitution is proven, not assumed: on 2026-10-05 Swift-1.5's vision -# stack was hash-compared against the base over HTTP range reads — 333/333 -# visual tensor names and 6/6 sampled tensors byte-identical (merger fc1/fc2, -# blocks 0/13/26) — so the base-model projector IS the model's own projector. -# Public repos — no auth header, and never a token on the command line (it is +# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS +# **with the MTP head in the file** (the -mtp variant), plus the +# PUBLISHER's OWN vision projector (mmproj-BF16.gguf, same repo). Public +# repo — no auth header, and never a token on the command line (it is # world-readable in /proc//cmdline). # -# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized file -# on a 31 GiB payload); -f/--fail-with-body so an HTTP error is not a success, -# then byte count and the Hub LFS sha256. Idempotent: complete files are -# skipped. The projector is stored under a self-documenting dest name; the -# URL name and sha stay pinned in the table below. +# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized +# file on a 31 GiB payload); -f/--fail-with-body so an HTTP error is not a +# success, then byte count and sha256. This repo is on Xet storage, so +# the Hub tree/blob APIs expose no LFS oid: the sha256 values below were +# computed locally after the first verified fetch (byte counts match the +# Hub's declared sizes). Idempotent: complete files are skipped. The +# projector is stored under a self-documenting dest name; the URL name +# and sha stay pinned in the table below. # -# 2026-10-05 (Frank, evening): the payload moved to Swift-1.5-27B GSQ-RCO -# IQ3_S-mtp — a 27B on the 24 GiB card: ONE 131,072-token slot, q8_0 K + q8_0 -# V, the in-file MTP head drafting 3. This supersedes the Ornith-1.5-9B Q8_0 -# swap (commits 77decb5..380aafa, same day), which was committed but never -# deployed — the live service was still MiMo-V2.6 Q6_K when Swift cut over. -# Rollback payloads stay on disk: /home/dank/models/Ornith-Q8/ and -# /home/dank/models/MiMo-Q6K/ (the payload that was live until this swap). -# Their re-fetch recipes are in git history (predecessors of this script). +# 2026-10-06 (Frank, evening): the payload moved to the UNCELERSED +# Swift-1.5-Qwen3.8-27B at RCO-IQ3_XXS (1.56 GiB lighter than the +# IQ3_S-mtp it replaced; the lighter quant funds the MTP draft ctx plus +# the second 131,072 slot). Rollback payload stays on disk: +# /home/dank/models/Swift15-IQ3S/ (the IQ3_S-mtp that was live until this +# swap); earlier payloads Ornith-Q8/ and MiMo-Q6K/ as well. Their +# re-fetch recipes are in git history (predecessors of this script). # # Usage: # scripts/download-frank-small.sh [DEST_DIR] [--verify-only] @@ -39,12 +35,12 @@ for arg in "$@"; do *) DEST="$arg" ;; esac done -DEST="${DEST:-/home/dank/models/Swift15-IQ3S}" +DEST="${DEST:-/home/dank/models/Swift15-UncIQ3XXS}" -# repo | hub-name | dest-name | bytes | sha256 (the Hub LFS oid) +# repo | hub-name | dest-name | bytes | sha256 (computed locally; Xet repo) FILES=( - "ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF|Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf|Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf|12120016896|9aecf1cd41b2cb2f32a74e0d889e33855ebef43b26f43b43feb5720239e677e5" - "unsloth/Qwen3.8-27B-GGUF|mmproj-F16.gguf|mmproj-Qwen3.8-27B-F16.gguf|927607488|cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e" + "ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF|RCO/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf|Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf|10442827392|e13d3e7400b6c7f980ebd30c0edd6be43eb1865b50be8562a9db042706c7d803" + "ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF|mmproj-BF16.gguf|mmproj-BF16.gguf|931145888|19acd7fcf4ee504328566eddf0f8e603ef853a205b4830209d2a4908213abdea" ) mkdir -p "$DEST" diff --git a/scripts/frank-small.env b/scripts/frank-small.env index be4f8a8..a4312bc 100644 --- a/scripts/frank-small.env +++ b/scripts/frank-small.env @@ -1,71 +1,55 @@ # frank-small — llama.cpp (master, CUDA sm_86) serving -# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S **-mtp** on the RTX 3090, -# :7771. +# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS +# **with the MTP head in the file** on the RTX 3090, :7771. # -# 2026-10-05 (Frank, evening): the payload moved to Swift-1.5 Qwen3.8-27B -# GSQ-RCO IQ3_S WITH the MTP head in the file, each conversation capped at -# 131,072 tokens, KV at FP8 class, vision on. Header-verified facts that -# re-price the shape (read from the Hub file 2026-10-05, HTTP range + struct -# parse): qwen35 hybrid like MiMo/Ornith but bigger — block_count 65 (64 real -# layers + the blk.64 MTP head), full_attention_interval 4 => SIXTEEN -# full-attention layers (MiMo had 8, Ornith 9), 24 Q heads / 4 KV heads, -# k/v length 256, native window 262144, dense, nextn_predict_layers=1 with -# blk.64.nextn.{enorm,hnorm,shared_head_norm} (the -mtp file's head tensors -# are Q6_K/F32 per the publisher RECIPE, "MTP_OK" verified). +# 2026-10-06 (Frank, evening): the payload moved to ajgazin's UNCELERSED +# Swift-1.5-Qwen3.8-27B at RCO-IQ3_XXS (10,442,827,392 B) — 1.56 GiB +# lighter than the IQ3_S-mtp it replaces — with MTP BACK ON (draft-mtp, +# depth 3), TWO 131,072 slots, and ubatch 512. The uncensored finetune +# changes behavior by design (publisher: 23/100 refusals vs Swift 1.5's +# 98/100, KL 0.0884 from Swift 1.5) — Frank's order. The RCO tier is the +# same per-tensor ALLOCATION recipe as the IQ3_S file (histogram: iq3_s +# 97 / bf16 96 / iq3_xxs 83 / iq2_s 72 / iq4_xs 33 / iq2_xs 32 / iq2_xxs +# 30 / q2_k 28 / q4_k 19 / q6_k 8 / iq1_s 4 / iq1_m 4 — all stock types, +# stock llama.cpp loads it); publisher KLD: Mean 0.002631, 90 % 0.0421 +# (kld_results.csv). RCO files are "more experimental" per the README — +# the quality floor is the IQ3_S rung, not a promise. # -# THE KV CONTRACT IS NOW q8_0 K + q8_0 V — Frank's order for THIS swap -# ("k/v at FP8", 2026-10-05), superseding the same-day f16/f16 rule, which -# was written for the Ornith 9B fit ("no further quantization anywhere on -# this endpoint"). llama.cpp has no true fp8 KV dtype; q8_0 is the 8-bit -# class (1.0625 B/elem => 8.5 effective bits) and the only FP8-class lever -# this runtime moves. Arithmetic that makes the order load-bearing: -# KV rate = 16 attn layers x 4 heads x 256 x 1.0625 B, K and V again -# => 34 KiB/token (f16 would be 64 KiB/token — halogen's 27B card quotes -# the same f16 number; at f16 the 131,072 slot alone would cost 8.0 GiB). +# The architecture is unchanged (same model family): qwen35 hybrid — +# block_count 65 (64 real layers + the blk.64 MTP head), +# full_attention_interval 4 => SIXTEEN full-attention layers, 24 Q heads +# / 4 KV heads, k/v length 256, native window 262144, dense, +# nextn_predict_layers=1. The MTP head ships in every main GGUF of this +# repo (README); --spec-type draft-mtp loads it against the main model's +# own weights. # -# 2026-10-06 (Frank): drop the MTP drafter (skip ngram too — on a qwen35 -# hybrid, ngram speculation disables recurrent-state rollback: fleet lesson -# from the frank-large chain measurement), drop ubatch to 512, and take TWO -# conversations at 131,072 — "should be a very tight fit", accepted -# explicitly, i.e. the 2-3 GiB floor is knowingly violated. Measured at the -# 1x131072 MTP boot (2026-10-05): 19,404 used / 4,721 MiB free. +# THE KV CONTRACT STAYS q8_0 K + q8_0 V (Frank, 2026-10-05): 34 KiB/token +# (16 attn layers x 4 heads x 256 x 1.0625 B, K and V). # -# Why TWO slots at 131,072 fits, by arithmetic (pool = 2 x 131,072 = -# 262,144; the per-conversation cap stays Frank's 128k): +# Why TWO slots + MTP fits now (it did not under IQ3_S): +# card 24,576 MiB. Weights 9,963 MiB (10,442,827,392 B; IQ3_S was +# 11,561). Projector ~1,290 MiB (the PUBLISHER's OWN mmproj-BF16.gguf, +# 931,145,888 B, at the measured ~1.45x card law — no substitution). +# Compute buffers at ubatch 512 ~1.3 GiB. CUDA ctx ~500 MiB. DeltaNet +# state ~150 MiB/slot x 2. MTP draft ctx back: ~0.4 GiB. Non-KV ~13.9 +# GiB. Two q8_0 slots = 8,704 MiB. Predicted free ~2.6-3.2 GiB — back +# over the old 2-3 GiB floor that the IQ3_S two-slot boot violated +# (measured there: 1,397 MiB free). MEASURE nvidia-smi free at boot +# (quote the free field) and let the runbook carry the number. # -# card 24,576 MiB. Weights 11,561 MiB (the IQ3_S-mtp file; the GSQ -# "IQ3_S" tier is a per-tensor ALLOCATION mix — histogram IQ2_XS 144 / -# IQ3_XXS 96 / Q6_K 78 / F32 360 / BF16 96 / Q3_K_M 39 ... all stock -# types, stock llama.cpp loads it). Projector ~1,350 MiB (927,607,488 B -# F16 at the measured ~1.45x card law). Compute buffers at ubatch 512 -# ~1.3 GiB (inferred from the measured 616 MiB/doubling rung on the 9B). -# CUDA ctx ~500 MiB. DeltaNet state ~150 MiB/slot x 2. NO draft context -# (the MTP draft ctx cost ~0.3-0.5 GiB and is gone with speculation). -# Non-KV ~15.5 GiB. Two q8_0 slots = 8,704 MiB (34 KiB/token x 131,072 -# each). Predicted free at boot ~0.3-1.3 GiB. Compute buffers allocate -# at graph capture during BOOT, so a miss is loud and fast; MEASURE -# nvidia-smi free at boot (quote the free field) and let the runbook -# carry the number. -# -# Payload history (for the rollback trail): the Ornith-1.5-9B Q8_0 swap of -# earlier on 2026-10-05 (commits 77decb5..380aafa) was committed but never -# deployed — the live service was still MiMo-V2.6 Q6_K (2 x 200,704, f16 -# KV) when Swift cut over. Rollback payloads on disk: Ornith-Q8/ and -# MiMo-Q6K/ (one MODEL_FILE + shape edit back; see the runbook). +# Payload history (rollback trail): /home/dank/models/Swift15-IQ3S/ (the +# IQ3_S-mtp that was live until this swap), Ornith-Q8/, MiMo-Q6K/ all +# stay on disk; one MODEL_FILE + shape edit back each. LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server -MODEL_FILE=/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf +MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf -# Vision: no Swift repo ships a projector (ukisai's release is -# language-model-only by their own README), so the projector comes from the -# SHARED BASE — unsloth's mmproj-F16.gguf built from Qwen/Qwen3.8-27B, -# stored as mmproj-Qwen3.8-27B-F16.gguf (927,607,488 B, LFS sha256 -# cbb841a9..., fetched by scripts/download-frank-small.sh). The substitution -# is proven, not assumed: Swift-1.5's vision stack (tower + merger) was -# hash-compared against the base on 2026-10-05 — 333/333 visual tensor names -# and 6/6 sampled tensors byte-identical — so the base projector IS the -# model's own projector (Swift's RL/OPD post-training touched text only). +# Vision: the publisher SHIPS the projector — mmproj-BF16.gguf +# (931,145,888 B) from the same repo, fetched by +# scripts/download-frank-small.sh. No substitution needed for this +# payload (the unsloth base-tower proof of 2026-10-05 remains the +# fallback argument for the IQ3_S rollback payload). # Empty string = text-only on purpose. -MMPROJ_FILE=/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf +MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf # Fleet endpoint: same port and alias as every frank-small before it, so the # harness models.yml id and the homepage probe do not move. @@ -100,13 +84,14 @@ UBATCH_SIZE=512 CACHE_TYPE_K=q8_0 CACHE_TYPE_V=q8_0 -# Speculative decoding is OFF (Frank, 2026-10-06): no draft-mtp, and -# ngram-mod is skipped too — on a qwen35 hybrid ngram disables -# recurrent-state rollback, so a rejected draft would corrupt the DeltaNet -# state. SPEC_TYPE=none strips BOTH speculative flags in the serve script -# (flags move as a set). The -mtp file still carries the head; it is simply -# not drafted with. Measured MTP reference (2026-10-05): 44.8 % acceptance -# at the recipe temperature => 58.9 t/s, 52.1 % at temp 0.8 => 66.1 t/s. -SPEC_TYPE=none +# Speculative decoding is ON (Frank, 2026-10-06 evening): draft-mtp at +# depth 3 against the head IN the file (the -mtp variant ships the +# blk.64.nextn.* tensors). ngram-mod stays skipped — on a qwen35 hybrid +# it disables recurrent-state rollback, so a rejected draft would +# corrupt the DeltaNet state; MTP needs no such bypass (its draft ctx +# re-derives state from the target's own weights). The flags move as a +# set: the serve script emits BOTH only when SPEC_TYPE names a drafter. +SPEC_TYPE=draft-mtp +SPEC_DRAFT_N_MAX=3 NGL=99 diff --git a/scripts/frank-small.service b/scripts/frank-small.service index 2dcef60..a897f9c 100644 --- a/scripts/frank-small.service +++ b/scripts/frank-small.service @@ -1,5 +1,5 @@ [Unit] -Description=llama.cpp Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S-mtp GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, no speculative decoding, mmproj vision) on the RTX 3090 (dankclaw, frank-small) +Description=llama.cpp Swift-1.5-Qwen3.8-27B Uncensored RCO IQ3_XXS GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, draft-mtp depth 3, mmproj vision) on the RTX 3090 (dankclaw, frank-small) After=network-online.target Wants=network-online.target diff --git a/scripts/serve-frank-small.sh b/scripts/serve-frank-small.sh index 39753e0..727bc43 100755 --- a/scripts/serve-frank-small.sh +++ b/scripts/serve-frank-small.sh @@ -1,24 +1,24 @@ #!/usr/bin/env bash -# Serve ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF (IQ3_S **-mtp**) as -# frank-small on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA -# sm_86). +# Serve ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF +# (RCO-IQ3_XXS, **MTP head in the file**) as frank-small on Dankclaw's +# RTX 3090 via llama.cpp (master build, CUDA sm_86). # -# 2026-10-06 (Frank): the 128k cap keeps the per-conversation window but -# the pool doubles to TWO 131,072 conversations (262144 / 2), speculative -# decoding is OFF (no draft-mtp; ngram skipped — on a qwen35 hybrid it -# disables recurrent-state rollback), and ubatch drops to 512. Frank -# accepted the tight fit explicitly (the 2-3 GiB floor is knowingly -# violated; frank-small.env carries the arithmetic). The q8_0 KV contract -# (FP8 class) is unchanged; the per-tensor flags stay explicit either way. +# 2026-10-06 (Frank): TWO 131,072 conversations (262144 / 2) at the 128k +# cap, ubatch 512, and MTP BACK ON (draft-mtp, depth 3 — the payload is +# 1.56 GiB lighter than the IQ3_S-mtp it replaced, which is what funds +# the draft ctx plus the second slot; ngram stays skipped — on a qwen35 +# hybrid it disables recurrent-state rollback). The q8_0 KV contract +# (FP8 class) is unchanged; the per-tensor flags stay explicit either +# way. # # Architecture: qwen35 hybrid — 64 real layers, 16 full-attention + 48 # Gated DeltaNet (full_attention_interval 4), native 262144 window, plus # the blk.64 MTP head (nextn_predict_layers=1). KV lives only in the 16 # attention layers (34 KiB/token at q8_0); the DeltaNet state is per-slot -# fixed size. Vision comes from the projector file (the GGUF carries no -# vision tensors; unsloth's mmproj built from the shared Qwen3.8-27B base — -# proven byte-identical to Swift's own tower, see download-frank-small.sh), -# so --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the +# fixed size. +# Vision comes from the projector file (the GGUF carries no vision +# tensors; the publisher SHIPS mmproj-BF16.gguf in the same repo), so +# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the # documented text-only opt-out. # # Sampling: llama.cpp takes it per request (the fleet harness sends its @@ -38,7 +38,7 @@ fi # --- defaults (used when the env file does not set them) ------------------- : "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}" -: "${MODEL_FILE:=/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}" +: "${MODEL_FILE:=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}" : "${ALIAS:=frank-small}" : "${BIND_ADDR:=0.0.0.0}" : "${LISTEN_PORT:=7771}" @@ -61,15 +61,16 @@ fi # visible in the argv. : "${CACHE_TYPE_K:=q8_0}" : "${CACHE_TYPE_V:=q8_0}" -# Speculative decoding is OFF by default (Frank, 2026-10-06): SPEC_TYPE -# none strips BOTH speculative flags (flags move as a set). Setting a -# drafter here re-enables the pair — draft-mtp needs the head that is IN -# the -mtp GGUF. -: "${SPEC_TYPE:=none}" +# Speculative decoding is ON by default (Frank, 2026-10-06 evening): +# draft-mtp against the head that is IN the file. SPEC_TYPE none strips +# BOTH speculative flags (flags move as a set) — a no-spec baseline must +# not carry half a spec pair. +: "${SPEC_TYPE:=draft-mtp}" +: "${SPEC_DRAFT_N_MAX:=3}" : "${NGL:=99}" # Vision projector. No colon on purpose: an explicitly empty value means # "text-only, on purpose" and must NOT be replaced by the default path. -: "${MMPROJ_FILE=/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf}" +: "${MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}" # Fail loudly rather than half-loading: a missing file is a download that # has not finished, and Restart=always would turn it into a crash loop that diff --git a/tests/frank_small_unit.bats b/tests/frank_small_unit.bats index 55d3b4d..5f2f67e 100755 --- a/tests/frank_small_unit.bats +++ b/tests/frank_small_unit.bats @@ -1,15 +1,15 @@ -#!/usr/bin/env bats # Seam: the frank-small llama.cpp root unit on dankclaw. -# frank-small is the RTX 3090 endpoint; since 2026-10-06 it serves -# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B; the -mtp file's -# head is present but NOT drafted — speculation is off) via a CUDA (sm_86) -# build of llama.cpp master: TWO 131,072-token conversations at Frank's -# 128k cap (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), and a -# vision projector. Frank accepted the tight fit explicitly — the card may -# land under the old 2-3 GiB floor; the seam pins the arithmetic, the boot -# log carries the measured free. This seam runs the serve script against a -# stub engine and asserts on the argv it actually execs, so what is pinned -# is the command the operator gets. +# frank-small is the RTX 3090 endpoint; since 2026-10-06 (evening) it +# serves ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at +# RCO-IQ3_XXS (27B dense qwen3_5 hybrid) via a CUDA (sm_86) build of +# llama.cpp master: TWO 131,072-token conversations at Frank's 128k cap +# (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), the +# publisher's own BF16 vision projector, and MTP back ON (draft-mtp +# depth 3 — the head is in the file; the lighter quant funds the draft +# ctx plus the second slot). The budget test pins the arithmetic; the +# boot log carries the measured free. This seam runs the serve script +# against a stub engine and asserts on the argv it actually execs, so +# what is pinned is the command the operator gets. load helpers @@ -18,9 +18,9 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh" ENV_FILE="$REPO_ROOT/scripts/frank-small.env" REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" ENV_LARGE="$REPO_ROOT/scripts/frank-large.env" -TARGET_STUB="Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf" +TARGET_STUB="Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf" # Same basename as the shipped projector: the test env repoints it into $STAGE. -MM_STUB="mmproj-Qwen3.8-27B-F16.gguf" +MM_STUB="mmproj-BF16.gguf" setup() { STAGE="$(mktemp -d)" @@ -83,9 +83,10 @@ argv_value() { grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT" } -@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B IQ3_S-mtp GGUF" { +@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B Uncensored RCO-IQ3_XXS GGUF" { + # The payload path IS the endpoint's identity. grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE" - grep -qE "^MODEL_FILE=/home/dank/models/Swift15-IQ3S/$TARGET_STUB$" "$ENV_FILE" + grep -qE "^MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/$TARGET_STUB$" "$ENV_FILE" } @test "the deployer gates on exactly the payload paths the env file ships" { @@ -102,7 +103,7 @@ argv_value() { done } -@test "the shipped env serves the fleet shape: one conversation at the 128k cap" { +@test "the pool is exactly the native 262144 window, so each of the two conversations is 131072" { serve_argv [ "$status" -eq 0 ] [ "$(argv_value --alias)" = "frank-small" ] @@ -135,27 +136,26 @@ argv_value() { [ "$(argv_value --flash-attn)" = "on" ] } -@test "the KV budget fits TWO 128k slots (Frank's tight-fit order) and forbids a third" { - # Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub - # file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64 - # real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv - # heads x 256 dim x 1.0625 B = 17 KiB/token for K and the same for V - # => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64). - # The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) + - # vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured - # ~1.45x cost law) + compute buffers at ubatch 512 + CUDA graphs + TWO - # DeltaNet states (~150 MiB each) and NO draft context (speculation is - # off) => 15 GiB budgeted. 2026-10-06 (Frank): two 131,072 slots are - # the order — the budget lands AT the card (tight fit, accepted - # explicitly; the old 2-3 GiB floor is knowingly violated) and a THIRD - # slot must not fit. Arithmetic, not a boot log: change CTX_SIZE and - # this bites before the deploy does; the boot's nvidia-smi free is the - # measured truth. +@test "the KV budget fits two 131072 slots at q8_0 and forbids a third" { + # Rate, from the RCO-IQ3_XXS GGUF header (same qwen35 architecture): + # SIXTEEN full-attention layers (block_count 65 = 64 real layers + the + # blk.64 MTP head, full_attention_interval 4) x 4 kv heads x 256 dim x + # 1.0625 B = 17 KiB/token for K and the same for V => 34 KiB/token at + # q8_0 (budgeted 37 with slack; f16 would be 64). The non-KV half: + # RCO-IQ3_XXS weights 10,148 MiB (10,442,827,392 B) — 1.56 GiB LIGHTER + # than the IQ3_S-mtp predecessor — + vision tower ~1.4 GiB (931,145,888 + # B BF16 mmproj at the measured ~1.45x cost law) + compute buffers at + # ubatch 512 + CUDA graphs + TWO DeltaNet states (~150 MiB each) + the + # MTP draft ctx => 14 GiB budgeted. Two 131,072 slots are the + # 2026-10-06 shape (the lighter quant funds the draft ctx plus the + # second slot); a THIRD slot must not fit. Arithmetic, not a boot log: + # change CTX_SIZE and this bites before the deploy does; the boot's + # nvidia-smi free is the measured truth. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 37 / 1048576)) third_slot_kv=$(( (ctx + 131072) * 37 / 1048576 )) - [ $((kv_gib + 15)) -le 24 ] - [ $((third_slot_kv + 15)) -gt 24 ] + [ $((kv_gib + 14)) -le 24 ] + [ $((third_slot_kv + 14)) -gt 24 ] } @test "the compute-buffer pair follows the env file and stays self-consistent" { @@ -185,19 +185,21 @@ argv_value() { argv_value --ubatch-size | grep -qE '^[0-9]+$' } -@test "speculation is off: no half spec pair reaches the argv" { - # Frank, 2026-10-06: drop MTP, skip ngram-mod (on a qwen35 hybrid - # ngram disables recurrent-state rollback — a rejected draft corrupts - # the DeltaNet state). The flags move as a set, so the no-spec contract - # is ABSENCE of both in the argv the supervisor runs, and the env must - # carry the same contract. The -mtp file's head stays; it is not - # drafted with. Reference numbers stay in frank-small.env for the day - # MTP returns (44.8 % acceptance at the recipe temp => 58.9 t/s). +@test "speculation is on: the full draft-mtp pair reaches the argv" { + # Frank, 2026-10-06 evening: MTP back on. The head is IN the file, so + # draft-mtp drafts against the main model's weights (no second model + # copy). ngram-mod stays skipped: on a qwen35 hybrid it disables + # recurrent-state rollback — a rejected draft corrupts the DeltaNet + # state. The flags move as a set: the pair is present together or + # absent together — assert both in the argv, and the env carries the + # same contract. The serve script's defaults ship the pair even for an + # env that names no drafter. serve_argv [ "$status" -eq 0 ] - ! grep -qxF -- '--spec-type' <<<"$output" - ! grep -qxF -- '--spec-draft-n-max' <<<"$output" - grep -qE '^SPEC_TYPE=none$' "$ENV_FILE" + [ "$(argv_value --spec-type)" = "draft-mtp" ] + [ "$(argv_value --spec-draft-n-max)" = "3" ] + grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE" + grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE" } @test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {