dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-small: Swift-1.5-27B Uncensored RCO-IQ3_XXS with MTP back on (two 131072 slots, ubatch 512, q8_0 KV, publisher's BF16 mmproj)

Browse Source
This commit is contained in:
Frank
2026-10-06 01:10:43 -07:00
parent 713b126639
commit da36fa32cf
6 changed files with 151 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/frank_small_unit.bats
+47 -45
View File
@@ -1,15 +1,15 @@
#!/usr/bin/env bats
# Seam: the frank-small llama.cpp root unit on dankclaw.
# frank-small is the RTX 3090 endpoint; since 2026-10-06 it serves
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B; the -mtp file's
# head is present but NOT drafted — speculation is off) via a CUDA (sm_86)
# build of llama.cpp master: TWO 131,072-token conversations at Frank's
# 128k cap (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), and a
# vision projector. Frank accepted the tight fit explicitly — the card may
# land under the old 2-3 GiB floor; the seam pins the arithmetic, the boot
# log carries the measured free. This seam runs the serve script against a
# stub engine and asserts on the argv it actually execs, so what is pinned
# is the command the operator gets.
# frank-small is the RTX 3090 endpoint; since 2026-10-06 (evening) it
# serves ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at
# RCO-IQ3_XXS (27B dense qwen3_5 hybrid) via a CUDA (sm_86) build of
# llama.cpp master: TWO 131,072-token conversations at Frank's 128k cap
# (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), the
# publisher's own BF16 vision projector, and MTP back ON (draft-mtp
# depth 3 — the head is in the file; the lighter quant funds the draft
# ctx plus the second slot). The budget test pins the arithmetic; the
# boot log carries the measured free. This seam runs the serve script
# against a stub engine and asserts on the argv it actually execs, so
# what is pinned is the command the operator gets.
load helpers
@@ -18,9 +18,9 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
TARGET_STUB="Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf"
TARGET_STUB="Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf"
# Same basename as the shipped projector: the test env repoints it into $STAGE.
MM_STUB="mmproj-Qwen3.8-27B-F16.gguf"
MM_STUB="mmproj-BF16.gguf"
setup() {
STAGE="$(mktemp -d)"
@@ -83,9 +83,10 @@ argv_value() {
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
}
@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B IQ3_S-mtp GGUF" {
@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B Uncensored RCO-IQ3_XXS GGUF" {
# The payload path IS the endpoint's identity.
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
grep -qE "^MODEL_FILE=/home/dank/models/Swift15-IQ3S/$TARGET_STUB$" "$ENV_FILE"
grep -qE "^MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/$TARGET_STUB$" "$ENV_FILE"
}
@test "the deployer gates on exactly the payload paths the env file ships" {
@@ -102,7 +103,7 @@ argv_value() {
done
}
@test "the shipped env serves the fleet shape: one conversation at the 128k cap" {
@test "the pool is exactly the native 262144 window, so each of the two conversations is 131072" {
serve_argv
[ "$status" -eq 0 ]
[ "$(argv_value --alias)" = "frank-small" ]
@@ -135,27 +136,26 @@ argv_value() {
[ "$(argv_value --flash-attn)" = "on" ]
}
@test "the KV budget fits TWO 128k slots (Frank's tight-fit order) and forbids a third" {
# Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub
# file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64
# real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv
# heads x 256 dim x 1.0625 B = 17 KiB/token for K and the same for V
# => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64).
# The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) +
# vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured
# ~1.45x cost law) + compute buffers at ubatch 512 + CUDA graphs + TWO
# DeltaNet states (~150 MiB each) and NO draft context (speculation is
# off) => 15 GiB budgeted. 2026-10-06 (Frank): two 131,072 slots are
# the order — the budget lands AT the card (tight fit, accepted
# explicitly; the old 2-3 GiB floor is knowingly violated) and a THIRD
# slot must not fit. Arithmetic, not a boot log: change CTX_SIZE and
# this bites before the deploy does; the boot's nvidia-smi free is the
# measured truth.
@test "the KV budget fits two 131072 slots at q8_0 and forbids a third" {
# Rate, from the RCO-IQ3_XXS GGUF header (same qwen35 architecture):
# SIXTEEN full-attention layers (block_count 65 = 64 real layers + the
# blk.64 MTP head, full_attention_interval 4) x 4 kv heads x 256 dim x
# 1.0625 B = 17 KiB/token for K and the same for V => 34 KiB/token at
# q8_0 (budgeted 37 with slack; f16 would be 64). The non-KV half:
# RCO-IQ3_XXS weights 10,148 MiB (10,442,827,392 B) — 1.56 GiB LIGHTER
# than the IQ3_S-mtp predecessor — + vision tower ~1.4 GiB (931,145,888
# B BF16 mmproj at the measured ~1.45x cost law) + compute buffers at
# ubatch 512 + CUDA graphs + TWO DeltaNet states (~150 MiB each) + the
# MTP draft ctx => 14 GiB budgeted. Two 131,072 slots are the
# 2026-10-06 shape (the lighter quant funds the draft ctx plus the
# second slot); a THIRD slot must not fit. Arithmetic, not a boot log:
# change CTX_SIZE and this bites before the deploy does; the boot's
# nvidia-smi free is the measured truth.
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
kv_gib=$((ctx * 37 / 1048576))
third_slot_kv=$(( (ctx + 131072) * 37 / 1048576 ))
[ $((kv_gib + 15)) -le 24 ]
[ $((third_slot_kv + 15)) -gt 24 ]
[ $((kv_gib + 14)) -le 24 ]
[ $((third_slot_kv + 14)) -gt 24 ]
}
@test "the compute-buffer pair follows the env file and stays self-consistent" {
@@ -185,19 +185,21 @@ argv_value() {
argv_value --ubatch-size | grep -qE '^[0-9]+$'
}
@test "speculation is off: no half spec pair reaches the argv" {
# Frank, 2026-10-06: drop MTP, skip ngram-mod (on a qwen35 hybrid
# ngram disables recurrent-state rollback — a rejected draft corrupts
# the DeltaNet state). The flags move as a set, so the no-spec contract
# is ABSENCE of both in the argv the supervisor runs, and the env must
# carry the same contract. The -mtp file's head stays; it is not
# drafted with. Reference numbers stay in frank-small.env for the day
# MTP returns (44.8 % acceptance at the recipe temp => 58.9 t/s).
@test "speculation is on: the full draft-mtp pair reaches the argv" {
# Frank, 2026-10-06 evening: MTP back on. The head is IN the file, so
# draft-mtp drafts against the main model's weights (no second model
# copy). ngram-mod stays skipped: on a qwen35 hybrid it disables
# recurrent-state rollback — a rejected draft corrupts the DeltaNet
# state. The flags move as a set: the pair is present together or
# absent together — assert both in the argv, and the env carries the
# same contract. The serve script's defaults ship the pair even for an
# env that names no drafter.
serve_argv
[ "$status" -eq 0 ]
! grep -qxF -- '--spec-type' <<<"$output"
! grep -qxF -- '--spec-draft-n-max' <<<"$output"
grep -qE '^SPEC_TYPE=none$' "$ENV_FILE"
[ "$(argv_value --spec-type)" = "draft-mtp" ]
[ "$(argv_value --spec-draft-n-max)" = "3" ]
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE"
}
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {