frank-small: Swift-1.5 IQ3_S-mtp swap (parallel session) + fleet docs index
The parallel session's uncommitted swap, committed at Frank's direction (commit whatever's in the working tree): - llama.cpp master on the RTX 3090 now serves ukisai's Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S WITH the in-file MTP head, ONE 131,072-token slot at Frank's 128k cap, q8_0 K + q8_0 V, vision through the base-model projector; MiMo-V2.6 Q6_K (2x200704 f16 KV) retired to rollback target. - env/serve/download + seam twins updated together (deployer payload gates rode the previous commit's deploy-remote.sh). - docs: new swift15 runbook; MiMo Q6K runbook retired; AGENTS.md topology and both vault indexes follow (root index ACTIVE/RETIRED flip, test 135 seam green). - comfyui: two qwen-image t2i workflow drops under dankclaw/comfyui/ workflows/ (12 KiB).
This commit is contained in:
Frank
+51
-49
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-small llama.cpp root unit on dankclaw.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-05 it serves
|
||||
# ornith-ai/Ornith-1.5-9B-GGUF:Q8_0 via a CUDA (sm_86) build of
|
||||
# llama.cpp master, chosen because it is the only runtime here that moves K and
|
||||
# V independently. Since 2026-10-03 it runs TWO unquantized contexts of
|
||||
# 200,704 tokens each and carries a vision projector, and the card must keep
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-05 evening it serves
|
||||
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B, MTP head in the
|
||||
# file) via a CUDA (sm_86) build of llama.cpp master: ONE 131,072-token
|
||||
# conversation at Frank's 128k cap, q8_0 K + q8_0 V (FP8 class), the
|
||||
# in-file MTP head drafting 3, and a vision projector. The card must keep
|
||||
# 2-3 GiB free. This seam runs the serve script against a stub engine and
|
||||
# asserts on the argv it actually execs, so what is pinned is the
|
||||
# command the operator gets.
|
||||
@@ -16,9 +16,9 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
|
||||
TARGET_STUB="Ornith-1.5-9B-Q8_0.gguf"
|
||||
TARGET_STUB="Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf"
|
||||
# Same basename as the shipped projector: the test env repoints it into $STAGE.
|
||||
MM_STUB="mmproj-Ornith-1.5-9B-BF16.gguf"
|
||||
MM_STUB="mmproj-Qwen3.8-27B-F16.gguf"
|
||||
|
||||
setup() {
|
||||
STAGE="$(mktemp -d)"
|
||||
@@ -81,9 +81,9 @@ argv_value() {
|
||||
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the shipped env points at the llama.cpp runtime and the Ornith-1.5-9B Q8_0 GGUF" {
|
||||
@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B IQ3_S-mtp GGUF" {
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=/home/dank/models/Ornith-Q8/$TARGET_STUB$" "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=/home/dank/models/Swift15-IQ3S/$TARGET_STUB$" "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||||
@@ -100,7 +100,7 @@ argv_value() {
|
||||
done
|
||||
}
|
||||
|
||||
@test "the shipped env serves the fleet shape: one native 262144-token context" {
|
||||
@test "the shipped env serves the fleet shape: one conversation at the 128k cap" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --alias)" = "frank-small" ]
|
||||
@@ -110,54 +110,50 @@ argv_value() {
|
||||
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
||||
# llama.cpp divides --ctx-size across slots and a mamba hybrid cannot
|
||||
# share one cache, so the observable contract is the per-slot window.
|
||||
# 2026-10-05 (Frank): NO KV quantization forces ONE slot at the NATIVE
|
||||
# window — Ornith's 9 full-attn layers price f16 KV at 36 KiB/token, and
|
||||
# two 200,704 slots cannot fit the 2-3 GiB floor on 24 GiB with Q8
|
||||
# weights. The window per conversation ROSE 200,704 -> 262,144.
|
||||
# 2026-10-05 evening (Frank): the 27B Swift-1.5 swap caps each
|
||||
# conversation at 131,072, and ONE slot is the maximum — a second
|
||||
# 131,072 slot does not fit the 2-3 GiB floor (KV budget test below).
|
||||
[ "$(argv_value --parallel)" -eq 1 ]
|
||||
ctx="$(argv_value --ctx-size)"
|
||||
par="$(argv_value --parallel)"
|
||||
[ $((ctx % par)) -eq 0 ]
|
||||
[ $((ctx / par)) -eq 262144 ]
|
||||
[ $((ctx / par)) -eq 131072 ]
|
||||
}
|
||||
|
||||
@test "the KV contract is unquantized both halves, flash-attn on" {
|
||||
@test "the KV contract is q8_0 both halves (FP8 class), flash-attn on" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --cache-type-k)" = "f16" ]
|
||||
# Frank, 2026-10-05: NO KV quantization on this endpoint. The Q8_0 rung
|
||||
# funds itself by UBATCH first (2048 -> 1024), then pool steps — a q8_0-V
|
||||
# proposal was written and overruled. The flag pair is what makes this
|
||||
# Frank, 2026-10-05 evening: "k/v at FP8" for the Swift-1.5 swap —
|
||||
# llama.cpp's FP8 class is q8_0 (1.0625 B/elem => 8.5 effective bits;
|
||||
# there is no true fp8 KV dtype). This SUPERSEDES the same-day f16/f16
|
||||
# rule written for the Ornith 9B fit. The flag pair is what makes this
|
||||
# observable — a build without the FA path silently falls back, so both
|
||||
# must appear in the argv the supervisor runs.
|
||||
[ "$(argv_value --cache-type-v)" = "f16" ]
|
||||
[ "$(argv_value --cache-type-k)" = "q8_0" ]
|
||||
[ "$(argv_value --cache-type-v)" = "q8_0" ]
|
||||
[ "$(argv_value --flash-attn)" = "on" ]
|
||||
}
|
||||
|
||||
@test "the KV budget leaves 2-3 GiB of the 3090 free" {
|
||||
# Rate, from the Ornith GGUF header (read from the Hub file, 2026-10-05):
|
||||
# NINE full-attention layers (block_count 33, full_attention_interval 4)
|
||||
# x 4 kv heads x 256 dim x 2 B = 18 KiB/token for K and the same for V
|
||||
# => 36 KiB/token (budgeted 37 with slack; MiMo measured 24.5 — one
|
||||
# fewer layer AND one fewer KV head). The non-KV half: Q8_0 weights
|
||||
# 9,334 MiB (9,786,060,384 B) + vision tower ~1.3 GiB (mmproj 879 MiB at
|
||||
# the measured ~1.5x cost law) + DeltaNet state + compute buffers +
|
||||
# CUDA graphs => 11 GiB budgeted. Two asserts, both about the output
|
||||
# the operator sees: the pool must still fit with 2 GiB spare, and it
|
||||
# must be large enough that 5 GiB spare is NOT reachable (an undersized
|
||||
# pool is as wrong as an oversized one).
|
||||
@test "the KV budget fits the 2-3 GiB floor and forbids a second 128k slot" {
|
||||
# Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub
|
||||
# file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64
|
||||
# real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv
|
||||
# heads x 256 dim x 1.0625 B = 17 KiB/token for K and the same for V
|
||||
# => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64).
|
||||
# The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) +
|
||||
# vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured
|
||||
# ~1.45x cost law) + DeltaNet states + compute buffers at ubatch 1024 +
|
||||
# CUDA graphs + the MTP draft context => 17 GiB budgeted. Two asserts,
|
||||
# both about the output the operator sees: the pool must still fit with
|
||||
# 2 GiB spare, and a SECOND 131,072 slot must not fit — Frank's "as
|
||||
# many 128k contexts as fit" has the answer 1 on this card, and the
|
||||
# pool itself is capped by the 128k law, not by the card.
|
||||
# Arithmetic, not a boot log: change CTX_SIZE and this bites before the
|
||||
# deploy does. At Ornith's rate the two-200704-slot pool (14.1 GiB KV)
|
||||
# FAILS the first assert — which is exactly why the shipped shape is
|
||||
# ONE native slot.
|
||||
# deploy does.
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv_gib=$((ctx * 37 / 1048576))
|
||||
[ $((kv_gib + 11 + 2)) -le 24 ]
|
||||
[ $((kv_gib + 11 + 5)) -gt 24 ]
|
||||
# And the ceiling that sets the per-slot window: two NATIVE windows cannot
|
||||
# be paid for in full precision on this card.
|
||||
native=$((2 * 262144))
|
||||
[ $((native * 37 / 1048576 + 11)) -gt 24 ]
|
||||
[ $((kv_gib + 17 + 2)) -le 24 ]
|
||||
[ $((2 * kv_gib + 17)) -gt 24 ]
|
||||
}
|
||||
|
||||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||||
@@ -187,16 +183,22 @@ argv_value() {
|
||||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||||
}
|
||||
|
||||
@test "no speculative draft ships by default" {
|
||||
# The qwen35 GGUF has no MTP tensors, and the cross-model DFlash draft
|
||||
# is a measured decision the harness makes later, not a default.
|
||||
! grep -q -- '--model-draft' "$SERVE"
|
||||
! grep -q -- '--spec-type' "$SERVE"
|
||||
! grep -q '^DRAFT' "$ENV_FILE"
|
||||
@test "the in-file MTP head ships as draft-mtp at depth 3" {
|
||||
# The -mtp GGUF carries the head (blk.64.nextn.*, nextn_predict_layers
|
||||
# =1): draft-mtp loads it against the main model's own weights — no
|
||||
# --model-draft and no second draft KV pool. Speculative flags move as
|
||||
# a set; both must appear in the argv the supervisor runs, and the env
|
||||
# must carry the same contract the argv executes.
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||||
[ "$(argv_value --spec-draft-n-max)" = "3" ]
|
||||
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
|
||||
grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
||||
# The Q6_K GGUF carries no `v.*` tensors, so the projector argument IS
|
||||
# The Swift GGUF carries no vision tensors, so the projector argument IS
|
||||
# the vision capability: no flag, no eyes.
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
Reference in New Issue
Block a user