frank-small: Swift-1.5-27B Uncensored RCO-IQ3_XXS with MTP back on (two 131072 slots, ubatch 512, q8_0 KV, publisher's BF16 mmproj)
This commit is contained in:
@@ -1207,14 +1207,13 @@ deploy_llm_large() {
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
# The projector IS the vision capability here: no Swift repo ships one (the
|
||||
# GGUF carries no vision tensors), so the file is unsloth's mmproj built
|
||||
# from the shared Qwen3.8-27B base — proven byte-identical to Swift's own
|
||||
# vision stack. A deploy that starts without this file silently ships a
|
||||
# text-only endpoint. Checked unconditionally, like frank-large's.
|
||||
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf}"
|
||||
# The projector IS the vision capability here: the GGUF carries no vision
|
||||
# tensors, and the publisher ships mmproj-BF16.gguf in the same repo, so a
|
||||
# deploy that starts without this file silently ships a text-only
|
||||
# endpoint. Checked unconditionally, like frank-large's.
|
||||
LLM_SMALL_MMPROJ="${LLM_SMALL_MMPROJ:-/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}"
|
||||
|
||||
deploy_llm_small() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
|
||||
@@ -1,31 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the frank-small payload for llama.cpp on the RTX 3090:
|
||||
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S **with the MTP head in the
|
||||
# file** (the -mtp variant), plus a vision projector. No Swift repo ships a
|
||||
# projector — ukisai's release is language-model-only ("A Swift 27B vision
|
||||
# projector has not been verified for this release") — so this script fetches
|
||||
# unsloth's mmproj-F16.gguf built from the shared base Qwen/Qwen3.8-27B.
|
||||
# That substitution is proven, not assumed: on 2026-10-05 Swift-1.5's vision
|
||||
# stack was hash-compared against the base over HTTP range reads — 333/333
|
||||
# visual tensor names and 6/6 sampled tensors byte-identical (merger fc1/fc2,
|
||||
# blocks 0/13/26) — so the base-model projector IS the model's own projector.
|
||||
# Public repos — no auth header, and never a token on the command line (it is
|
||||
# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS
|
||||
# **with the MTP head in the file** (the -mtp variant), plus the
|
||||
# PUBLISHER's OWN vision projector (mmproj-BF16.gguf, same repo). Public
|
||||
# repo — no auth header, and never a token on the command line (it is
|
||||
# world-readable in /proc/<pid>/cmdline).
|
||||
#
|
||||
# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized file
|
||||
# on a 31 GiB payload); -f/--fail-with-body so an HTTP error is not a success,
|
||||
# then byte count and the Hub LFS sha256. Idempotent: complete files are
|
||||
# skipped. The projector is stored under a self-documenting dest name; the
|
||||
# URL name and sha stay pinned in the table below.
|
||||
# Fresh downloads, no resume (`curl -C -` once produced a 213 MB oversized
|
||||
# file on a 31 GiB payload); -f/--fail-with-body so an HTTP error is not a
|
||||
# success, then byte count and sha256. This repo is on Xet storage, so
|
||||
# the Hub tree/blob APIs expose no LFS oid: the sha256 values below were
|
||||
# computed locally after the first verified fetch (byte counts match the
|
||||
# Hub's declared sizes). Idempotent: complete files are skipped. The
|
||||
# projector is stored under a self-documenting dest name; the URL name
|
||||
# and sha stay pinned in the table below.
|
||||
#
|
||||
# 2026-10-05 (Frank, evening): the payload moved to Swift-1.5-27B GSQ-RCO
|
||||
# IQ3_S-mtp — a 27B on the 24 GiB card: ONE 131,072-token slot, q8_0 K + q8_0
|
||||
# V, the in-file MTP head drafting 3. This supersedes the Ornith-1.5-9B Q8_0
|
||||
# swap (commits 77decb5..380aafa, same day), which was committed but never
|
||||
# deployed — the live service was still MiMo-V2.6 Q6_K when Swift cut over.
|
||||
# Rollback payloads stay on disk: /home/dank/models/Ornith-Q8/ and
|
||||
# /home/dank/models/MiMo-Q6K/ (the payload that was live until this swap).
|
||||
# Their re-fetch recipes are in git history (predecessors of this script).
|
||||
# 2026-10-06 (Frank, evening): the payload moved to the UNCELERSED
|
||||
# Swift-1.5-Qwen3.8-27B at RCO-IQ3_XXS (1.56 GiB lighter than the
|
||||
# IQ3_S-mtp it replaced; the lighter quant funds the MTP draft ctx plus
|
||||
# the second 131,072 slot). Rollback payload stays on disk:
|
||||
# /home/dank/models/Swift15-IQ3S/ (the IQ3_S-mtp that was live until this
|
||||
# swap); earlier payloads Ornith-Q8/ and MiMo-Q6K/ as well. Their
|
||||
# re-fetch recipes are in git history (predecessors of this script).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/download-frank-small.sh [DEST_DIR] [--verify-only]
|
||||
@@ -39,12 +35,12 @@ for arg in "$@"; do
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
DEST="${DEST:-/home/dank/models/Swift15-IQ3S}"
|
||||
DEST="${DEST:-/home/dank/models/Swift15-UncIQ3XXS}"
|
||||
|
||||
# repo | hub-name | dest-name | bytes | sha256 (the Hub LFS oid)
|
||||
# repo | hub-name | dest-name | bytes | sha256 (computed locally; Xet repo)
|
||||
FILES=(
|
||||
"ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF|Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf|Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf|12120016896|9aecf1cd41b2cb2f32a74e0d889e33855ebef43b26f43b43feb5720239e677e5"
|
||||
"unsloth/Qwen3.8-27B-GGUF|mmproj-F16.gguf|mmproj-Qwen3.8-27B-F16.gguf|927607488|cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e"
|
||||
"ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF|RCO/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf|Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf|10442827392|e13d3e7400b6c7f980ebd30c0edd6be43eb1865b50be8562a9db042706c7d803"
|
||||
"ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF|mmproj-BF16.gguf|mmproj-BF16.gguf|931145888|19acd7fcf4ee504328566eddf0f8e603ef853a205b4830209d2a4908213abdea"
|
||||
)
|
||||
|
||||
mkdir -p "$DEST"
|
||||
|
||||
+52
-67
@@ -1,71 +1,55 @@
|
||||
# frank-small — llama.cpp (master, CUDA sm_86) serving
|
||||
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S **-mtp** on the RTX 3090,
|
||||
# :7771.
|
||||
# ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF:RCO-IQ3_XXS
|
||||
# **with the MTP head in the file** on the RTX 3090, :7771.
|
||||
#
|
||||
# 2026-10-05 (Frank, evening): the payload moved to Swift-1.5 Qwen3.8-27B
|
||||
# GSQ-RCO IQ3_S WITH the MTP head in the file, each conversation capped at
|
||||
# 131,072 tokens, KV at FP8 class, vision on. Header-verified facts that
|
||||
# re-price the shape (read from the Hub file 2026-10-05, HTTP range + struct
|
||||
# parse): qwen35 hybrid like MiMo/Ornith but bigger — block_count 65 (64 real
|
||||
# layers + the blk.64 MTP head), full_attention_interval 4 => SIXTEEN
|
||||
# full-attention layers (MiMo had 8, Ornith 9), 24 Q heads / 4 KV heads,
|
||||
# k/v length 256, native window 262144, dense, nextn_predict_layers=1 with
|
||||
# blk.64.nextn.{enorm,hnorm,shared_head_norm} (the -mtp file's head tensors
|
||||
# are Q6_K/F32 per the publisher RECIPE, "MTP_OK" verified).
|
||||
# 2026-10-06 (Frank, evening): the payload moved to ajgazin's UNCELERSED
|
||||
# Swift-1.5-Qwen3.8-27B at RCO-IQ3_XXS (10,442,827,392 B) — 1.56 GiB
|
||||
# lighter than the IQ3_S-mtp it replaces — with MTP BACK ON (draft-mtp,
|
||||
# depth 3), TWO 131,072 slots, and ubatch 512. The uncensored finetune
|
||||
# changes behavior by design (publisher: 23/100 refusals vs Swift 1.5's
|
||||
# 98/100, KL 0.0884 from Swift 1.5) — Frank's order. The RCO tier is the
|
||||
# same per-tensor ALLOCATION recipe as the IQ3_S file (histogram: iq3_s
|
||||
# 97 / bf16 96 / iq3_xxs 83 / iq2_s 72 / iq4_xs 33 / iq2_xs 32 / iq2_xxs
|
||||
# 30 / q2_k 28 / q4_k 19 / q6_k 8 / iq1_s 4 / iq1_m 4 — all stock types,
|
||||
# stock llama.cpp loads it); publisher KLD: Mean 0.002631, 90 % 0.0421
|
||||
# (kld_results.csv). RCO files are "more experimental" per the README —
|
||||
# the quality floor is the IQ3_S rung, not a promise.
|
||||
#
|
||||
# THE KV CONTRACT IS NOW q8_0 K + q8_0 V — Frank's order for THIS swap
|
||||
# ("k/v at FP8", 2026-10-05), superseding the same-day f16/f16 rule, which
|
||||
# was written for the Ornith 9B fit ("no further quantization anywhere on
|
||||
# this endpoint"). llama.cpp has no true fp8 KV dtype; q8_0 is the 8-bit
|
||||
# class (1.0625 B/elem => 8.5 effective bits) and the only FP8-class lever
|
||||
# this runtime moves. Arithmetic that makes the order load-bearing:
|
||||
# KV rate = 16 attn layers x 4 heads x 256 x 1.0625 B, K and V again
|
||||
# => 34 KiB/token (f16 would be 64 KiB/token — halogen's 27B card quotes
|
||||
# the same f16 number; at f16 the 131,072 slot alone would cost 8.0 GiB).
|
||||
# The architecture is unchanged (same model family): qwen35 hybrid —
|
||||
# block_count 65 (64 real layers + the blk.64 MTP head),
|
||||
# full_attention_interval 4 => SIXTEEN full-attention layers, 24 Q heads
|
||||
# / 4 KV heads, k/v length 256, native window 262144, dense,
|
||||
# nextn_predict_layers=1. The MTP head ships in every main GGUF of this
|
||||
# repo (README); --spec-type draft-mtp loads it against the main model's
|
||||
# own weights.
|
||||
#
|
||||
# 2026-10-06 (Frank): drop the MTP drafter (skip ngram too — on a qwen35
|
||||
# hybrid, ngram speculation disables recurrent-state rollback: fleet lesson
|
||||
# from the frank-large chain measurement), drop ubatch to 512, and take TWO
|
||||
# conversations at 131,072 — "should be a very tight fit", accepted
|
||||
# explicitly, i.e. the 2-3 GiB floor is knowingly violated. Measured at the
|
||||
# 1x131072 MTP boot (2026-10-05): 19,404 used / 4,721 MiB free.
|
||||
# THE KV CONTRACT STAYS q8_0 K + q8_0 V (Frank, 2026-10-05): 34 KiB/token
|
||||
# (16 attn layers x 4 heads x 256 x 1.0625 B, K and V).
|
||||
#
|
||||
# Why TWO slots at 131,072 fits, by arithmetic (pool = 2 x 131,072 =
|
||||
# 262,144; the per-conversation cap stays Frank's 128k):
|
||||
# Why TWO slots + MTP fits now (it did not under IQ3_S):
|
||||
# card 24,576 MiB. Weights 9,963 MiB (10,442,827,392 B; IQ3_S was
|
||||
# 11,561). Projector ~1,290 MiB (the PUBLISHER's OWN mmproj-BF16.gguf,
|
||||
# 931,145,888 B, at the measured ~1.45x card law — no substitution).
|
||||
# Compute buffers at ubatch 512 ~1.3 GiB. CUDA ctx ~500 MiB. DeltaNet
|
||||
# state ~150 MiB/slot x 2. MTP draft ctx back: ~0.4 GiB. Non-KV ~13.9
|
||||
# GiB. Two q8_0 slots = 8,704 MiB. Predicted free ~2.6-3.2 GiB — back
|
||||
# over the old 2-3 GiB floor that the IQ3_S two-slot boot violated
|
||||
# (measured there: 1,397 MiB free). MEASURE nvidia-smi free at boot
|
||||
# (quote the free field) and let the runbook carry the number.
|
||||
#
|
||||
# card 24,576 MiB. Weights 11,561 MiB (the IQ3_S-mtp file; the GSQ
|
||||
# "IQ3_S" tier is a per-tensor ALLOCATION mix — histogram IQ2_XS 144 /
|
||||
# IQ3_XXS 96 / Q6_K 78 / F32 360 / BF16 96 / Q3_K_M 39 ... all stock
|
||||
# types, stock llama.cpp loads it). Projector ~1,350 MiB (927,607,488 B
|
||||
# F16 at the measured ~1.45x card law). Compute buffers at ubatch 512
|
||||
# ~1.3 GiB (inferred from the measured 616 MiB/doubling rung on the 9B).
|
||||
# CUDA ctx ~500 MiB. DeltaNet state ~150 MiB/slot x 2. NO draft context
|
||||
# (the MTP draft ctx cost ~0.3-0.5 GiB and is gone with speculation).
|
||||
# Non-KV ~15.5 GiB. Two q8_0 slots = 8,704 MiB (34 KiB/token x 131,072
|
||||
# each). Predicted free at boot ~0.3-1.3 GiB. Compute buffers allocate
|
||||
# at graph capture during BOOT, so a miss is loud and fast; MEASURE
|
||||
# nvidia-smi free at boot (quote the free field) and let the runbook
|
||||
# carry the number.
|
||||
#
|
||||
# Payload history (for the rollback trail): the Ornith-1.5-9B Q8_0 swap of
|
||||
# earlier on 2026-10-05 (commits 77decb5..380aafa) was committed but never
|
||||
# deployed — the live service was still MiMo-V2.6 Q6_K (2 x 200,704, f16
|
||||
# KV) when Swift cut over. Rollback payloads on disk: Ornith-Q8/ and
|
||||
# MiMo-Q6K/ (one MODEL_FILE + shape edit back; see the runbook).
|
||||
# Payload history (rollback trail): /home/dank/models/Swift15-IQ3S/ (the
|
||||
# IQ3_S-mtp that was live until this swap), Ornith-Q8/, MiMo-Q6K/ all
|
||||
# stay on disk; one MODEL_FILE + shape edit back each.
|
||||
LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server
|
||||
MODEL_FILE=/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf
|
||||
MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf
|
||||
|
||||
# Vision: no Swift repo ships a projector (ukisai's release is
|
||||
# language-model-only by their own README), so the projector comes from the
|
||||
# SHARED BASE — unsloth's mmproj-F16.gguf built from Qwen/Qwen3.8-27B,
|
||||
# stored as mmproj-Qwen3.8-27B-F16.gguf (927,607,488 B, LFS sha256
|
||||
# cbb841a9..., fetched by scripts/download-frank-small.sh). The substitution
|
||||
# is proven, not assumed: Swift-1.5's vision stack (tower + merger) was
|
||||
# hash-compared against the base on 2026-10-05 — 333/333 visual tensor names
|
||||
# and 6/6 sampled tensors byte-identical — so the base projector IS the
|
||||
# model's own projector (Swift's RL/OPD post-training touched text only).
|
||||
# Vision: the publisher SHIPS the projector — mmproj-BF16.gguf
|
||||
# (931,145,888 B) from the same repo, fetched by
|
||||
# scripts/download-frank-small.sh. No substitution needed for this
|
||||
# payload (the unsloth base-tower proof of 2026-10-05 remains the
|
||||
# fallback argument for the IQ3_S rollback payload).
|
||||
# Empty string = text-only on purpose.
|
||||
MMPROJ_FILE=/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf
|
||||
MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf
|
||||
|
||||
# Fleet endpoint: same port and alias as every frank-small before it, so the
|
||||
# harness models.yml id and the homepage probe do not move.
|
||||
@@ -100,13 +84,14 @@ UBATCH_SIZE=512
|
||||
CACHE_TYPE_K=q8_0
|
||||
CACHE_TYPE_V=q8_0
|
||||
|
||||
# Speculative decoding is OFF (Frank, 2026-10-06): no draft-mtp, and
|
||||
# ngram-mod is skipped too — on a qwen35 hybrid ngram disables
|
||||
# recurrent-state rollback, so a rejected draft would corrupt the DeltaNet
|
||||
# state. SPEC_TYPE=none strips BOTH speculative flags in the serve script
|
||||
# (flags move as a set). The -mtp file still carries the head; it is simply
|
||||
# not drafted with. Measured MTP reference (2026-10-05): 44.8 % acceptance
|
||||
# at the recipe temperature => 58.9 t/s, 52.1 % at temp 0.8 => 66.1 t/s.
|
||||
SPEC_TYPE=none
|
||||
# Speculative decoding is ON (Frank, 2026-10-06 evening): draft-mtp at
|
||||
# depth 3 against the head IN the file (the -mtp variant ships the
|
||||
# blk.64.nextn.* tensors). ngram-mod stays skipped — on a qwen35 hybrid
|
||||
# it disables recurrent-state rollback, so a rejected draft would
|
||||
# corrupt the DeltaNet state; MTP needs no such bypass (its draft ctx
|
||||
# re-derives state from the target's own weights). The flags move as a
|
||||
# set: the serve script emits BOTH only when SPEC_TYPE names a drafter.
|
||||
SPEC_TYPE=draft-mtp
|
||||
SPEC_DRAFT_N_MAX=3
|
||||
|
||||
NGL=99
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=llama.cpp Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S-mtp GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, no speculative decoding, mmproj vision) on the RTX 3090 (dankclaw, frank-small)
|
||||
Description=llama.cpp Swift-1.5-Qwen3.8-27B Uncensored RCO IQ3_XXS GGUF (qwen35 hybrid, 2x131072 slots, q8_0 K+V, draft-mtp depth 3, mmproj vision) on the RTX 3090 (dankclaw, frank-small)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
|
||||
@@ -1,24 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF (IQ3_S **-mtp**) as
|
||||
# frank-small on Dankclaw's RTX 3090 via llama.cpp (master build, CUDA
|
||||
# sm_86).
|
||||
# Serve ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF
|
||||
# (RCO-IQ3_XXS, **MTP head in the file**) as frank-small on Dankclaw's
|
||||
# RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
||||
#
|
||||
# 2026-10-06 (Frank): the 128k cap keeps the per-conversation window but
|
||||
# the pool doubles to TWO 131,072 conversations (262144 / 2), speculative
|
||||
# decoding is OFF (no draft-mtp; ngram skipped — on a qwen35 hybrid it
|
||||
# disables recurrent-state rollback), and ubatch drops to 512. Frank
|
||||
# accepted the tight fit explicitly (the 2-3 GiB floor is knowingly
|
||||
# violated; frank-small.env carries the arithmetic). The q8_0 KV contract
|
||||
# (FP8 class) is unchanged; the per-tensor flags stay explicit either way.
|
||||
# 2026-10-06 (Frank): TWO 131,072 conversations (262144 / 2) at the 128k
|
||||
# cap, ubatch 512, and MTP BACK ON (draft-mtp, depth 3 — the payload is
|
||||
# 1.56 GiB lighter than the IQ3_S-mtp it replaced, which is what funds
|
||||
# the draft ctx plus the second slot; ngram stays skipped — on a qwen35
|
||||
# hybrid it disables recurrent-state rollback). The q8_0 KV contract
|
||||
# (FP8 class) is unchanged; the per-tensor flags stay explicit either
|
||||
# way.
|
||||
#
|
||||
# Architecture: qwen35 hybrid — 64 real layers, 16 full-attention + 48
|
||||
# Gated DeltaNet (full_attention_interval 4), native 262144 window, plus
|
||||
# the blk.64 MTP head (nextn_predict_layers=1). KV lives only in the 16
|
||||
# attention layers (34 KiB/token at q8_0); the DeltaNet state is per-slot
|
||||
# fixed size. Vision comes from the projector file (the GGUF carries no
|
||||
# vision tensors; unsloth's mmproj built from the shared Qwen3.8-27B base —
|
||||
# proven byte-identical to Swift's own tower, see download-frank-small.sh),
|
||||
# so --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the
|
||||
# fixed size.
|
||||
# Vision comes from the projector file (the GGUF carries no vision
|
||||
# tensors; the publisher SHIPS mmproj-BF16.gguf in the same repo), so
|
||||
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the
|
||||
# documented text-only opt-out.
|
||||
#
|
||||
# Sampling: llama.cpp takes it per request (the fleet harness sends its
|
||||
@@ -38,7 +38,7 @@ fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) -------------------
|
||||
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
: "${MODEL_FILE:=/home/dank/models/Swift15-IQ3S/Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
||||
: "${MODEL_FILE:=/home/dank/models/Swift15-UncIQ3XXS/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf}"
|
||||
: "${ALIAS:=frank-small}"
|
||||
: "${BIND_ADDR:=0.0.0.0}"
|
||||
: "${LISTEN_PORT:=7771}"
|
||||
@@ -61,15 +61,16 @@ fi
|
||||
# visible in the argv.
|
||||
: "${CACHE_TYPE_K:=q8_0}"
|
||||
: "${CACHE_TYPE_V:=q8_0}"
|
||||
# Speculative decoding is OFF by default (Frank, 2026-10-06): SPEC_TYPE
|
||||
# none strips BOTH speculative flags (flags move as a set). Setting a
|
||||
# drafter here re-enables the pair — draft-mtp needs the head that is IN
|
||||
# the -mtp GGUF.
|
||||
: "${SPEC_TYPE:=none}"
|
||||
# Speculative decoding is ON by default (Frank, 2026-10-06 evening):
|
||||
# draft-mtp against the head that is IN the file. SPEC_TYPE none strips
|
||||
# BOTH speculative flags (flags move as a set) — a no-spec baseline must
|
||||
# not carry half a spec pair.
|
||||
: "${SPEC_TYPE:=draft-mtp}"
|
||||
: "${SPEC_DRAFT_N_MAX:=3}"
|
||||
: "${NGL:=99}"
|
||||
# Vision projector. No colon on purpose: an explicitly empty value means
|
||||
# "text-only, on purpose" and must NOT be replaced by the default path.
|
||||
: "${MMPROJ_FILE=/home/dank/models/Swift15-IQ3S/mmproj-Qwen3.8-27B-F16.gguf}"
|
||||
: "${MMPROJ_FILE=/home/dank/models/Swift15-UncIQ3XXS/mmproj-BF16.gguf}"
|
||||
|
||||
# Fail loudly rather than half-loading: a missing file is a download that
|
||||
# has not finished, and Restart=always would turn it into a crash loop that
|
||||
|
||||
+47
-45
@@ -1,15 +1,15 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-small llama.cpp root unit on dankclaw.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-06 it serves
|
||||
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B; the -mtp file's
|
||||
# head is present but NOT drafted — speculation is off) via a CUDA (sm_86)
|
||||
# build of llama.cpp master: TWO 131,072-token conversations at Frank's
|
||||
# 128k cap (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), and a
|
||||
# vision projector. Frank accepted the tight fit explicitly — the card may
|
||||
# land under the old 2-3 GiB floor; the seam pins the arithmetic, the boot
|
||||
# log carries the measured free. This seam runs the serve script against a
|
||||
# stub engine and asserts on the argv it actually execs, so what is pinned
|
||||
# is the command the operator gets.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-06 (evening) it
|
||||
# serves ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF at
|
||||
# RCO-IQ3_XXS (27B dense qwen3_5 hybrid) via a CUDA (sm_86) build of
|
||||
# llama.cpp master: TWO 131,072-token conversations at Frank's 128k cap
|
||||
# (pool 262144 / parallel 2), q8_0 K + q8_0 V (FP8 class), the
|
||||
# publisher's own BF16 vision projector, and MTP back ON (draft-mtp
|
||||
# depth 3 — the head is in the file; the lighter quant funds the draft
|
||||
# ctx plus the second slot). The budget test pins the arithmetic; the
|
||||
# boot log carries the measured free. This seam runs the serve script
|
||||
# against a stub engine and asserts on the argv it actually execs, so
|
||||
# what is pinned is the command the operator gets.
|
||||
|
||||
load helpers
|
||||
|
||||
@@ -18,9 +18,9 @@ SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
|
||||
TARGET_STUB="Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf"
|
||||
TARGET_STUB="Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-RCO-IQ3_XXS.gguf"
|
||||
# Same basename as the shipped projector: the test env repoints it into $STAGE.
|
||||
MM_STUB="mmproj-Qwen3.8-27B-F16.gguf"
|
||||
MM_STUB="mmproj-BF16.gguf"
|
||||
|
||||
setup() {
|
||||
STAGE="$(mktemp -d)"
|
||||
@@ -83,9 +83,10 @@ argv_value() {
|
||||
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B IQ3_S-mtp GGUF" {
|
||||
@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B Uncensored RCO-IQ3_XXS GGUF" {
|
||||
# The payload path IS the endpoint's identity.
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=/home/dank/models/Swift15-IQ3S/$TARGET_STUB$" "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=/home/dank/models/Swift15-UncIQ3XXS/$TARGET_STUB$" "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||||
@@ -102,7 +103,7 @@ argv_value() {
|
||||
done
|
||||
}
|
||||
|
||||
@test "the shipped env serves the fleet shape: one conversation at the 128k cap" {
|
||||
@test "the pool is exactly the native 262144 window, so each of the two conversations is 131072" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --alias)" = "frank-small" ]
|
||||
@@ -135,27 +136,26 @@ argv_value() {
|
||||
[ "$(argv_value --flash-attn)" = "on" ]
|
||||
}
|
||||
|
||||
@test "the KV budget fits TWO 128k slots (Frank's tight-fit order) and forbids a third" {
|
||||
# Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub
|
||||
# file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64
|
||||
# real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv
|
||||
# heads x 256 dim x 1.0625 B = 17 KiB/token for K and the same for V
|
||||
# => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64).
|
||||
# The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) +
|
||||
# vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured
|
||||
# ~1.45x cost law) + compute buffers at ubatch 512 + CUDA graphs + TWO
|
||||
# DeltaNet states (~150 MiB each) and NO draft context (speculation is
|
||||
# off) => 15 GiB budgeted. 2026-10-06 (Frank): two 131,072 slots are
|
||||
# the order — the budget lands AT the card (tight fit, accepted
|
||||
# explicitly; the old 2-3 GiB floor is knowingly violated) and a THIRD
|
||||
# slot must not fit. Arithmetic, not a boot log: change CTX_SIZE and
|
||||
# this bites before the deploy does; the boot's nvidia-smi free is the
|
||||
# measured truth.
|
||||
@test "the KV budget fits two 131072 slots at q8_0 and forbids a third" {
|
||||
# Rate, from the RCO-IQ3_XXS GGUF header (same qwen35 architecture):
|
||||
# SIXTEEN full-attention layers (block_count 65 = 64 real layers + the
|
||||
# blk.64 MTP head, full_attention_interval 4) x 4 kv heads x 256 dim x
|
||||
# 1.0625 B = 17 KiB/token for K and the same for V => 34 KiB/token at
|
||||
# q8_0 (budgeted 37 with slack; f16 would be 64). The non-KV half:
|
||||
# RCO-IQ3_XXS weights 10,148 MiB (10,442,827,392 B) — 1.56 GiB LIGHTER
|
||||
# than the IQ3_S-mtp predecessor — + vision tower ~1.4 GiB (931,145,888
|
||||
# B BF16 mmproj at the measured ~1.45x cost law) + compute buffers at
|
||||
# ubatch 512 + CUDA graphs + TWO DeltaNet states (~150 MiB each) + the
|
||||
# MTP draft ctx => 14 GiB budgeted. Two 131,072 slots are the
|
||||
# 2026-10-06 shape (the lighter quant funds the draft ctx plus the
|
||||
# second slot); a THIRD slot must not fit. Arithmetic, not a boot log:
|
||||
# change CTX_SIZE and this bites before the deploy does; the boot's
|
||||
# nvidia-smi free is the measured truth.
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv_gib=$((ctx * 37 / 1048576))
|
||||
third_slot_kv=$(( (ctx + 131072) * 37 / 1048576 ))
|
||||
[ $((kv_gib + 15)) -le 24 ]
|
||||
[ $((third_slot_kv + 15)) -gt 24 ]
|
||||
[ $((kv_gib + 14)) -le 24 ]
|
||||
[ $((third_slot_kv + 14)) -gt 24 ]
|
||||
}
|
||||
|
||||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||||
@@ -185,19 +185,21 @@ argv_value() {
|
||||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||||
}
|
||||
|
||||
@test "speculation is off: no half spec pair reaches the argv" {
|
||||
# Frank, 2026-10-06: drop MTP, skip ngram-mod (on a qwen35 hybrid
|
||||
# ngram disables recurrent-state rollback — a rejected draft corrupts
|
||||
# the DeltaNet state). The flags move as a set, so the no-spec contract
|
||||
# is ABSENCE of both in the argv the supervisor runs, and the env must
|
||||
# carry the same contract. The -mtp file's head stays; it is not
|
||||
# drafted with. Reference numbers stay in frank-small.env for the day
|
||||
# MTP returns (44.8 % acceptance at the recipe temp => 58.9 t/s).
|
||||
@test "speculation is on: the full draft-mtp pair reaches the argv" {
|
||||
# Frank, 2026-10-06 evening: MTP back on. The head is IN the file, so
|
||||
# draft-mtp drafts against the main model's weights (no second model
|
||||
# copy). ngram-mod stays skipped: on a qwen35 hybrid it disables
|
||||
# recurrent-state rollback — a rejected draft corrupts the DeltaNet
|
||||
# state. The flags move as a set: the pair is present together or
|
||||
# absent together — assert both in the argv, and the env carries the
|
||||
# same contract. The serve script's defaults ship the pair even for an
|
||||
# env that names no drafter.
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
! grep -qxF -- '--spec-type' <<<"$output"
|
||||
! grep -qxF -- '--spec-draft-n-max' <<<"$output"
|
||||
grep -qE '^SPEC_TYPE=none$' "$ENV_FILE"
|
||||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||||
[ "$(argv_value --spec-draft-n-max)" = "3" ]
|
||||
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
|
||||
grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
||||
|
||||
Reference in New Issue
Block a user