dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-small: SGLang MiMo-V2.6-Distill-Qwen-9B INT8 (qwen3_5 hybrid, RTX 3090)

Browse Source 
Replaces llama.cpp Gemma4-12B-QAT (payload + binary retained as rollback).
SGLang in a self-contained python3.12 venv with an extracted CUDA 13.0
nvcc toolchain for the sgl-kernel/flashinfer JIT.

Shape: fp8 KV, pinned 606,208-token pool / 4 x 151,552 slots, bf16 mamba
states pinned at 32 slots (the engine autofills the ssm pool to a fixed
~5.1 GB regardless of dtype), eager prefill (no headroom for graph
capture), no spec decode (the checkpoint ships no mtp.* tensors despite
mtp_num_hidden_layers=1 — NEXTN drafts with an uninitialised 4.24 GiB
ghost at 0% acceptance).

Verified: 83.9 tok/s decode, 170k single prompt, 4 concurrent 54k
requests, vision (in-checkpoint tower), reasoning_content field.
This commit is contained in:
Frank committed 2026-10-01 06:55:48 -07:00
1 parent 7da504a651
commit 0c5e054fb8
12 files changed
+1005 -328
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/frank_small_unit.bats
+74 -61
View File
@@ -1,25 +1,30 @@
#!/usr/bin/env bats
# Seam: the frank-small mainline-llama.cpp root unit on dankclaw.
# frank-small is the RTX 3090 endpoint created on 2026-09-16, when frank-large
# went back to the halogen engine on the iGPU. This seam pins what the flags
# alone do not say: boot persistence for a NATIVE runtime (no generated quadlet
# is involved), the repo-relative paths the unit executes, the separate client
# port, the MTP contract (the head is a separate sidecar file loaded as the
# draft model, chained behind ngram-mod), the vision handoff from frank-large,
# and the KV budget: q8_0 quantized KV whose context --fit sizes to the card.
# Seam: the frank-small SGLang root unit on dankclaw.
# frank-small is the RTX 3090 endpoint; since 2026-10-01 it serves
# letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound (qwen3_5 hybrid,
# INT8 W8A16 AutoRound, in-checkpoint vision) via SGLang in a
# python3.12 venv. This seam pins what the flags alone do not say: boot
# persistence for a NATIVE runtime (no generated quadlet is involved), the
# repo-relative paths the unit executes, the separate client port, the
# no-spec-decode gate (the checkpoint ships NO mtp.* tensors, so a NEXTN
# draft would be an uninitialised ghost model), the 8-shard payload gate,
# and the KV budget: fp8 KV at 0.98 utilization whose 4-slot grid must fit
# the card's 24 GiB.
load helpers
UNIT="$REPO_ROOT/scripts/frank-small.service"
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
DOWNLOAD="$REPO_ROOT/scripts/download-frank-small.sh"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
@test "frank-small unit, serve script and env exist in scripts/" {
@test "frank-small unit, serve script, env and download script exist in scripts/" {
[ -f "$UNIT" ]
[ -f "$SERVE" ]
[ -f "$ENV_FILE" ]
[ -f "$DOWNLOAD" ]
}
@test "frank-small is boot-persistent, self-healing and unprivileged" {
@@ -35,64 +40,67 @@ ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
}
@test "frank-small offloads every layer to the 3090 with flash attention" {
grep -q 'llama.cpp-mainline/build-cuda/bin/llama-server' "$ENV_FILE"
grep -q '^NGL=999$' "$ENV_FILE"
grep -q -- '--gpu-layers "$NGL"' "$SERVE"
grep -q '^FLASH_ATTN=on$' "$ENV_FILE"
grep -q -- '--flash-attn "$FLASH_ATTN"' "$SERVE"
# Prometheus scrapes this endpoint; without --metrics there is no /metrics.
grep -q -- '--metrics' "$SERVE"
@test "frank-small serves the MiMo-V2.6 checkpoint from the self-contained SGLang venv" {
grep -q '^SGLANG_BIN=/home/dank/GitPackages/sglang-cu128/bin/sglang$' "$ENV_FILE"
grep -q '^MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound$' "$ENV_FILE"
grep -q -- '--model-path "$MODEL_DIR"' "$SERVE"
# The AutoRound checkpoint declares its quant method; SGLang must load it
# as auto-round, not re-quantize online.
grep -q '^QUANTIZATION=auto-round$' "$ENV_FILE"
grep -q -- '--quantization "$QUANTIZATION"' "$SERVE"
}
@test "frank-small caps its CPU RAM cache below the build default" {
# Without the flag the 8192 MiB default returns and the env cap is a
# silent no-op; this memory-tight host needs the cap to hold.
grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE"
}
@test "frank-small drafts with the MTP sidecar chained behind ngram-mod" {
grep -q '^SPEC_TYPE=draft-mtp,ngram-mod$' "$ENV_FILE"
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
# The Gemma4 MTP head is a separate assistant file (gemma4-assistant, 4
# nextn layers); it loads as --model-draft and shares the target's last
# layers' KV. ngram-mod is draftless, so the chain is legal.
grep -q '^DRAFT_FILE=mtp-gemma-4-12B-it\.gguf$' "$ENV_FILE"
grep -q -- '--model-draft "$MODEL_DIR/$DRAFT_FILE"' "$SERVE"
@test "frank-small runs WITHOUT speculative decoding (no MTP weights shipped)" {
# The config declares mtp_num_hidden_layers=1 but the safetensors index
# carries zero mtp.* tensors (base and INT8 conversion alike, verified
# 2026-10-01). SGLang would draft with an uninitialised 4.24 GiB model
# at 0% acceptance; the serve script must not pass any speculative flag.
! grep -q -- '--speculative-' "$SERVE"
! grep -q '^SPEC_' "$ENV_FILE"
}
@test "frank-small carries the fleet's image path, and only it" {
grep -q '^MMPROJ_PATH=.*mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16\.gguf$' "$ENV_FILE"
grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE"
# The vision tower is baked into the safetensors (qwen3_5_vision): no
# separate projector file, and the serve script must not look for one.
! grep -q '^MMPROJ_PATH=' "$ENV_FILE"
! grep -q -- '--mmproj' "$SERVE"
! grep -q '^ENABLE_VISION=1$' "$ENV_LARGE"
}
@test "frank-small's slot grid is full-window and fits the 24 GiB card" {
local parallel cache_k cache_v ctx fixed per_slot
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
[ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$ctx" ]
# Every slot must be the model's FULL trained window (262144), not a
# divided one -- CTX_SIZE is the total pool and --parallel divides it.
[ $((ctx / parallel)) -eq 262144 ]
[ $((ctx % parallel)) -eq 0 ]
[ "$cache_k" = "q8_0" ] && [ "$cache_v" = "q8_0" ]
# Fit budget from the measured boot arithmetic on the 3090 (24576
# MiB): one full-context slot idles at 11,996 MiB, four at 22,346,
# so each slot costs (22346-11996)/3 = 3450 MiB and the fixed part
# (weights, projector, sidecar draft, compute buffers, CUDA context)
# is 11996-3450 = 8546 MiB. If the config asks for more slots than
# the card holds, this inequality fails before deploy instead of the
# KV allocation failing at boot.
fixed=8546
per_slot=3450
[ $((fixed + parallel * per_slot)) -le 24576 ]
# Quantized KV on CUDA needs the GGML_CUDA_FA_ALL_QUANTS build option,
# without which prefill collapses about 12x (see the CUDA lesson in
# AGENTS.md). The build-cuda tree must carry it.
grep -q '^FLASH_ATTN=on$' "$ENV_FILE"
@test "frank-small's slot grid is fp8 KV and fits the measured card budget" {
local ctx running kv ssmdtype mambasize required
ctx="$(sed -nE 's/^CONTEXT_LENGTH=([0-9]+)$/\1/p' "$ENV_FILE")"
running="$(sed -nE 's/^MAX_RUNNING=([0-9]+)$/\1/p' "$ENV_FILE")"
kv="$(sed -nE 's/^KV_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
ssmdtype="$(sed -nE 's/^MAMBA_SSM_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
mambasize="$(sed -nE 's/^MAX_MAMBA_CACHE_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
[ -n "$ctx" ] && [ -n "$running" ] && [ -n "$kv" ]
# fp8 KV is the only dtype whose 16 KiB/token makes four slots fit;
# 64 tokens per MiB of pool.
case "$kv" in fp8*) ;; *) return 1 ;; esac
# The DeltaNet ssm state pool is the hidden fit cost: the engine
# autofills mamba slots to spend a fixed ~5.1 GB budget, so dtype
# alone does not shrink it — the SLOT COUNT must be pinned (bf16,
# bounded). These two gates are what make the measured pool below
# exist; change either and re-measure the pool at boot.
[ "$ssmdtype" = "bfloat16" ]
[ -n "$mambasize" ] && [ "$mambasize" -ge "$running" ] && [ "$mambasize" -le 64 ]
grep -q -- '--mamba-ssm-dtype "$MAMBA_SSM_DTYPE"' "$SERVE"
grep -q -- '--max-mamba-cache-size "$MAX_MAMBA_CACHE_SIZE"' "$SERVE"
# The slot window is inside the model's native 262144 window.
[ "$ctx" -le 262144 ]
# The grid must equal the pinned pool. The engine's autofit picks a
# bigger pool (687,388 tokens) but leaves ~0.53 GB — Triton's
# deferred kernel loads take most of it and eager-prefill activations
# then OOM on the first long request. 606,208 (4 x 151,552) is the
# verified ceiling: 170k single request and 4 concurrent 54k requests
# run against it (2026-10-01). A weights or mamba-flag change moves
# this number; re-measure before trusting the grid.
pinned_pool=606208
required=$((running * ctx))
[ "$required" -le "$pinned_pool" ]
grep -q -- '--max-total-tokens "$MAX_TOTAL_TOKENS"' "$SERVE"
grep -q -- '--disable-prefill-cuda-graph' "$SERVE"
}
@test "frank-small runs on its own port beside frank-large's" {
@@ -106,16 +114,21 @@ ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
! grep -q '^PORT=7771$' "$ENV_LARGE"
}
@test "the deploy handler installs the root unit and refuses a missing model" {
@test "the deploy handler installs the root unit and refuses a missing payload" {
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY"
grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY"
grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY"
grep -q 'Gemma4-12B-QAT' "$REMOTE_DEPLOY"
grep -q 'MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound' "$REMOTE_DEPLOY"
grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY"
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
}
@test "frank-small's serve script refuses to start on a half-finished download" {
# The payload is 8 safetensors shards; the serve script counts them and
# the download script pins each one's LFS sha256.
grep -q 'frank-small: missing' "$SERVE"
grep -q 'expected 8 safetensors shards' "$SERVE"
grep -q 'model-00001-of-00008.safetensors' "$DOWNLOAD"
grep -q 'model-00008-of-00008.safetensors' "$DOWNLOAD"
}