dank/serverAppConfigs
1
0 Fork 0
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-large: roll back to gufo Qwen3.8-27B Q4_K_XL, one session x 262144 f16 KV

Browse Source 
frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144,
asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
This commit is contained in:
Frank committed 2026-10-01 22:40:59 -07:00
1 parent e452a1afbf
commit 8551bb006b
11 files changed
+729 -816
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/frank_small_unit.bats
+120 -94
View File
@@ -1,30 +1,65 @@
#!/usr/bin/env bats
# Seam: the frank-small SGLang root unit on dankclaw.
# frank-small is the RTX 3090 endpoint; since 2026-10-01 it serves
# letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound (qwen3_5 hybrid,
# INT8 W8A16 AutoRound, in-checkpoint vision) via SGLang in a
# python3.12 venv. This seam pins what the flags alone do not say: boot
# persistence for a NATIVE runtime (no generated quadlet is involved), the
# repo-relative paths the unit executes, the separate client port, the
# no-spec-decode gate (the checkpoint ships NO mtp.* tensors, so a NEXTN
# draft would be an uninitialised ghost model), the 8-shard payload gate,
# and the KV budget: fp8 KV at 0.98 utilization whose 4-slot grid must fit
# the card's 24 GiB.
# Seam: the frank-small llama.cpp root unit on dankclaw.
# frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of
# llama.cpp master, chosen for the ASYMMETRIC KV Frank asked for: K f16
# (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. This
# seam runs the serve script against a stub engine and asserts on the argv
# it actually execs, so what is pinned is the command the operator gets.
load helpers
UNIT="$REPO_ROOT/scripts/frank-small.service"
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
DOWNLOAD="$REPO_ROOT/scripts/download-frank-small.sh"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf"
@test "frank-small unit, serve script, env and download script exist in scripts/" {
setup() {
STAGE="$(mktemp -d)"
RUNTIME_DIR="$STAGE/llama"
MODEL_DIR="$STAGE/models"
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR"
cat > "$RUNTIME_DIR/bin/llama-server" <<'STUB'
#!/usr/bin/env bash
printf '%s\n' "$@"
STUB
chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server"
touch "$MODEL_DIR/$TARGET_STUB"
}
teardown() {
rm -rf "$STAGE"
}
# serve_argv [env-file] — run the serve script and capture the engine's
# argv, one token per line, in $output.
serve_argv() {
local envf="${1:-$STAGE/test.env}"
cat > "$envf" <<EOF
LLAMA_BIN=$RUNTIME_DIR/bin/llama-server
MODEL_FILE=$MODEL_DIR/$TARGET_STUB
ALIAS=frank-small
BIND_ADDR=0.0.0.0
LISTEN_PORT=7771
CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")
CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")
CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
EOF
run env FRANK_SMALL_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
}
argv_value() {
grep -A1 -xF -- "$1" <<<"$output" | tail -1
}
@test "frank-small unit, serve script and env exist in scripts/" {
[ -f "$UNIT" ]
[ -f "$SERVE" ]
[ -f "$ENV_FILE" ]
[ -f "$DOWNLOAD" ]
}
@test "frank-small is boot-persistent, self-healing and unprivileged" {
@@ -33,103 +68,94 @@ ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
grep -q '^RestartSec=' "$UNIT"
grep -q '^User=dank$' "$UNIT"
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
}
@test "frank-small runs the repo's own serve script with the repo's env" {
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh$' "$UNIT"
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
}
@test "frank-small serves the MiMo-V2.6 checkpoint from the self-contained SGLang venv" {
grep -q '^SGLANG_BIN=/home/dank/GitPackages/sglang-cu128/bin/sglang$' "$ENV_FILE"
grep -q '^MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound$' "$ENV_FILE"
grep -q -- '--model-path "$MODEL_DIR"' "$SERVE"
# The AutoRound checkpoint declares its quant method; SGLang must load it
# as auto-round, not re-quantize online.
grep -q '^QUANTIZATION=auto-round$' "$ENV_FILE"
grep -q -- '--quantization "$QUANTIZATION"' "$SERVE"
@test "the shipped env points at the llama.cpp runtime and the Q6_K GGUF" {
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q6K/$TARGET_STUB$" "$ENV_FILE"
}
@test "frank-small runs WITHOUT speculative decoding (no MTP weights shipped)" {
# The config declares mtp_num_hidden_layers=1 but the safetensors index
# carries zero mtp.* tensors (base and INT8 conversion alike, verified
# 2026-10-01). SGLang would draft with an uninitialised 4.24 GiB model
# at 0% acceptance; the serve script must not pass any speculative flag.
! grep -q -- '--speculative-' "$SERVE"
! grep -q '^SPEC_' "$ENV_FILE"
@test "the shipped env serves the fleet shape: two full native windows" {
serve_argv
[ "$status" -eq 0 ]
[ "$(argv_value --alias)" = "frank-small" ]
[ "$(argv_value --port)" = "7771" ]
[ "$(argv_value --host)" = "0.0.0.0" ]
[ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ]
[ "$(argv_value --parallel)" -eq 2 ]
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
# llama.cpp divides --ctx-size across slots: two slots means the pool
# must be 2 x the window, and the window never exceeds the native one.
ctx="$(argv_value --ctx-size)"
par="$(argv_value --parallel)"
[ $((ctx / par)) -eq 262144 ]
}
@test "frank-small carries the fleet's image path, and only it" {
# The vision tower is baked into the safetensors (qwen3_5_vision): no
# separate projector file, and the serve script must not look for one.
! grep -q '^MMPROJ_PATH=' "$ENV_FILE"
@test "the KV contract is asymmetric: unquantized K, q8_0 V, flash-attn on" {
serve_argv
[ "$status" -eq 0 ]
[ "$(argv_value --cache-type-k)" = "f16" ]
[ "$(argv_value --cache-type-v)" = "q8_0" ]
# q8_0 on V only exists on the flash-attention path; without this flag
# pair the KV contract in the env file is a lie the engine never sees.
[ "$(argv_value --flash-attn)" = "on" ]
}
@test "the KV budget fits the 3090 with room for checkpoints" {
# K f16 (16 KiB/token) + V q8_0 (~8.5, budget 25 with slack) over the
# whole pool, plus 7.8 GiB weights and ~2 GiB of compute buffers and
# CUDA graphs, must stay under the card's 24 GiB. This is arithmetic,
# not a boot log: change CTX_SIZE and it bites before deploy does.
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
kv_gib=$((ctx * 25 / 1048576))
[ $((kv_gib + 8 + 2)) -le 24 ]
}
@test "no speculative draft ships by default" {
# The qwen35 GGUF has no MTP tensors, and the cross-model DFlash draft
# is a measured decision the harness makes later, not a default.
! grep -q -- '--model-draft' "$SERVE"
! grep -q -- '--spec-type' "$SERVE"
! grep -q '^DRAFT' "$ENV_FILE"
}
@test "frank-small stays text-only on the GGUF path and speaks the fleet reasoning field" {
# The GGUF carries no vision projector; the image path left with the
# SGLang checkpoint. models.yml declares frank-small text-only since
# the 2026-10-01 night swap.
! grep -q -- '--mmproj' "$SERVE"
! grep -q '^ENABLE_VISION=1$' "$ENV_LARGE"
# deepseek format = thinking lands in reasoning_content, the field the
# harness has always read on this endpoint (SGLang parity).
serve_argv
[ "$(argv_value --reasoning-format)" = "deepseek" ]
argv_value --jinja >/dev/null
grep -qxF -- '--jinja' <<<"$output"
}
@test "frank-small's slot grid is fp8 KV and fits the measured card budget" {
local ctx running kv ssmdtype mambasize required
ctx="$(sed -nE 's/^CONTEXT_LENGTH=([0-9]+)$/\1/p' "$ENV_FILE")"
running="$(sed -nE 's/^MAX_RUNNING=([0-9]+)$/\1/p' "$ENV_FILE")"
kv="$(sed -nE 's/^KV_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
ssmdtype="$(sed -nE 's/^MAMBA_SSM_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
mambasize="$(sed -nE 's/^MAX_MAMBA_CACHE_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
[ -n "$ctx" ] && [ -n "$running" ] && [ -n "$kv" ]
# fp8 KV is the only dtype whose 16 KiB/token makes four slots fit;
# 64 tokens per MiB of pool.
case "$kv" in fp8*) ;; *) return 1 ;; esac
# The DeltaNet ssm state pool is the hidden fit cost: the engine
# autofills mamba slots to spend a fixed ~5.1 GB budget, so dtype
# alone does not shrink it — the SLOT COUNT must be pinned (bf16,
# bounded). These two gates are what make the measured pool below
# exist; change either and re-measure the pool at boot.
[ "$ssmdtype" = "bfloat16" ]
[ -n "$mambasize" ] && [ "$mambasize" -ge "$running" ] && [ "$mambasize" -le 64 ]
grep -q -- '--mamba-ssm-dtype "$MAMBA_SSM_DTYPE"' "$SERVE"
grep -q -- '--max-mamba-cache-size "$MAX_MAMBA_CACHE_SIZE"' "$SERVE"
# The slot window is inside the model's native 262144 window.
[ "$ctx" -le 262144 ]
# The grid must equal the pinned pool. The engine's autofit picks a
# bigger pool (687,388 tokens) but leaves ~0.53 GB — Triton's
# deferred kernel loads take most of it and eager-prefill activations
# then OOM on the first long request. 606,208 (4 x 151,552) is the
# verified ceiling: 170k single request and 4 concurrent 54k requests
# run against it (2026-10-01). A weights or mamba-flag change moves
# this number; re-measure before trusting the grid.
pinned_pool=606208
required=$((running * ctx))
[ "$required" -le "$pinned_pool" ]
grep -q -- '--max-total-tokens "$MAX_TOTAL_TOKENS"' "$SERVE"
grep -q -- '--disable-prefill-cuda-graph' "$SERVE"
@test "the serve script refuses a missing runtime or payload" {
rm -f "$RUNTIME_DIR/bin/llama-server"
serve_argv
[ "$status" -ne 0 ]
[[ "$output" == *"frank-small: missing"* ]]
}
@test "frank-small runs on its own port beside frank-large's" {
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
grep -q '^ALIAS=frank-small$' "$ENV_FILE"
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
# The two endpoints cannot share a port: frank-large publishes
# 7770:7770 through podman, this one binds 7771 natively. (The
# frank-large vLLM cutover renamed PORT to FRANK_LARGE_PUBLISH.)
grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_LARGE"
! grep -q '^FRANK_LARGE_PUBLISH=7771' "$ENV_LARGE"
}
@test "the deploy handler installs the root unit and refuses a missing payload" {
@test "the deploy handler gates on the binary and the GGUF, then MainPID-hardens health" {
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
grep -q 'LLM_SMALL_RUNTIME' "$REMOTE_DEPLOY"
grep -q 'LLM_SMALL_MODEL_FILE' "$REMOTE_DEPLOY"
grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY"
grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY"
grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY"
grep -q 'MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound' "$REMOTE_DEPLOY"
grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY"
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
grep -q 'frank-small.service has no MainPID' "$REMOTE_DEPLOY"
}
@test "frank-small's serve script refuses to start on a half-finished download" {
# The payload is 8 safetensors shards; the serve script counts them and
# the download script pins each one's LFS sha256.
grep -q 'frank-small: missing' "$SERVE"
grep -q 'expected 8 safetensors shards' "$SERVE"
grep -q 'model-00001-of-00008.safetensors' "$DOWNLOAD"
grep -q 'model-00008-of-00008.safetensors' "$DOWNLOAD"
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
# frank-large's gufo sweep must not take down frank-small, whose
# process is llama-server on the SAME host since this swap.
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
local large
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
! grep -q 'pkill -x llama-server' <<<"$large"
}