Frank
baf33d016f
The parallel session's uncommitted swap, committed at Frank's direction (commit whatever's in the working tree): - llama.cpp master on the RTX 3090 now serves ukisai's Swift-1.5-Qwen3.8-27B GSQ-RCO IQ3_S WITH the in-file MTP head, ONE 131,072-token slot at Frank's 128k cap, q8_0 K + q8_0 V, vision through the base-model projector; MiMo-V2.6 Q6_K (2x200704 f16 KV) retired to rollback target. - env/serve/download + seam twins updated together (deployer payload gates rode the previous commit's deploy-remote.sh). - docs: new swift15 runbook; MiMo Q6K runbook retired; AGENTS.md topology and both vault indexes follow (root index ACTIVE/RETIRED flip, test 135 seam green). - comfyui: two qwen-image t2i workflow drops under dankclaw/comfyui/ workflows/ (12 KiB).
257 lines
11 KiB
Bash
Executable File
257 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-small llama.cpp root unit on dankclaw.
|
|
# frank-small is the RTX 3090 endpoint; since 2026-10-05 evening it serves
|
|
# ukisai/Swift-1.5-Qwen3.8-27B-GSQ-RCO-GGUF:IQ3_S-mtp (27B, MTP head in the
|
|
# file) via a CUDA (sm_86) build of llama.cpp master: ONE 131,072-token
|
|
# conversation at Frank's 128k cap, q8_0 K + q8_0 V (FP8 class), the
|
|
# in-file MTP head drafting 3, and a vision projector. The card must keep
|
|
# 2-3 GiB free. This seam runs the serve script against a stub engine and
|
|
# asserts on the argv it actually execs, so what is pinned is the
|
|
# command the operator gets.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-small.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
|
|
TARGET_STUB="Swift-1.5-Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf"
|
|
# Same basename as the shipped projector: the test env repoints it into $STAGE.
|
|
MM_STUB="mmproj-Qwen3.8-27B-F16.gguf"
|
|
|
|
setup() {
|
|
STAGE="$(mktemp -d)"
|
|
RUNTIME_DIR="$STAGE/llama"
|
|
MODEL_DIR="$STAGE/models"
|
|
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR"
|
|
cat > "$RUNTIME_DIR/bin/llama-server" <<'STUB'
|
|
#!/usr/bin/env bash
|
|
printf '%s\n' "$@"
|
|
STUB
|
|
chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server"
|
|
touch "$MODEL_DIR/$TARGET_STUB" "$MODEL_DIR/$MM_STUB"
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$STAGE"
|
|
}
|
|
|
|
# serve_argv [env-file] — run the serve script and capture the engine's
|
|
# argv, one token per line, in $output.
|
|
serve_argv() {
|
|
local envf="${1:-$STAGE/test.env}"
|
|
local mmproj
|
|
mmproj="$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")"
|
|
cat > "$envf" <<EOF
|
|
LLAMA_BIN=$RUNTIME_DIR/bin/llama-server
|
|
MODEL_FILE=$MODEL_DIR/$TARGET_STUB
|
|
MMPROJ_FILE=$MODEL_DIR/${mmproj##*/}
|
|
ALIAS=frank-small
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7771
|
|
CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
|
PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")
|
|
BATCH_SIZE=$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
|
UBATCH_SIZE=$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
|
CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")
|
|
CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
|
|
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
|
EOF
|
|
run env FRANK_SMALL_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
|
|
}
|
|
|
|
argv_value() {
|
|
grep -A1 -xF -- "$1" <<<"$output" | tail -1
|
|
}
|
|
|
|
@test "frank-small unit, serve script and env exist in scripts/" {
|
|
[ -f "$UNIT" ]
|
|
[ -f "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
}
|
|
|
|
@test "frank-small is boot-persistent, self-healing and unprivileged" {
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh$' "$UNIT"
|
|
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
|
|
}
|
|
|
|
@test "the shipped env points at the llama.cpp runtime and the Swift-1.5-27B IQ3_S-mtp GGUF" {
|
|
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
|
grep -qE "^MODEL_FILE=/home/dank/models/Swift15-IQ3S/$TARGET_STUB$" "$ENV_FILE"
|
|
}
|
|
|
|
@test "the deployer gates on exactly the payload paths the env file ships" {
|
|
# Anti-drift: the deploy handler checks files by its own defaults. If
|
|
# those ever differ from what the unit actually loads, a green deploy
|
|
# would be gating on a file nobody reads — and a missing projector would
|
|
# ship a blind endpoint.
|
|
local env_v dep_v path pair
|
|
for pair in "MODEL_FILE LLM_SMALL_MODEL_FILE" "MMPROJ_FILE LLM_SMALL_MMPROJ"; do
|
|
read -r env_v dep_v <<<"$pair"
|
|
path="$(sed -nE "s/^${env_v}=(.+)\$/\1/p" "$ENV_FILE" | tail -1)"
|
|
[ -n "$path" ]
|
|
grep -qF -- "${dep_v}=\"\${${dep_v}:-${path}}\"" "$REMOTE_DEPLOY"
|
|
done
|
|
}
|
|
|
|
@test "the shipped env serves the fleet shape: one conversation at the 128k cap" {
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
[ "$(argv_value --alias)" = "frank-small" ]
|
|
[ "$(argv_value --port)" = "7771" ]
|
|
[ "$(argv_value --host)" = "0.0.0.0" ]
|
|
[ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ]
|
|
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
|
# llama.cpp divides --ctx-size across slots and a mamba hybrid cannot
|
|
# share one cache, so the observable contract is the per-slot window.
|
|
# 2026-10-05 evening (Frank): the 27B Swift-1.5 swap caps each
|
|
# conversation at 131,072, and ONE slot is the maximum — a second
|
|
# 131,072 slot does not fit the 2-3 GiB floor (KV budget test below).
|
|
[ "$(argv_value --parallel)" -eq 1 ]
|
|
ctx="$(argv_value --ctx-size)"
|
|
par="$(argv_value --parallel)"
|
|
[ $((ctx % par)) -eq 0 ]
|
|
[ $((ctx / par)) -eq 131072 ]
|
|
}
|
|
|
|
@test "the KV contract is q8_0 both halves (FP8 class), flash-attn on" {
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
# Frank, 2026-10-05 evening: "k/v at FP8" for the Swift-1.5 swap —
|
|
# llama.cpp's FP8 class is q8_0 (1.0625 B/elem => 8.5 effective bits;
|
|
# there is no true fp8 KV dtype). This SUPERSEDES the same-day f16/f16
|
|
# rule written for the Ornith 9B fit. The flag pair is what makes this
|
|
# observable — a build without the FA path silently falls back, so both
|
|
# must appear in the argv the supervisor runs.
|
|
[ "$(argv_value --cache-type-k)" = "q8_0" ]
|
|
[ "$(argv_value --cache-type-v)" = "q8_0" ]
|
|
[ "$(argv_value --flash-attn)" = "on" ]
|
|
}
|
|
|
|
@test "the KV budget fits the 2-3 GiB floor and forbids a second 128k slot" {
|
|
# Rate, from the Swift-1.5 IQ3_S-mtp GGUF header (read from the Hub
|
|
# file, 2026-10-05): SIXTEEN full-attention layers (block_count 65 = 64
|
|
# real layers + the blk.64 MTP head, full_attention_interval 4) x 4 kv
|
|
# heads x 256 dim x 1.0625 B = 17 KiB/token for K and the same for V
|
|
# => 34 KiB/token at q8_0 (budgeted 37 with slack; f16 would be 64).
|
|
# The non-KV half: IQ3_S-mtp weights 11,561 MiB (12,120,016,896 B) +
|
|
# vision tower ~1.35 GiB (927,607,488 B F16 mmproj at the measured
|
|
# ~1.45x cost law) + DeltaNet states + compute buffers at ubatch 1024 +
|
|
# CUDA graphs + the MTP draft context => 17 GiB budgeted. Two asserts,
|
|
# both about the output the operator sees: the pool must still fit with
|
|
# 2 GiB spare, and a SECOND 131,072 slot must not fit — Frank's "as
|
|
# many 128k contexts as fit" has the answer 1 on this card, and the
|
|
# pool itself is capped by the 128k law, not by the card.
|
|
# Arithmetic, not a boot log: change CTX_SIZE and this bites before the
|
|
# deploy does.
|
|
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
kv_gib=$((ctx * 37 / 1048576))
|
|
[ $((kv_gib + 17 + 2)) -le 24 ]
|
|
[ $((2 * kv_gib + 17)) -gt 24 ]
|
|
}
|
|
|
|
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
|
# This endpoint ran on llama.cpp's own defaults until 2026-10-03, so the
|
|
# contract is: whatever rung the repo carries is the rung the engine is
|
|
# exec'd with — the number a benchmark attributes a prefill rate to must be
|
|
# a repo value, never an engine default nobody wrote down. Asserted against
|
|
# the env rather than a literal so a retune moves one file, not two.
|
|
local env_batch env_ubatch
|
|
env_batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
env_ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
[ "$(argv_value --batch-size)" = "$env_batch" ]
|
|
[ "$(argv_value --ubatch-size)" = "$env_ubatch" ]
|
|
# llama.cpp refuses ubatch > batch, so a retune that inverts the pair has
|
|
# to fail here instead of at boot on a live endpoint.
|
|
[ "$env_batch" -ge "$env_ubatch" ]
|
|
|
|
# And an env that names neither flag still ships an explicit pair: the
|
|
# serve script's fallbacks are written-down values, not blanks the engine
|
|
# fills in silently.
|
|
sed -i '/^BATCH_SIZE=/d;/^UBATCH_SIZE=/d' "$STAGE/test.env"
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
argv_value --batch-size | grep -qE '^[0-9]+$'
|
|
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
|
}
|
|
|
|
@test "the in-file MTP head ships as draft-mtp at depth 3" {
|
|
# The -mtp GGUF carries the head (blk.64.nextn.*, nextn_predict_layers
|
|
# =1): draft-mtp loads it against the main model's own weights — no
|
|
# --model-draft and no second draft KV pool. Speculative flags move as
|
|
# a set; both must appear in the argv the supervisor runs, and the env
|
|
# must carry the same contract the argv executes.
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
|
[ "$(argv_value --spec-draft-n-max)" = "3" ]
|
|
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
|
|
grep -qE '^SPEC_DRAFT_N_MAX=3$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
|
# The Swift GGUF carries no vision tensors, so the projector argument IS
|
|
# the vision capability: no flag, no eyes.
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
[ "$(argv_value --mmproj)" = "$MODEL_DIR/$MM_STUB" ]
|
|
|
|
# The documented opt-out must reach the engine as ABSENCE of the flag,
|
|
# not as an argument pointing at an empty string.
|
|
sed -i 's|^MMPROJ_FILE=.*|MMPROJ_FILE=|' "$STAGE/test.env"
|
|
run env FRANK_SMALL_ENV_FILE="$STAGE/test.env" nvidia-modprobe=true bash "$SERVE"
|
|
[ "$status" -eq 0 ]
|
|
! grep -qxF -- '--mmproj' <<<"$output"
|
|
}
|
|
|
|
@test "the fleet reasoning contract survives the reslice to two wide contexts" {
|
|
serve_argv
|
|
[ "$status" -eq 0 ]
|
|
# deepseek format = thinking lands in reasoning_content, the field the
|
|
# harness reads on this endpoint; --jinja is what makes
|
|
# chat_template_kwargs reach the template at all.
|
|
[ "$(argv_value --reasoning-format)" = "deepseek" ]
|
|
argv_value --jinja >/dev/null
|
|
grep -qxF -- '--jinja' <<<"$output"
|
|
}
|
|
|
|
@test "the serve script refuses a missing runtime, GGUF or projector" {
|
|
rm -f "$MODEL_DIR/$MM_STUB"
|
|
serve_argv
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"frank-small: missing"* ]]
|
|
|
|
rm -f "$RUNTIME_DIR/bin/llama-server"
|
|
serve_argv
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"frank-small: missing"* ]]
|
|
}
|
|
|
|
@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" {
|
|
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
|
|
grep -q 'LLM_SMALL_RUNTIME' "$REMOTE_DEPLOY"
|
|
grep -q 'LLM_SMALL_MODEL_FILE' "$REMOTE_DEPLOY"
|
|
grep -q 'LLM_SMALL_MMPROJ' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
|
grep -q 'frank-small.service has no MainPID' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
|
|
# frank-large's gufo sweep must not take down frank-small, whose
|
|
# process is llama-server on the SAME host since this swap.
|
|
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
|
|
local large
|
|
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
|
|
! grep -q 'pkill -x llama-server' <<<"$large"
|
|
}
|