frank-large: roll back to gufo Qwen3.8-27B Q4_K_XL, one session x 262144 f16 KV
frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144, asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
This commit is contained in:
Frank
+294
-162
@@ -1,206 +1,338 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: frank-large — the Dankclaw iGPU endpoint on :7770.
|
||||
# Seam: the frank-large root unit on dankclaw.
|
||||
#
|
||||
# This file asserts the vLLM ROOT-UNIT shape of the 2026-10-01 cutover: the
|
||||
# BF16 MiMo-V2.6-Distill-Qwen-9B payload (the same arch frank-small serves
|
||||
# INT8 on the 3090) under the digest-pinned kyuz0 TheRock gfx1151 image,
|
||||
# with CROSS-MODEL MTP spec decode from the Qwen3.5-9B head. The gufo-era
|
||||
# laws (built-from-source runtime, GGUF schema gates, DFlash2-on-27B) retired
|
||||
# with that runtime; rollback is the tag frank-large-gufo-q4kxl-20261001 and
|
||||
# scripts/build-gufo.sh stays on disk for it.
|
||||
# Since 2026-09-28 this endpoint is served by gufo (gufo-org/gufo), the
|
||||
# Strix Halo inference engine built from the pinned v0.1.1 tag, serving
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) UD-Q8_K_XL with the z-lab
|
||||
# DFlash2 Q4_K_M drafter. Two changes landed on this endpoint that day: the
|
||||
# engine (from the stock llama.cpp prebuilt) and the payload (from
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL). Port, alias and client contract did
|
||||
# not move.
|
||||
#
|
||||
# What the seam pins that the flags alone do not say: the measured laws of
|
||||
# the 2026-10-01 window (BF16 decode 13.1 t/s is ~94% of the LPDDR5x roof,
|
||||
# so spec is the only way past the wall; MTP-2 measured +51% decode against
|
||||
# DFlash-8's +33%; every spec arm pays ~55% of prefill; spec arms do NOT
|
||||
# reproduce the no-spec greedy text on vLLM), the roof-free shape (KV grows
|
||||
# only on the 8 full-attention layers, so 4 x 262,144 fits the GTT
|
||||
# aperture), the client contract difference (vLLM emits `reasoning`, not
|
||||
# llama.cpp's `reasoning_content`), and the deployer's image/payload/spec
|
||||
# gates with the false-green MainPID check.
|
||||
|
||||
# Most of this seam runs the serve script against a stub engine and asserts
|
||||
# on the argv it actually execs, so what is pinned is the command the
|
||||
# operator gets rather than the text of the script: a tunable renamed,
|
||||
# dropped, or silently defaulted away fails here the way it would fail a
|
||||
# deployment.
|
||||
load helpers
|
||||
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
||||
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
||||
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
||||
FETCH="$REPO_ROOT/scripts/download-frank-large-q4.sh"
|
||||
DRAFTER_FETCH="$REPO_ROOT/scripts/download-frank-large.sh"
|
||||
BUILD="$REPO_ROOT/scripts/build-gufo.sh"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
FETCH_TARGET="$REPO_ROOT/scripts/download-frank-large-mimo.sh"
|
||||
FETCH_DRAFT="$REPO_ROOT/scripts/download-frank-large-dflash.sh"
|
||||
FETCH_MTPSRC="$REPO_ROOT/scripts/download-frank-large-qwen35.sh"
|
||||
# The deployed target's filename, kept here so a rename in the env file and a
|
||||
# rename in this stub cannot drift apart unnoticed.
|
||||
TARGET_STUB="Qwen3.8-27B-UD-Q4_K_XL.gguf"
|
||||
setup() {
|
||||
STAGE="$(mktemp -d)"
|
||||
RUNTIME_DIR="$STAGE/gufo"
|
||||
MODEL_DIR="$STAGE/models"
|
||||
DRAFTER_DIR="$STAGE/draft"
|
||||
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR" "$DRAFTER_DIR"
|
||||
# A stub engine: the serve script execs it, so $@ is exactly what gufo
|
||||
# would be invoked with.
|
||||
cat > "$RUNTIME_DIR/bin/gufo" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$@"
|
||||
STUB
|
||||
chmod +x "$RUNTIME_DIR/bin/gufo"
|
||||
# The target and the drafter live in SEPARATE directories since the
|
||||
# 2026-09-28 payload swap: the target came from a different repository,
|
||||
# the DFlash2 drafter did not move. A stub that colocated them would
|
||||
# never notice DRAFTER_DIR going stale.
|
||||
: > "$MODEL_DIR/$TARGET_STUB"
|
||||
: > "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
STUB_ENV="$STAGE/stub.env"
|
||||
cat > "$STUB_ENV" <<EOF
|
||||
RUNTIME_DIR=$RUNTIME_DIR
|
||||
MODEL_DIR=$MODEL_DIR
|
||||
DRAFTER_DIR=$DRAFTER_DIR
|
||||
EOF
|
||||
}
|
||||
|
||||
@test "frank-large unit, serve script and env exist in scripts/" {
|
||||
teardown() {
|
||||
if [ -n "${STAGE:-}" ]; then
|
||||
rm -rf "$STAGE"
|
||||
fi
|
||||
}
|
||||
|
||||
# serve_argv [env-file] — run the serve script and capture the engine's argv,
|
||||
# one token per line, in $output. No argument exercises the built-in defaults
|
||||
# with the stub's paths.
|
||||
serve_argv() {
|
||||
local envfile="${1:-$STUB_ENV}"
|
||||
if [ -n "$envfile" ]; then
|
||||
FRANK_LARGE_ENV_FILE="$envfile" run bash "$SERVE"
|
||||
else
|
||||
run env FRANK_LARGE_ENV_FILE="$STAGE/absent.env" bash "$SERVE"
|
||||
fi
|
||||
}
|
||||
|
||||
# argv_has <token> — exact token match, so --port never satisfies a test
|
||||
# looking for a different flag.
|
||||
argv_has() {
|
||||
grep -qxF -- "$1" <<<"$output"
|
||||
}
|
||||
|
||||
# argv_value <flag> — the token following <flag>, empty if the flag is absent.
|
||||
argv_value() {
|
||||
local i
|
||||
i="$(grep -nx -- "$1" <<<"$output" | cut -d: -f1)"
|
||||
[ -n "$i" ] || return 0
|
||||
sed -n "$((i + 1))p" <<<"$output"
|
||||
}
|
||||
|
||||
@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" {
|
||||
[ -f "$ENV_FILE" ]
|
||||
[ -f "$UNIT" ]
|
||||
[ -x "$SERVE" ]
|
||||
[ -f "$ENV_FILE" ]
|
||||
[ -x "$BUILD" ]
|
||||
}
|
||||
|
||||
@test "frank-large is boot-persistent and self-healing" {
|
||||
@test "frank-large is boot-persistent, self-healing and unprivileged" {
|
||||
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
||||
# Restart=always, not on-failure: vLLM's API server can drain and exit 0
|
||||
# after an engine fault (the frank-primary 2026-10-01 lesson), so
|
||||
# on-failure would leave the endpoint dark until a human notices.
|
||||
grep -q '^Restart=always$' "$UNIT"
|
||||
! grep -q '^Restart=on-failure$' "$UNIT"
|
||||
grep -q '^Restart=on-failure$' "$UNIT"
|
||||
grep -q '^RestartSec=' "$UNIT"
|
||||
# ExecStop runs `podman stop -t 120`; TimeoutStopSec MUST exceed it or
|
||||
# every stop records 'Failed with result timeout' (frank-primary law).
|
||||
grep -q '^ExecStop=.*podman stop -t 120 frank-large-mimo' "$UNIT"
|
||||
stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT")
|
||||
[ -n "$stop_t" ] && [ "$stop_t" -gt 120 ]
|
||||
# StartLimitIntervalSec belongs to [Unit]; a [Service] copy is an
|
||||
# 'Unknown key' warning on every load.
|
||||
[ "$(grep -c '^StartLimitIntervalSec=' "$UNIT")" = "1" ]
|
||||
awk '/^\[Service\]/{in_svc=1} /^StartLimitIntervalSec=/{if(in_svc) exit 1}' "$UNIT"
|
||||
grep -q '^User=dank$' "$UNIT"
|
||||
# The engine pins buffers; the memlock budget must be unbounded or the
|
||||
# load fails on the RLIMIT.
|
||||
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
grep -q '^OOMScoreAdjust=-500$' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the unit runs the repo's own serve script with the repo's env" {
|
||||
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh$' "$UNIT"
|
||||
# The unit runs the repo's own serve script with the repo's env.
|
||||
grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT"
|
||||
grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the serve shape is direct podman run, not the retired gufo or llama shapes" {
|
||||
grep -q 'podman run' "$SERVE"
|
||||
grep -q 'exec podman wait "$FRANK_LARGE_CONTAINER"' "$SERVE"
|
||||
# The gufo runtime and the llama.cpp prebuilt are both retired engines;
|
||||
# nothing in this trio may reach for them.
|
||||
! grep -q 'gufo serve' "$SERVE"
|
||||
! grep -q 'llama-server' "$SERVE"
|
||||
! grep -q -- '--sessions' "$SERVE"
|
||||
! grep -q -- '--model-draft' "$SERVE"
|
||||
! grep -q -- '--dflash-model' "$SERVE"
|
||||
@test "the fleet endpoint and alias survive the cutover" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
# frank-large at :7770, whatever engine answers it.
|
||||
[ "$(argv_value --served-model-name)" = "frank-large" ]
|
||||
[ "$(argv_value --port)" = "7770" ]
|
||||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "the serve script gates the digest-pinned image and the complete payload" {
|
||||
grep -q 'FRANK_LARGE_IMAGE' "$SERVE"
|
||||
grep -q 'RepoDigests' "$SERVE"
|
||||
grep -q 'FRANK_LARGE_MODEL_SHARDS' "$SERVE"
|
||||
grep -q 'config.json' "$SERVE"
|
||||
grep -q 'model.safetensors.index.json' "$SERVE"
|
||||
# Whole /dev/dri, never individual render nodes (Dankserver 2026-09-29:
|
||||
# renderD-only mounts enumerate the GPU but cannot create a context).
|
||||
grep -q -- '--device /dev/dri' "$SERVE"
|
||||
! grep -q 'renderD' "$SERVE"
|
||||
# The env file pins by digest, not by the floating tag.
|
||||
! grep -qE '^FRANK_LARGE_IMAGE=.*:latest$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:' "$ENV_FILE"
|
||||
@test "the built-in defaults are one session of the full 262144 window with DFlash2" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
# 262144 is the model's native window (the operator's call, 2026-09-21).
|
||||
[ "$(argv_value --context)" -eq 262144 ]
|
||||
# One session is the single stream every prior frank-large served.
|
||||
[ "$(argv_value --sessions)" -eq 1 ]
|
||||
[ "$(argv_value --speculative)" = "dflash2" ]
|
||||
# A draft ceiling of 0 would silently serve the autoregressive rate.
|
||||
[ "$(argv_value --draft-tokens)" -gt 0 ]
|
||||
}
|
||||
|
||||
@test "the qualified engine shape is in the env file" {
|
||||
# 2026-10-01 window: 262144 is the model's native window and the GTT
|
||||
# aperture swallows it — KV grows ONLY on the 8 full-attention layers
|
||||
# (32 KiB/token bf16); the 24 GDN layers hold fixed ~25 MiB state per
|
||||
# running sequence. SEQS 4 keeps the four-session contract; the pool is
|
||||
# shared by the scheduler instead of carved per session like gufo.
|
||||
grep -q '^FRANK_LARGE_MAXLEN=262144$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_SEQS=4$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_UTIL=0.45$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_MODEL_SHARDS=4$' "$ENV_FILE"
|
||||
@test "the shipped env file, not just the defaults, serves the deployed shape" {
|
||||
# The defaults are a floor; systemd sources the env file. Only the
|
||||
# on-host paths are redirected at the stub, so every fleet value in the
|
||||
# shipped file is the one under test. MODEL_FILE and DRAFTER_FILE are
|
||||
# absolute in the shipped file, so they need redirecting too — a sed that
|
||||
# only rewrote MODEL_DIR would leave them pointing at /home/dank and the
|
||||
# test would pass for the wrong reason (or fail on a box that has them).
|
||||
sed -e "s#^RUNTIME_DIR=.*#RUNTIME_DIR=$RUNTIME_DIR#" \
|
||||
-e "s#^MODEL_DIR=.*#MODEL_DIR=$MODEL_DIR#" \
|
||||
-e "s#^MODEL_FILE=.*#MODEL_FILE=$MODEL_DIR/$TARGET_STUB#" \
|
||||
-e "s#^DRAFTER_DIR=.*#DRAFTER_DIR=$DRAFTER_DIR#" \
|
||||
-e "s#^DRAFTER_FILE=.*#DRAFTER_FILE=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf#" \
|
||||
"$ENV_FILE" > "$STAGE/shipped.env"
|
||||
serve_argv "$STAGE/shipped.env"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --served-model-name)" = "frank-large" ]
|
||||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||||
[ "$(argv_value --port)" = "7770" ]
|
||||
# 2026-10-01 night: Frank rolled frank-large back to ONE session at the
|
||||
# full native window (gufo Q4_K_XL, f16 KV — gufo has no KV quant lever).
|
||||
[ "$(argv_value --context)" -eq 262144 ]
|
||||
[ "$(argv_value --sessions)" -eq 1 ]
|
||||
[ "$(argv_value --speculative)" = "dflash2" ]
|
||||
[ "$(argv_value --draft-tokens)" -gt 0 ]
|
||||
[ "$(argv_value --draft-policy)" = "adaptive" ]
|
||||
# The shipped file must name the target the fleet actually serves, not a
|
||||
# path that merely happens to exist.
|
||||
[ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ]
|
||||
[ "$(argv_value --dflash-model)" = "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" ]
|
||||
}
|
||||
|
||||
@test "the serve script wires the engine flags from the env" {
|
||||
grep -q -- '--max-model-len "$FRANK_LARGE_MAXLEN"' "$SERVE"
|
||||
grep -q -- '--max-num-seqs "$FRANK_LARGE_SEQS"' "$SERVE"
|
||||
grep -q -- '--gpu-memory-utilization "$FRANK_LARGE_UTIL"' "$SERVE"
|
||||
grep -q -- '--dtype bfloat16' "$SERVE"
|
||||
grep -q -- '--trust-remote-code' "$SERVE"
|
||||
grep -q -- '--ulimit memlock=-1:-1' "$SERVE"
|
||||
# vllm runs INSIDE the container: the model path handed to `serve` is
|
||||
# the /models mount path, never the host path (2026-10-01: the host
|
||||
# path crash-looped 51 times on "Repo id must be in the form…").
|
||||
grep -q 'serve "$MODEL_CONTAINER_DIR"' "$SERVE"
|
||||
grep -q 'MODEL_CONTAINER_DIR="/models/' "$SERVE"
|
||||
! grep -qE '^\s+serve "\$FRANK_LARGE_MODEL_DIR"' "$SERVE"
|
||||
@test "the engine is pointed at the payload that is actually on disk" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ -f "$(argv_value --model)" ]
|
||||
# A drafter path that does not exist would load the target and decode at
|
||||
# the autoregressive rate, which reads as "the new runtime is slow".
|
||||
[ -f "$(argv_value --dflash-model)" ]
|
||||
[ "$(argv_value --model)" != "$(argv_value --dflash-model)" ]
|
||||
}
|
||||
|
||||
@test "speculative decode is cross-model MTP BY MEASUREMENT" {
|
||||
# The 2026-10-01 window on this host: base 13.1 t/s, DFlash-8 17.4,
|
||||
# MTP-2 19.8 (+51%). MTP acceptance is 71%/44% per position (token rate
|
||||
# 57%) because the head trained on the distill's SOURCE (Qwen/Qwen3.5-9B,
|
||||
# the only place mtp.* tensors exist for this family) still reads MiMo's
|
||||
# hidden states. Beyond 2 draft tokens is UNMEASURED; the DFlash drafter
|
||||
# stays on disk as the alternative method.
|
||||
grep -q '^FRANK_LARGE_SPEC=mtp$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_SPEC_TOKENS=2$' "$ENV_FILE"
|
||||
# The JSON is built ONLY when SPEC is set, and the draft path handed to
|
||||
# the engine is the CONTAINER mount path, never the host path.
|
||||
grep -q -- '--speculative-config' "$SERVE"
|
||||
grep -q 'FRANK_LARGE_SPEC_CONTAINER_DIR="/models/' "$SERVE"
|
||||
# The draft gate accepts BOTH weight layouts: a single
|
||||
# model.safetensors (dflash) and a sharded index (the Qwen3.5-9B MTP
|
||||
# head). A gate keyed only to the single file rejects the deployed arm.
|
||||
grep -q 'FRANK_LARGE_SPEC_DIR/model.safetensors.index.json' "$SERVE"
|
||||
# itself carries ZERO mtp.* tensors (that absence is what makes SGLang's
|
||||
# NEXTN on frank-small a 0.00-acceptance ghost).
|
||||
grep -q 'ZERO mtp' "$ENV_FILE"
|
||||
@test "frank-large stays text-only" {
|
||||
# The payload has no mmproj, so an --mmproj would point at nothing. The
|
||||
# fleet's image path is frank-small: its SGLang MiMo payload carries the
|
||||
# vision tower inside the checkpoint (no projector file at all), and
|
||||
# models.yml declares frank-large text-only.
|
||||
serve_argv
|
||||
! argv_has --mmproj
|
||||
grep -q 'MiMo-Q6K' "$REPO_ROOT/scripts/frank-small.env"
|
||||
}
|
||||
|
||||
@test "tools and reasoning are wired — the client contract is served" {
|
||||
grep -q '^FRANK_LARGE_REASONING_PARSER=qwen3$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_TOOL_PARSER=qwen3_coder$' "$ENV_FILE"
|
||||
grep -q -- '--reasoning-parser "$FRANK_LARGE_REASONING_PARSER"' "$SERVE"
|
||||
grep -q -- '--enable-auto-tool-choice' "$SERVE"
|
||||
grep -q -- '--tool-call-parser "$FRANK_LARGE_TOOL_PARSER"' "$SERVE"
|
||||
# vLLM's field name is `reasoning`, NOT llama.cpp/gufo's
|
||||
# `reasoning_content` — the harness models.yml compat block must match.
|
||||
grep -q 'reasoning' "$ENV_FILE"
|
||||
@test "the engine is asked for greedy decoding and for preserved prior reasoning" {
|
||||
# Greedy is what this endpoint has always served; a silent switch to
|
||||
# sampled output would be a model-behaviour change disguised as a
|
||||
# runtime swap. Prior thinking blocks must survive across turns or a
|
||||
# multi-turn agent loses its own reasoning between tool calls.
|
||||
serve_argv
|
||||
[ "$(argv_value --temperature)" = "0" ]
|
||||
[ "$(argv_value --preserve-thinking)" = "on" ]
|
||||
}
|
||||
|
||||
@test "the serve script clears stale containers including the bench identity" {
|
||||
grep -q '^FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo"$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_CONTAINER=frank-large-mimo$' "$ENV_FILE"
|
||||
grep -q 'FRANK_LARGE_LEGACY_CONTAINERS' "$SERVE"
|
||||
@test "the serve script refuses a missing runtime with the build pointer" {
|
||||
rm -f "$RUNTIME_DIR/bin/gufo"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"runtime not found"* ]]
|
||||
[[ "$output" == *"build-gufo.sh"* ]]
|
||||
# A restart loop would be the alternative; nothing may be exec'd.
|
||||
! argv_has serve
|
||||
}
|
||||
|
||||
@test "frank-large holds the single fleet port under its own alias" {
|
||||
grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_FILE"
|
||||
grep -q '^FRANK_LARGE_SERVED_NAME=frank-large$' "$ENV_FILE"
|
||||
@test "the serve script refuses a missing drafter with a fetch pointer" {
|
||||
# The drafter lives in its own directory now, so the target can be present
|
||||
# and this still has to be caught.
|
||||
rm -f "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"missing asset"* ]]
|
||||
[[ "$output" == *"Qwen3.8-27B-DFlash2-Q4_K_M.gguf"* ]]
|
||||
[[ "$output" == *"download-frank-large"* ]]
|
||||
! argv_has serve
|
||||
}
|
||||
|
||||
@test "no bare HIP-reserved env var is defined" {
|
||||
# HIP reads bare TEMP/HOST/GPU_MAX_HEAP_SIZE; the serve script sources
|
||||
# this file with set -a, so an unprefixed key leaks into the engine.
|
||||
! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
||||
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
||||
@test "the serve script refuses a missing target and names the file" {
|
||||
rm -f "$MODEL_DIR/$TARGET_STUB"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"missing asset"* ]]
|
||||
[[ "$output" == *"$TARGET_STUB"* ]]
|
||||
}
|
||||
|
||||
@test "the fetch scripts carry the verified payload and draft manifests" {
|
||||
grep -q 'XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B' "$FETCH_TARGET"
|
||||
[ "$(grep -c 'safetensors.*[0-9a-f]\{64\}' "$FETCH_TARGET")" = "4" ]
|
||||
grep -q -- '--verify-only' "$FETCH_TARGET"
|
||||
grep -q 'z-lab/Qwen3.5-9B-DFlash' "$FETCH_DRAFT"
|
||||
grep -q -- '--verify-only' "$FETCH_DRAFT"
|
||||
# The env file's payload/draft dirs must be where the fetch scripts write.
|
||||
grep -q 'MiMo-V2.6-Distill-Qwen-9B-BF16' "$FETCH_TARGET"
|
||||
grep -q 'Qwen3.5-9B-DFlash' "$FETCH_DRAFT"
|
||||
grep -q 'Qwen/Qwen3.5-9B' "$FETCH_MTPSRC"
|
||||
grep -q '/home/dank/models/Qwen3.5-9B' "$FETCH_MTPSRC"
|
||||
@test "both payload files are fetched resumably and verified against declared sizes + sha256" {
|
||||
# The target and the drafter come from DIFFERENT repositories, fetched by
|
||||
# two scripts. Anchor the repo checks to the URL ASSIGNMENT, not to the
|
||||
# file: the same repo string also appears in each script's header comment,
|
||||
# so a plain grep stayed green after the URL was repointed at another
|
||||
# repository -- the script would then download something the serve script
|
||||
# never asked for.
|
||||
grep -qE '^URL="https://huggingface\.co/unsloth/Qwen3\.8-27B-GGUF/resolve/main/' "$FETCH"
|
||||
grep -qE 'https://huggingface\.co/z-lab/Qwen3\.8-27B-DFlash2-GGUF/resolve/main/' "$DRAFTER_FETCH"
|
||||
for f in "$FETCH" "$DRAFTER_FETCH"; do
|
||||
# -f so an HTTP error is a failure rather than a small "artifact";
|
||||
# -C - so an interrupted 29 GiB run resumes.
|
||||
grep -q -- '-fL -C -' "$f"
|
||||
# The size and the sha256 are compared after the transfer, not trusted.
|
||||
grep -q 'SHA256 MISMATCH' "$f"
|
||||
grep -q 'SIZE MISMATCH' "$f"
|
||||
done
|
||||
}
|
||||
|
||||
@test "the gufo deploy gates are retired with the gufo runtime" {
|
||||
! grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY"
|
||||
! grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY"
|
||||
! grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY"
|
||||
! grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY"
|
||||
# build-gufo.sh stays on disk as the rollback path to the tagged engine.
|
||||
[ -f "$REPO_ROOT/scripts/build-gufo.sh" ]
|
||||
@test "the serve script and the fetch scripts agree on the payload filenames" {
|
||||
# The serve script's MODEL_FILE and the fetch script's REL are two
|
||||
# independent spellings of one filename, and a repo is not required to
|
||||
# keep them in step: a re-upload under a new name, or an env edited
|
||||
# without the fetch, leaves the endpoint unable to start with a
|
||||
# "missing asset" that no test would have predicted. Bind them here.
|
||||
local served fetched drafter_served drafter_fetched
|
||||
served="$(basename "$(grep '^MODEL_FILE=' "$ENV_FILE" | cut -d= -f2-)")"
|
||||
fetched="$(sed -n 's/^REL="\(.*\)"$/\1/p' "$FETCH")"
|
||||
[ -n "$served" ]
|
||||
[ "$served" = "$fetched" ]
|
||||
drafter_served="$(basename "$(grep '^DRAFTER_FILE=' "$ENV_FILE" | cut -d= -f2-)")"
|
||||
drafter_fetched="$(sed -n 's/^ *"\(Qwen3\.8-27B-DFlash2[^ ]*\.gguf\) .*/\1/p' "$DRAFTER_FETCH")"
|
||||
[ -n "$drafter_served" ]
|
||||
[ "$drafter_served" = "$drafter_fetched" ]
|
||||
}
|
||||
|
||||
@test "the deployer gates image payload and spec then health-checks with MainPID" {
|
||||
grep -q 'LLM_LARGE_CONTAINER=frank-large-mimo' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"' "$REMOTE_DEPLOY"
|
||||
grep -q 'FRANK_LARGE_IMAGE' "$REMOTE_DEPLOY"
|
||||
grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY"
|
||||
grep -q 'FRANK_LARGE_MODEL_SHARDS' "$REMOTE_DEPLOY"
|
||||
# The deployer's draft gate must accept the SHARDED MTP layout too —
|
||||
# the 2026-10-01 first deploy died on exactly this single-file check.
|
||||
grep -q 'spec_dir/model.safetensors.index.json' "$REMOTE_DEPLOY"
|
||||
grep -q 'FRANK_LARGE_SPEC=' "$REMOTE_DEPLOY"
|
||||
@test "the drafter is still fetched by the script that carries its Hub repo" {
|
||||
# Guards a future "simplification" that folds the drafter into the
|
||||
# payload-fetch script and silently orphans the artifact the serve script
|
||||
# still points at via DRAFTER_DIR. The literal drafter name must not
|
||||
# appear in that script AT ALL, comments included.
|
||||
grep -q 'Qwen3.8-27B-UD-Q8_K_XL' "$DRAFTER_FETCH"
|
||||
! grep -q 'DFlash2' "$FETCH"
|
||||
}
|
||||
|
||||
@test "the build script refuses a source tree that is not the pinned commit" {
|
||||
# gufo ships no prebuilt Linux binaries, so the runtime is built here.
|
||||
# A tag that moves under us would silently install different kernels
|
||||
# under the same RUNTIME_DIR, so the pin is enforced, not documented.
|
||||
local src="$STAGE/not-the-pinned-commit"
|
||||
mkdir -p "$src"
|
||||
git -C "$src" init -q
|
||||
git -C "$src" -c user.email=t@t -c user.name=t commit -q --allow-empty -m drift
|
||||
run env GUFO_SRC_DIR="$src" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"GUFO_COMMIT mismatch"* ]]
|
||||
}
|
||||
|
||||
@test "the build script reports a runtime that was never built" {
|
||||
# --verify on a host with no install must say so rather than exit 0 on
|
||||
# an empty check.
|
||||
run env GUFO_SRC_DIR="$STAGE/no-source-tree" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"no gufo at"* ]]
|
||||
}
|
||||
|
||||
@test "the deploy handler gates on the runtime, the payload and the port" {
|
||||
grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
||||
grep -q 'MainPID --value frank-large.service' "$REMOTE_DEPLOY"
|
||||
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "the deploy handler will not stop a serving endpoint it cannot replace" {
|
||||
# The runtime gate must be live (a real conditional on an executable
|
||||
# path, not a branch that never fires) and it must return BEFORE the
|
||||
# stop, or a host whose gufo was never built takes frank-large down on
|
||||
# its way to reporting the failure.
|
||||
local large gate bail stop
|
||||
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||||
[[ "$large" == *'build-gufo.sh'* ]]
|
||||
gate="$(grep -n 'if \[\[ ! -x "\$LLM_LARGE_RUNTIME" \]\]' <<<"$large" | head -1 | cut -d: -f1)"
|
||||
stop="$(grep -n 'systemctl stop frank-large' <<<"$large" | head -1 | cut -d: -f1)"
|
||||
[ -n "$gate" ]
|
||||
[ -n "$stop" ]
|
||||
bail="$(awk -v g="$gate" 'NR > g && /return 1/ { print NR; exit }' <<<"$large")"
|
||||
[ -n "$bail" ]
|
||||
[ "$bail" -lt "$stop" ]
|
||||
}
|
||||
|
||||
@test "the stray-run sweep keys on the drafter and cannot take frank-small down" {
|
||||
# The sweep used to key on the TARGET filename, which the 2026-09-28
|
||||
# payload swap moved to a different repository. It would have stopped
|
||||
# matching the live process, and a stray run would then hold 7770 past the
|
||||
# port gate and fail the deploy. The drafter is the part that did NOT
|
||||
# move, and every frank-large command line carries it via --dflash-model.
|
||||
grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY"
|
||||
# frank-small serves a different GGUF on the 3090 with no DFlash2 drafter,
|
||||
# so a binary-name sweep is the only thing that would endanger it.
|
||||
local large
|
||||
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||||
! grep -q 'pkill -x llama-server' <<<"$large"
|
||||
! grep -q 'pkill -x gufo' <<<"$large"
|
||||
}
|
||||
|
||||
@test "the deploy handler retires the halogen quadlet without deleting it" {
|
||||
grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY"
|
||||
grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY"
|
||||
! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY"
|
||||
# Nothing live ships for this endpoint as a container any more. The
|
||||
# archived copy is host state in an ignored .AGENTTRASH — deliberately
|
||||
# untracked, and present only in a primary checkout — so search the
|
||||
# tracked tree, not the filesystem.
|
||||
[ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ]
|
||||
! git -C "$REPO_ROOT" grep -q 'Image=.*halogen' -- dankclaw
|
||||
}
|
||||
|
||||
+120
-94
@@ -1,30 +1,65 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-small SGLang root unit on dankclaw.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-01 it serves
|
||||
# letechlead/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound (qwen3_5 hybrid,
|
||||
# INT8 W8A16 AutoRound, in-checkpoint vision) via SGLang in a
|
||||
# python3.12 venv. This seam pins what the flags alone do not say: boot
|
||||
# persistence for a NATIVE runtime (no generated quadlet is involved), the
|
||||
# repo-relative paths the unit executes, the separate client port, the
|
||||
# no-spec-decode gate (the checkpoint ships NO mtp.* tensors, so a NEXTN
|
||||
# draft would be an uninitialised ghost model), the 8-shard payload gate,
|
||||
# and the KV budget: fp8 KV at 0.98 utilization whose 4-slot grid must fit
|
||||
# the card's 24 GiB.
|
||||
# Seam: the frank-small llama.cpp root unit on dankclaw.
|
||||
# frank-small is the RTX 3090 endpoint; since 2026-10-01 night it serves
|
||||
# bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF:Q6_K via a CUDA (sm_86) build of
|
||||
# llama.cpp master, chosen for the ASYMMETRIC KV Frank asked for: K f16
|
||||
# (unquantized) + V q8_0, which only llama.cpp exposes per-tensor. This
|
||||
# seam runs the serve script against a stub engine and asserts on the argv
|
||||
# it actually execs, so what is pinned is the command the operator gets.
|
||||
|
||||
load helpers
|
||||
|
||||
UNIT="$REPO_ROOT/scripts/frank-small.service"
|
||||
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
|
||||
DOWNLOAD="$REPO_ROOT/scripts/download-frank-small.sh"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
|
||||
TARGET_STUB="MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf"
|
||||
|
||||
@test "frank-small unit, serve script, env and download script exist in scripts/" {
|
||||
setup() {
|
||||
STAGE="$(mktemp -d)"
|
||||
RUNTIME_DIR="$STAGE/llama"
|
||||
MODEL_DIR="$STAGE/models"
|
||||
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR"
|
||||
cat > "$RUNTIME_DIR/bin/llama-server" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$@"
|
||||
STUB
|
||||
chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server"
|
||||
touch "$MODEL_DIR/$TARGET_STUB"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
rm -rf "$STAGE"
|
||||
}
|
||||
|
||||
# serve_argv [env-file] — run the serve script and capture the engine's
|
||||
# argv, one token per line, in $output.
|
||||
serve_argv() {
|
||||
local envf="${1:-$STAGE/test.env}"
|
||||
cat > "$envf" <<EOF
|
||||
LLAMA_BIN=$RUNTIME_DIR/bin/llama-server
|
||||
MODEL_FILE=$MODEL_DIR/$TARGET_STUB
|
||||
ALIAS=frank-small
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7771
|
||||
CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")
|
||||
CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
|
||||
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||||
EOF
|
||||
run env FRANK_SMALL_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
|
||||
}
|
||||
|
||||
argv_value() {
|
||||
grep -A1 -xF -- "$1" <<<"$output" | tail -1
|
||||
}
|
||||
|
||||
@test "frank-small unit, serve script and env exist in scripts/" {
|
||||
[ -f "$UNIT" ]
|
||||
[ -f "$SERVE" ]
|
||||
[ -f "$ENV_FILE" ]
|
||||
[ -f "$DOWNLOAD" ]
|
||||
}
|
||||
|
||||
@test "frank-small is boot-persistent, self-healing and unprivileged" {
|
||||
@@ -33,103 +68,94 @@ ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
|
||||
grep -q '^RestartSec=' "$UNIT"
|
||||
grep -q '^User=dank$' "$UNIT"
|
||||
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
}
|
||||
|
||||
@test "frank-small runs the repo's own serve script with the repo's env" {
|
||||
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh$' "$UNIT"
|
||||
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "frank-small serves the MiMo-V2.6 checkpoint from the self-contained SGLang venv" {
|
||||
grep -q '^SGLANG_BIN=/home/dank/GitPackages/sglang-cu128/bin/sglang$' "$ENV_FILE"
|
||||
grep -q '^MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound$' "$ENV_FILE"
|
||||
grep -q -- '--model-path "$MODEL_DIR"' "$SERVE"
|
||||
# The AutoRound checkpoint declares its quant method; SGLang must load it
|
||||
# as auto-round, not re-quantize online.
|
||||
grep -q '^QUANTIZATION=auto-round$' "$ENV_FILE"
|
||||
grep -q -- '--quantization "$QUANTIZATION"' "$SERVE"
|
||||
@test "the shipped env points at the llama.cpp runtime and the Q6_K GGUF" {
|
||||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=/home/dank/models/MiMo-Q6K/$TARGET_STUB$" "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "frank-small runs WITHOUT speculative decoding (no MTP weights shipped)" {
|
||||
# The config declares mtp_num_hidden_layers=1 but the safetensors index
|
||||
# carries zero mtp.* tensors (base and INT8 conversion alike, verified
|
||||
# 2026-10-01). SGLang would draft with an uninitialised 4.24 GiB model
|
||||
# at 0% acceptance; the serve script must not pass any speculative flag.
|
||||
! grep -q -- '--speculative-' "$SERVE"
|
||||
! grep -q '^SPEC_' "$ENV_FILE"
|
||||
@test "the shipped env serves the fleet shape: two full native windows" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --alias)" = "frank-small" ]
|
||||
[ "$(argv_value --port)" = "7771" ]
|
||||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||||
[ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ]
|
||||
[ "$(argv_value --parallel)" -eq 2 ]
|
||||
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
||||
# llama.cpp divides --ctx-size across slots: two slots means the pool
|
||||
# must be 2 x the window, and the window never exceeds the native one.
|
||||
ctx="$(argv_value --ctx-size)"
|
||||
par="$(argv_value --parallel)"
|
||||
[ $((ctx / par)) -eq 262144 ]
|
||||
}
|
||||
|
||||
@test "frank-small carries the fleet's image path, and only it" {
|
||||
# The vision tower is baked into the safetensors (qwen3_5_vision): no
|
||||
# separate projector file, and the serve script must not look for one.
|
||||
! grep -q '^MMPROJ_PATH=' "$ENV_FILE"
|
||||
@test "the KV contract is asymmetric: unquantized K, q8_0 V, flash-attn on" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --cache-type-k)" = "f16" ]
|
||||
[ "$(argv_value --cache-type-v)" = "q8_0" ]
|
||||
# q8_0 on V only exists on the flash-attention path; without this flag
|
||||
# pair the KV contract in the env file is a lie the engine never sees.
|
||||
[ "$(argv_value --flash-attn)" = "on" ]
|
||||
}
|
||||
|
||||
@test "the KV budget fits the 3090 with room for checkpoints" {
|
||||
# K f16 (16 KiB/token) + V q8_0 (~8.5, budget 25 with slack) over the
|
||||
# whole pool, plus 7.8 GiB weights and ~2 GiB of compute buffers and
|
||||
# CUDA graphs, must stay under the card's 24 GiB. This is arithmetic,
|
||||
# not a boot log: change CTX_SIZE and it bites before deploy does.
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv_gib=$((ctx * 25 / 1048576))
|
||||
[ $((kv_gib + 8 + 2)) -le 24 ]
|
||||
}
|
||||
|
||||
@test "no speculative draft ships by default" {
|
||||
# The qwen35 GGUF has no MTP tensors, and the cross-model DFlash draft
|
||||
# is a measured decision the harness makes later, not a default.
|
||||
! grep -q -- '--model-draft' "$SERVE"
|
||||
! grep -q -- '--spec-type' "$SERVE"
|
||||
! grep -q '^DRAFT' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "frank-small stays text-only on the GGUF path and speaks the fleet reasoning field" {
|
||||
# The GGUF carries no vision projector; the image path left with the
|
||||
# SGLang checkpoint. models.yml declares frank-small text-only since
|
||||
# the 2026-10-01 night swap.
|
||||
! grep -q -- '--mmproj' "$SERVE"
|
||||
! grep -q '^ENABLE_VISION=1$' "$ENV_LARGE"
|
||||
# deepseek format = thinking lands in reasoning_content, the field the
|
||||
# harness has always read on this endpoint (SGLang parity).
|
||||
serve_argv
|
||||
[ "$(argv_value --reasoning-format)" = "deepseek" ]
|
||||
argv_value --jinja >/dev/null
|
||||
grep -qxF -- '--jinja' <<<"$output"
|
||||
}
|
||||
|
||||
@test "frank-small's slot grid is fp8 KV and fits the measured card budget" {
|
||||
local ctx running kv ssmdtype mambasize required
|
||||
ctx="$(sed -nE 's/^CONTEXT_LENGTH=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
running="$(sed -nE 's/^MAX_RUNNING=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv="$(sed -nE 's/^KV_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
ssmdtype="$(sed -nE 's/^MAMBA_SSM_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
|
||||
mambasize="$(sed -nE 's/^MAX_MAMBA_CACHE_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
[ -n "$ctx" ] && [ -n "$running" ] && [ -n "$kv" ]
|
||||
# fp8 KV is the only dtype whose 16 KiB/token makes four slots fit;
|
||||
# 64 tokens per MiB of pool.
|
||||
case "$kv" in fp8*) ;; *) return 1 ;; esac
|
||||
# The DeltaNet ssm state pool is the hidden fit cost: the engine
|
||||
# autofills mamba slots to spend a fixed ~5.1 GB budget, so dtype
|
||||
# alone does not shrink it — the SLOT COUNT must be pinned (bf16,
|
||||
# bounded). These two gates are what make the measured pool below
|
||||
# exist; change either and re-measure the pool at boot.
|
||||
[ "$ssmdtype" = "bfloat16" ]
|
||||
[ -n "$mambasize" ] && [ "$mambasize" -ge "$running" ] && [ "$mambasize" -le 64 ]
|
||||
grep -q -- '--mamba-ssm-dtype "$MAMBA_SSM_DTYPE"' "$SERVE"
|
||||
grep -q -- '--max-mamba-cache-size "$MAX_MAMBA_CACHE_SIZE"' "$SERVE"
|
||||
# The slot window is inside the model's native 262144 window.
|
||||
[ "$ctx" -le 262144 ]
|
||||
# The grid must equal the pinned pool. The engine's autofit picks a
|
||||
# bigger pool (687,388 tokens) but leaves ~0.53 GB — Triton's
|
||||
# deferred kernel loads take most of it and eager-prefill activations
|
||||
# then OOM on the first long request. 606,208 (4 x 151,552) is the
|
||||
# verified ceiling: 170k single request and 4 concurrent 54k requests
|
||||
# run against it (2026-10-01). A weights or mamba-flag change moves
|
||||
# this number; re-measure before trusting the grid.
|
||||
pinned_pool=606208
|
||||
required=$((running * ctx))
|
||||
[ "$required" -le "$pinned_pool" ]
|
||||
grep -q -- '--max-total-tokens "$MAX_TOTAL_TOKENS"' "$SERVE"
|
||||
grep -q -- '--disable-prefill-cuda-graph' "$SERVE"
|
||||
@test "the serve script refuses a missing runtime or payload" {
|
||||
rm -f "$RUNTIME_DIR/bin/llama-server"
|
||||
serve_argv
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"frank-small: missing"* ]]
|
||||
}
|
||||
|
||||
@test "frank-small runs on its own port beside frank-large's" {
|
||||
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
|
||||
grep -q '^ALIAS=frank-small$' "$ENV_FILE"
|
||||
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
|
||||
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
|
||||
# The two endpoints cannot share a port: frank-large publishes
|
||||
# 7770:7770 through podman, this one binds 7771 natively. (The
|
||||
# frank-large vLLM cutover renamed PORT to FRANK_LARGE_PUBLISH.)
|
||||
grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_LARGE"
|
||||
! grep -q '^FRANK_LARGE_PUBLISH=7771' "$ENV_LARGE"
|
||||
}
|
||||
|
||||
@test "the deploy handler installs the root unit and refuses a missing payload" {
|
||||
@test "the deploy handler gates on the binary and the GGUF, then MainPID-hardens health" {
|
||||
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_SMALL_RUNTIME' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_SMALL_MODEL_FILE' "$REMOTE_DEPLOY"
|
||||
grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound' "$REMOTE_DEPLOY"
|
||||
grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY"
|
||||
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
||||
grep -q 'frank-small.service has no MainPID' "$REMOTE_DEPLOY"
|
||||
}
|
||||
|
||||
@test "frank-small's serve script refuses to start on a half-finished download" {
|
||||
# The payload is 8 safetensors shards; the serve script counts them and
|
||||
# the download script pins each one's LFS sha256.
|
||||
grep -q 'frank-small: missing' "$SERVE"
|
||||
grep -q 'expected 8 safetensors shards' "$SERVE"
|
||||
grep -q 'model-00001-of-00008.safetensors' "$DOWNLOAD"
|
||||
grep -q 'model-00008-of-00008.safetensors' "$DOWNLOAD"
|
||||
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
|
||||
# frank-large's gufo sweep must not take down frank-small, whose
|
||||
# process is llama-server on the SAME host since this swap.
|
||||
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
|
||||
local large
|
||||
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||||
! grep -q 'pkill -x llama-server' <<<"$large"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user