Frank
23f09788e9
DFlash2's sidecar (1.92 GiB + buffers on ROCm1) kept the 393216 pool off the table; the MTP head lives in the target file, so its cost is one nextn layer's KV (4096 B/pos) plus small buffers. Fit: ROCm0 29.77 / ROCm1 28.32 GiB, over the floor, 34:31 the balance point at this pool. n_max 3 is the measured MTP winner. The seam's fit is now drafter-aware, the slack is the deployed integer gap (1483 MiB), the ctx cap 393216 with the rope caveat (top 131k extrapolates on the base rope).
299 lines
15 KiB
Bash
Executable File
299 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-medium mainline-llama.cpp root unit on dankserver.
|
|
# frank-medium became a native llama.cpp service on 2026-09-16, replacing the
|
|
# vllm-radiance replica pair + routing proxy that served this endpoint for a
|
|
# day. What this seam pins is what the flags alone do not say: boot persistence
|
|
# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work
|
|
# on ONE model through a layer split with no collective in the path, that the
|
|
# DFlash2 drafter is wired with a depth its block size actually allows, that the
|
|
# runtime and payload are pinned rather than floated, and that one slot's KV
|
|
# still fits a 32 GiB card beside the weights it holds.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-medium.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-medium.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-medium.env"
|
|
BUILD="$REPO_ROOT/scripts/build-llama-rdna4.sh"
|
|
FETCH="$REPO_ROOT/scripts/download-frank-medium-gguf.sh"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
|
|
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
|
|
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
|
|
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
|
|
DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
|
|
|
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
|
|
[ -f "$UNIT" ]
|
|
[ -f "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
[ -f "$BUILD" ]
|
|
[ -f "$FETCH" ]
|
|
}
|
|
|
|
@test "frank-medium is boot-persistent, self-healing and unprivileged" {
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "frank-medium runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh$' "$UNIT"
|
|
grep -q 'FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env' "$UNIT"
|
|
}
|
|
|
|
@test "both R9700s serve ONE model, layer-split inside one process" {
|
|
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
|
|
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
|
|
# The split is a layer PAIR: the shares add up to the model's 65 blocks (one
|
|
# -ts unit per layer) and ROCm1 -- the card that also carries the drafter and
|
|
# the output layers -- takes the FEWER of the two. WHICH pair balances the
|
|
# cards is the seam test's job, which recomputes it from this file; pinning a
|
|
# literal here would only mean that re-measuring the balance breaks a test
|
|
# about parallelism.
|
|
local split_hi split_lo
|
|
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
|
|
[ -n "$split_hi" ] && [ -n "$split_lo" ]
|
|
[ $(( split_hi + split_lo )) -eq 65 ]
|
|
[ "$split_lo" -lt "$split_hi" ]
|
|
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
|
|
grep -q -- '--device "$DEVICES"' "$SERVE"
|
|
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
|
|
# Tensor split is the one shape this host cannot run: it needs a cross-card
|
|
# collective, and the second card cannot dispatch one (see the env file).
|
|
! grep -q '^SPLIT_MODE=tensor$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the iGPU is hidden so no layer can land on system RAM" {
|
|
grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
|
|
grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
|
|
}
|
|
|
|
@test "the drafter is the model's own MTP head, at the measured depth" {
|
|
# MTP's head lives in the target file (blk.<n>.nextn.*): no sidecar, no
|
|
# second copy of the weights, and it is the only drafter that fits the
|
|
# 393216 pool. n_max 3 is the measured MTP winner -- n=7 accepts 33 % of
|
|
# 147, n=3 accepts 58 % of 72 (the env's table; identical greedy text at
|
|
# every depth, so this is pure speed).
|
|
grep -qE '^SPEC_TYPE=draft-mtp$' "$ENV_FILE"
|
|
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
|
local n_max
|
|
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
[ -n "$n_max" ] && [ "$n_max" -eq 3 ]
|
|
# The DFlash2 sidecar stays on disk and in the fetch script -- one
|
|
# SPEC_TYPE change brings the faster chain back -- and ngram-mod's window
|
|
# is retained with it, so neither may be dropped from the env.
|
|
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE"
|
|
grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
|
|
# ...and neither is loaded: ngram-mod's flags only go out when the type
|
|
# contains ngram, so a pure MTP configuration is not handed them.
|
|
python3 -c "
|
|
lines = open('$SERVE').read().splitlines()
|
|
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
|
|
indent = len(lines[guard]) - len(lines[guard].lstrip())
|
|
end = next(i for i in range(guard + 1, len(lines))
|
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
|
assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch'
|
|
"
|
|
}
|
|
|
|
@test "a draft MODEL is passed only for DFlash types, never for MTP" {
|
|
# MTP builds its context against the TARGET model (its head lives in that
|
|
# GGUF). Passing --spec-draft-model alongside it points the draft context at
|
|
# the drafter's file, which has no MTP layers, and the server dies at load:
|
|
# context type MTP requested but model doesn't contain MTP layers
|
|
# failed to create MTP context
|
|
# So the flag must appear exactly once, inside the dflash branch.
|
|
python3 -c "
|
|
lines = open('$SERVE').read().splitlines()
|
|
guard = next(i for i, l in enumerate(lines) if '*dflash*' in l)
|
|
indent = len(lines[guard]) - len(lines[guard].lstrip())
|
|
end = next(i for i in range(guard + 1, len(lines))
|
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
|
block = chr(10).join(lines[guard:end])
|
|
assert '--spec-draft-model' in block, 'the dflash branch must pass the draft model'
|
|
assert sum(l.count('--spec-draft-model') for l in lines) == 1, 'only that branch may pass it'
|
|
print('draft-model wiring ok')
|
|
"
|
|
}
|
|
|
|
@test "the runtime is pinned to a commit and to gfx1201 alone" {
|
|
# A floating ref would silently move the service between engine versions,
|
|
# and an unpinned arch would build for a sibling GPU that loads and then
|
|
# segfaults on these cards.
|
|
grep -Eq '^LLAMA_REF="\$\{LLAMA_REF:-[0-9a-f]{40}\}"$' "$BUILD"
|
|
grep -Eq '^HIP_ARCH="\$\{HIP_ARCH:-gfx1201\}"$' "$BUILD"
|
|
grep -q 'GGML_HIP=ON' "$BUILD"
|
|
grep -q 'frank-medium.env' "$SERVE"
|
|
}
|
|
|
|
@test "the payload is fetched resumably and verified by size" {
|
|
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
|
|
grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH"
|
|
grep -q 'curl -fL -C -' "$FETCH"
|
|
grep -q 'verifying sizes' "$FETCH"
|
|
}
|
|
|
|
@test "both cards stay balanced and keep the operator's 1.5 GiB floor" {
|
|
local ctx parallel cache_k cache_v split_hi split_lo batch ubatch
|
|
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
|
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
|
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
|
|
batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
spec_type="$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")"
|
|
[ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$spec_type" ]
|
|
[ -n "$split_hi" ] && [ -n "$split_lo" ] && [ -n "$batch" ] && [ -n "$ubatch" ]
|
|
[ "$parallel" -ge 1 ]
|
|
[ "$batch" -ge "$ubatch" ]
|
|
[ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ]
|
|
# Per card, calibrated against rocm-smi at the 35:30 / ctx 262144 / batch
|
|
# 6144 x ubatch 512 / two slots / DFlash2 point, where the two cards
|
|
# measured 6.44 / 6.90 GiB free. The fit reproduces card 1 to 0.00 GiB
|
|
# and card 0 to 0.86 GiB at that point; it is a GATE, not a meter -- the
|
|
# deployed point is re-measured on the host after every split/ctx/batch
|
|
# move (the current one is 34:31 / 393216 / MTP, 2026-09-17).
|
|
#
|
|
# card 0 = its share of the weights and of the KV (qwen35 spends KV on
|
|
# its 16 full-attention layers only: 2 (K,V) x 4 heads x 256 dim
|
|
# x 2 B = 64 KiB per token at f16) plus ~1.4e6 B per ubatch unit
|
|
# of compute buffers
|
|
# card 1 = the same for its own share, plus the drafter. The drafter's
|
|
# charge depends on WHAT it is: DFlash2 is a sidecar file, so
|
|
# its weights (2.06e9 B) plus window-bounded KV (20480 B/pos x
|
|
# 2048) plus ~0.89 GiB of constant buffers; MTP's head lives in
|
|
# the target file (already charged above), so its cost is one
|
|
# nextn layer's KV (4096 B/position) plus small buffers. Its
|
|
# compute buffers are the heavier ones either way, because it
|
|
# holds the output layers and the drafter's graphs.
|
|
#
|
|
# BOTH ubatch coefficients are measured at the deployed shape, from the
|
|
# 1536 -> 512 move: 1.30 GiB on card 0 and 2.41 GiB on card 1. That
|
|
# asymmetry is why shrinking the batch moved the balance toward card 1 and
|
|
# cost card 0 a layer -- the split and the batch cannot be chosen apart.
|
|
#
|
|
# The layer it charges for is the AVERAGE (0.70 GiB: 31.46e9 of weights plus
|
|
# 16 GiB of KV over 65 layers). This stack is a hybrid, so real steps vary
|
|
# around that -- 0.43 GiB for a DeltaNet layer, 1.43 GiB for a full-attention
|
|
# one -- which makes the drift check below CONSERVATIVE: it overstates the gap
|
|
# a one-layer move creates, and so trips on moves a measurement might bless.
|
|
# That is the intent: moving the split means re-measuring, not re-arguing.
|
|
#
|
|
# The slack is the DEPLOYED gap, rounded up: at each shape the balance
|
|
# cannot be tuned finer than one layer, so the deployed split's gap IS
|
|
# what the check must accept, and a tighter slack would only reject it.
|
|
#
|
|
# What this model cannot see is CONTIGUITY -- the mechanism by which the
|
|
# DSpark-era 8192/2048 failed, a 2020 MiB pp buffer that found no room while
|
|
# every total looked fine. Total bytes are not the whole story for one big
|
|
# allocation.
|
|
local kv_pos=65536 draft_kv_pos=20480 mtp_kv_pos=4096 floor slack
|
|
local card_hi card_lo diff
|
|
local mmproj=931145888 # mmproj-BF16.gguf, autoloaded onto the primary device
|
|
floor=$(( 1536 * 1024 * 1024 )) # 1.5 GiB per card
|
|
# The current point is 34:31 at ctx 393216 with MTP: the fit gap is 1.44
|
|
# GiB, which is also the balance point at this pool, so the slack is that
|
|
# gap rounded up. (The 36:29 / 327680 / DFlash2 configuration's slack was
|
|
# 3225 MiB -- its 3.15 GiB gap was by construction, and is no longer the
|
|
# deployed shape.)
|
|
slack=$(( 1483 * 1024 * 1024 ))
|
|
card_hi=$(( MODEL_BYTES * split_hi / (split_hi + split_lo) \
|
|
+ kv_pos * ctx * split_hi / (split_hi + split_lo) \
|
|
+ 381898377 \
|
|
+ mmproj \
|
|
+ 1363 * 1024 * ubatch ))
|
|
if [[ "$spec_type" == *dflash* || "$spec_type" == *dspark* ]]; then
|
|
card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
|
|
+ DRAFT_BYTES \
|
|
+ kv_pos * ctx * split_lo / (split_hi + split_lo) \
|
|
+ draft_kv_pos * (ctx < 2048 ? ctx : 2048) \
|
|
+ 955046413 \
|
|
+ 2468 * 1024 * ubatch ))
|
|
else
|
|
# MTP: the head is in the target file (charged above); one nextn
|
|
# layer's KV plus ~0.2 GiB of buffers (estimate -- re-measure on the
|
|
# host after a drafter swap, as always).
|
|
card_lo=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
|
|
+ kv_pos * ctx * split_lo / (split_hi + split_lo) \
|
|
+ mtp_kv_pos * ctx \
|
|
+ 209715200 \
|
|
+ 2468 * 1024 * ubatch ))
|
|
fi
|
|
[ "$card_hi" -le $(( CARD_BYTES - floor )) ]
|
|
[ "$card_lo" -le $(( CARD_BYTES - floor )) ]
|
|
diff=$(( card_hi - card_lo )); [ "$diff" -ge 0 ] || diff=$(( -diff ))
|
|
[ "$diff" -le "$slack" ]
|
|
# ...and the window may not wander far past what the checkpoint was
|
|
# trained for (262144; the GGUF carries no rope scaling, so the top 131k
|
|
# of the 393216 pool extrapolates on the base rope).
|
|
[ "$ctx" -le 393216 ]
|
|
}
|
|
|
|
@test "vision is enabled: the BF16 projector loads with the model" {
|
|
grep -q '^NO_MMPROJ=0$' "$ENV_FILE"
|
|
grep -q '^MM_PROJ_FILE=mmproj-BF16.gguf$' "$ENV_FILE"
|
|
# Explicit flag, not autoload: this build starts text-only without it,
|
|
# and refuses to start without the projector file, so a missing file is
|
|
# a deploy error, not a silent text-only server.
|
|
grep -q -- '--mmproj "$MM_PROJ"' "$SERVE"
|
|
grep -q 'vision is enabled but the projector is missing' "$SERVE"
|
|
# And the download script fetches and byte-verifies it.
|
|
grep -q '^MM_PROJ_SIZE=931145888$' "$FETCH"
|
|
grep -q -- 'check "$MODEL_DIR/$MM_PROJ_FILE" "$MM_PROJ_SIZE"' "$FETCH"
|
|
}
|
|
|
|
@test "the fleet endpoint and the predecessor's sampler defaults survive" {
|
|
grep -q '^ALIAS=frank-medium$' "$ENV_FILE"
|
|
grep -q '^LISTEN_PORT=7770$' "$ENV_FILE"
|
|
grep -q -- '--alias "$ALIAS"' "$SERVE"
|
|
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
|
|
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
|
|
# Prometheus scrapes this endpoint for LLM fleet metrics.
|
|
grep -q -- '--metrics' "$SERVE"
|
|
# Requests that send no sampler fields get these instead of an error.
|
|
grep -q '^TEMPERATURE=0.7$' "$ENV_FILE"
|
|
grep -q '^TOP_P=0.95$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the deploy handler installs the root unit and refuses a missing payload" {
|
|
grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-medium.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl restart frank-medium.service' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
|
grep -q 'build-llama-rdna4.sh' "$REMOTE_DEPLOY"
|
|
grep -q 'download-frank-medium-gguf.sh' "$REMOTE_DEPLOY"
|
|
grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deploy handler retires the vLLM quadlets and frees the port they held" {
|
|
grep -q 'retire_vllm_medium_quadlets' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
# No image left to tag: the runtime is a native binary now.
|
|
# ...and no image work or quadlet install of the predecessor survives.
|
|
! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
|
|
! grep -q 'podman tag docker.io/stilldeadcode' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the retired vLLM deployment is gone from the repo, not merely unmapped" {
|
|
[ ! -d "$REPO_ROOT/dankserver/frank-medium" ]
|
|
[ ! -f "$REPO_ROOT/tests/frank_medium_quadlet.bats" ]
|
|
# ...and the archive is present, so the two assertions above cannot pass
|
|
# because a path went missing rather than because it stayed missing.
|
|
[ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container" ]
|
|
[ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh" ]
|
|
}
|
|
|
|
@test "the serve script refuses to start on an incomplete host" {
|
|
grep -q 'frank-medium: missing\|missing executable runtime\|missing or unreadable' "$SERVE"
|
|
grep -q 'missing or unreadable target model' "$SERVE"
|
|
grep -q 'missing or unreadable draft model' "$SERVE"
|
|
}
|