Frank
a4f4429a99
ngram-mod is draftless (added to the spec config vector unconditionally, while every model-backed drafter needs ctx_dft), so unlike MTP it can share the process with DFlash2. Chained at the fleet's 48/64 window, over three shapes: prose 1k/512 dflash 43.77 | ngram 17.72 (0 drafts) | chain 43.65 echo 1.5k/400 dflash 76.82 | ngram 102.50 | chain 200.58 structured/400 dflash 91.08 | ngram 17.80 (0 drafts) | chain 91.22 identity dflash 70.37 | ngram 17.77 | chain 70.40 On prose the chain is byte-identical to the drafter alone (same 119 drafts/49 accepts) -- that is what makes it a safe default -- and on prompt-echo traffic it is 2.6x. VRAM is unchanged: the matcher is a 64 MiB host-RAM hash table. The window sweep says keep the long one: 48/64 beats 16/32 (161.0) and 8/16 (113.8) on echo, and shortening it costs prose (43.9 -> 36.0) because the matcher then proposes on ordinary fragments and the rejections still have to be verified. That is the 'ngram hurts' result this fleet hit twice before. Two documentation traps recorded with it: ngram-mod prints NO startup banner in this build (its drafts are only visible behaviourally), and the harness's 'structured' shape is a negative control because its lines differ.
210 lines
10 KiB
Bash
Executable File
210 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-medium mainline-llama.cpp root unit on dankserver.
|
|
# frank-medium became a native llama.cpp service on 2026-09-16, replacing the
|
|
# vllm-radiance replica pair + routing proxy that served this endpoint for a
|
|
# day. What this seam pins is what the flags alone do not say: boot persistence
|
|
# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work
|
|
# on ONE model through a layer split with no collective in the path, that the
|
|
# DFlash2 drafter is wired with a depth its block size actually allows, that the
|
|
# runtime and payload are pinned rather than floated, and that one slot's KV
|
|
# still fits a 32 GiB card beside the weights it holds.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-medium.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-medium.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-medium.env"
|
|
BUILD="$REPO_ROOT/scripts/build-llama-rdna4.sh"
|
|
FETCH="$REPO_ROOT/scripts/download-frank-medium-gguf.sh"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
|
|
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
|
|
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
|
|
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
|
|
DRAFT_BYTES=2056414816 # DFlash2 Q8_0 drafter
|
|
|
|
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
|
|
[ -f "$UNIT" ]
|
|
[ -f "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
[ -f "$BUILD" ]
|
|
[ -f "$FETCH" ]
|
|
}
|
|
|
|
@test "frank-medium is boot-persistent, self-healing and unprivileged" {
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "frank-medium runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh$' "$UNIT"
|
|
grep -q 'FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env' "$UNIT"
|
|
}
|
|
|
|
@test "both R9700s serve ONE model, layer-split inside one process" {
|
|
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
|
|
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
|
|
grep -q '^TENSOR_SPLIT=1,1$' "$ENV_FILE"
|
|
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
|
|
grep -q -- '--device "$DEVICES"' "$SERVE"
|
|
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
|
|
# Tensor split is the one shape this host cannot run: it needs a cross-card
|
|
# collective, and the second card cannot dispatch one (see the env file).
|
|
! grep -q '^SPLIT_MODE=tensor$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the iGPU is hidden so no layer can land on system RAM" {
|
|
grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
|
|
grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
|
|
}
|
|
|
|
@test "the drafter runs in a chain with the draftless n-gram matcher" {
|
|
# The chain is the measured winner (see the runbook's A/B): free where the
|
|
# matcher cannot match, ~2.6x the drafter alone on prompt-echo traffic. A
|
|
# bare `draft-dflash` here would silently give that up.
|
|
grep -qE '^SPEC_TYPE=draft-dflash,ngram-mod$' "$ENV_FILE"
|
|
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DFlash2-Q8_0\.gguf$' "$ENV_FILE"
|
|
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
|
grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE"
|
|
# ngram-mod's own window -- and only when the type asks for it, so a pure
|
|
# DFlash or MTP configuration is not quietly handed n-gram parameters.
|
|
grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
|
|
grep -q '^SPEC_NGRAM_N_MIN=48$' "$ENV_FILE"
|
|
grep -q '^SPEC_NGRAM_N_MAX=64$' "$ENV_FILE"
|
|
grep -q -- '--spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH"' "$SERVE"
|
|
grep -q -- '--spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN"' "$SERVE"
|
|
grep -q -- '--spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX"' "$SERVE"
|
|
python3 -c "
|
|
lines = open('$SERVE').read().splitlines()
|
|
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
|
|
indent = len(lines[guard]) - len(lines[guard].lstrip())
|
|
end = next(i for i in range(guard + 1, len(lines))
|
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
|
assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch'
|
|
"
|
|
# DFlash2 drafts into a block of 8, so at most 7 draft tokens per
|
|
# verification step -- llama.cpp asserts n_max <= block_size - 1.
|
|
local n_max
|
|
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
[ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 7 ]
|
|
}
|
|
|
|
@test "a draft MODEL is passed only for DFlash types, never for MTP" {
|
|
# MTP builds its context against the TARGET model (its head lives in that
|
|
# GGUF). Passing --spec-draft-model alongside it points the draft context at
|
|
# the drafter's file, which has no MTP layers, and the server dies at load:
|
|
# context type MTP requested but model doesn't contain MTP layers
|
|
# failed to create MTP context
|
|
# So the flag must appear exactly once, inside the dflash branch.
|
|
python3 -c "
|
|
lines = open('$SERVE').read().splitlines()
|
|
guard = next(i for i, l in enumerate(lines) if '*dflash*' in l)
|
|
indent = len(lines[guard]) - len(lines[guard].lstrip())
|
|
end = next(i for i in range(guard + 1, len(lines))
|
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
|
block = chr(10).join(lines[guard:end])
|
|
assert '--spec-draft-model' in block, 'the dflash branch must pass the draft model'
|
|
assert sum(l.count('--spec-draft-model') for l in lines) == 1, 'only that branch may pass it'
|
|
print('draft-model wiring ok')
|
|
"
|
|
}
|
|
|
|
@test "the runtime is pinned to a commit and to gfx1201 alone" {
|
|
# A floating ref would silently move the service between engine versions,
|
|
# and an unpinned arch would build for a sibling GPU that loads and then
|
|
# segfaults on these cards.
|
|
grep -Eq '^LLAMA_REF="\$\{LLAMA_REF:-[0-9a-f]{40}\}"$' "$BUILD"
|
|
grep -Eq '^HIP_ARCH="\$\{HIP_ARCH:-gfx1201\}"$' "$BUILD"
|
|
grep -q 'GGML_HIP=ON' "$BUILD"
|
|
grep -q 'frank-medium.env' "$SERVE"
|
|
}
|
|
|
|
@test "the payload is fetched resumably and verified by size" {
|
|
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
|
|
grep -q 'Qwen3\.8-27B-DFlash2-Q8_0\.gguf' "$FETCH"
|
|
grep -q 'curl -fL -C -' "$FETCH"
|
|
grep -q 'verifying sizes' "$FETCH"
|
|
}
|
|
|
|
@test "one slot's KV still fits a card beside the weights that card holds" {
|
|
local ctx parallel cache_k cache_v
|
|
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
|
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
|
[ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ]
|
|
[ "$parallel" -ge 1 ]
|
|
[ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ]
|
|
# qwen35 spends KV on its 16 full-attention layers only: 2 (K,V) x 4 heads
|
|
# x 256 dim x 2 B = 64 KiB per token at f16. A layer split puts those
|
|
# layers (and their KV) on both cards, so the worst card carries half the
|
|
# weights, half the KV, the drafter and ~2 GiB of compute buffers.
|
|
#
|
|
# This model is deliberately calibrated against rocm-smi: at 262144 it
|
|
# predicts 28.53e9 B against the 29.14e9 B measured on the busier card
|
|
# (2026-09-16), i.e. ~2 % optimistic. Keep it that way -- a model that is
|
|
# pessimistic beyond a few GiB would reject a context that fits.
|
|
local per_card
|
|
per_card=$(( MODEL_BYTES / 2 + DRAFT_BYTES + (ctx * 65536) / 2 + 2 * 1024 * 1024 * 1024 ))
|
|
[ "$per_card" -le "$CARD_BYTES" ]
|
|
# ...and the window may not exceed what the checkpoint was trained for.
|
|
[ "$ctx" -le 262144 ]
|
|
}
|
|
|
|
@test "frank-medium is text only: the vision tower stays on frank-small" {
|
|
grep -q '^NO_MMPROJ=1$' "$ENV_FILE"
|
|
grep -q -- '--no-mmproj' "$SERVE"
|
|
}
|
|
|
|
@test "the fleet endpoint and the predecessor's sampler defaults survive" {
|
|
grep -q '^ALIAS=frank-medium$' "$ENV_FILE"
|
|
grep -q '^LISTEN_PORT=7770$' "$ENV_FILE"
|
|
grep -q -- '--alias "$ALIAS"' "$SERVE"
|
|
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
|
|
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
|
|
# Prometheus scrapes this endpoint for LLM fleet metrics.
|
|
grep -q -- '--metrics' "$SERVE"
|
|
# Requests that send no sampler fields get these instead of an error.
|
|
grep -q '^TEMPERATURE=0.7$' "$ENV_FILE"
|
|
grep -q '^TOP_P=0.95$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the deploy handler installs the root unit and refuses a missing payload" {
|
|
grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-medium.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl restart frank-medium.service' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
|
grep -q 'build-llama-rdna4.sh' "$REMOTE_DEPLOY"
|
|
grep -q 'download-frank-medium-gguf.sh' "$REMOTE_DEPLOY"
|
|
grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deploy handler retires the vLLM quadlets and frees the port they held" {
|
|
grep -q 'retire_vllm_medium_quadlets' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
# No image left to tag: the runtime is a native binary now.
|
|
# ...and no image work or quadlet install of the predecessor survives.
|
|
! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
|
|
! grep -q 'podman tag docker.io/stilldeadcode' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the retired vLLM deployment is gone from the repo, not merely unmapped" {
|
|
[ ! -d "$REPO_ROOT/dankserver/frank-medium" ]
|
|
[ ! -f "$REPO_ROOT/tests/frank_medium_quadlet.bats" ]
|
|
# ...and the archive is present, so the two assertions above cannot pass
|
|
# because a path went missing rather than because it stayed missing.
|
|
[ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container" ]
|
|
[ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh" ]
|
|
}
|
|
|
|
@test "the serve script refuses to start on an incomplete host" {
|
|
grep -q 'frank-medium: missing\|missing executable runtime\|missing or unreadable' "$SERVE"
|
|
grep -q 'missing or unreadable target model' "$SERVE"
|
|
grep -q 'missing or unreadable draft model' "$SERVE"
|
|
}
|