Frank
b6d4914fcc
DSpark declares no sliding window, so its draft context -- built with n_ctx = llama_n_ctx(ctx_tgt) -- caches the whole target window: 5 layers x 2 x 8 kv heads x 128 dim x 2 B = 20 KiB/position, i.e. 5.0 GiB at 262144, where DFlash2's sliding_window 2048 cost ~40 MiB. At a 1:1 split that put ROCm1 at 33.9 GB of 34.2 GB and the first draft graph capture died inside common_speculative_impl_draft_dflash::process with "ROCm error: out of memory" -- after the model had loaded and /health had answered, which is why the deploy looked like a crash-loop rather than a sizing failure. Handing three target layers to ROCm0 restores headroom on both cards at the full window. Measured at 262144: 28.52 GB / 30.71 GB, i.e. 5.30 GiB and 3.26 GiB free (was 2.7 GB free on the draft-carrying card with 1:1 + DFlash2). The unit seam now models the busier card as a skewed share plus the drafter's full-context KV, calibrated to the rocm-smi figure. AGENTS.md gains the durable lesson: read sliding_window out of the drafter's GGUF header before sizing a spec-decode deployment.
220 lines
11 KiB
Bash
Executable File
220 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-medium mainline-llama.cpp root unit on dankserver.
|
|
# frank-medium became a native llama.cpp service on 2026-09-16, replacing the
|
|
# vllm-radiance replica pair + routing proxy that served this endpoint for a
|
|
# day. What this seam pins is what the flags alone do not say: boot persistence
|
|
# for a NATIVE runtime (no generated quadlet is involved), that BOTH cards work
|
|
# on ONE model through a layer split with no collective in the path, that the
|
|
# DSpark drafter is wired with a depth its block size actually allows, that the
|
|
# runtime and payload are pinned rather than floated, and that one slot's KV
|
|
# still fits a 32 GiB card beside the weights it holds.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-medium.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-medium.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-medium.env"
|
|
BUILD="$REPO_ROOT/scripts/build-llama-rdna4.sh"
|
|
FETCH="$REPO_ROOT/scripts/download-frank-medium-gguf.sh"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
|
|
# Measured constants (rocm-smi / HF blob sizes), used by the fit arithmetic.
|
|
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it
|
|
MODEL_BYTES=31457990112 # UD-Q8_K_XL target
|
|
DRAFT_BYTES=1359284737 # DSpark Q8_0 drafter
|
|
|
|
@test "frank-medium's unit, serve script, env and its build/fetch scripts exist" {
|
|
[ -f "$UNIT" ]
|
|
[ -f "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
[ -f "$BUILD" ]
|
|
[ -f "$FETCH" ]
|
|
}
|
|
|
|
@test "frank-medium is boot-persistent, self-healing and unprivileged" {
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "frank-medium runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-medium.sh$' "$UNIT"
|
|
grep -q 'FRANK_MEDIUM_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-medium.env' "$UNIT"
|
|
}
|
|
|
|
@test "both R9700s serve ONE model, layer-split inside one process" {
|
|
grep -q '^SPLIT_MODE=layer$' "$ENV_FILE"
|
|
grep -q '^DEVICES=ROCm0,ROCm1$' "$ENV_FILE"
|
|
grep -q '^TENSOR_SPLIT=38,27$' "$ENV_FILE"
|
|
grep -q -- '--split-mode "$SPLIT_MODE"' "$SERVE"
|
|
grep -q -- '--device "$DEVICES"' "$SERVE"
|
|
grep -q -- '--tensor-split "$TENSOR_SPLIT"' "$SERVE"
|
|
# Tensor split is the one shape this host cannot run: it needs a cross-card
|
|
# collective, and the second card cannot dispatch one (see the env file).
|
|
! grep -q '^SPLIT_MODE=tensor$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the iGPU is hidden so no layer can land on system RAM" {
|
|
grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
|
|
grep -q '^export HIP_VISIBLE_DEVICES$' "$SERVE"
|
|
}
|
|
|
|
@test "the drafter runs in a chain with the draftless n-gram matcher" {
|
|
# The chain is the measured winner (see the runbook's A/B): free where the
|
|
# matcher cannot match, ~2.6x the drafter alone on prompt-echo traffic. A
|
|
# bare `draft-dflash` here would silently give that up.
|
|
grep -qE '^SPEC_TYPE=draft-dspark,ngram-mod$' "$ENV_FILE"
|
|
grep -q '^DRAFT_MODEL_FILE=Qwen3\.8-27B-DSpark-Q8_0\.gguf$' "$ENV_FILE"
|
|
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
|
grep -q -- '--spec-draft-model "$DRAFT_MODEL"' "$SERVE"
|
|
# ngram-mod's own window -- and only when the type asks for it, so a pure
|
|
# DFlash or MTP configuration is not quietly handed n-gram parameters.
|
|
grep -q '^SPEC_NGRAM_N_MATCH=24$' "$ENV_FILE"
|
|
grep -q '^SPEC_NGRAM_N_MIN=48$' "$ENV_FILE"
|
|
grep -q '^SPEC_NGRAM_N_MAX=64$' "$ENV_FILE"
|
|
grep -q -- '--spec-ngram-mod-n-match "$SPEC_NGRAM_N_MATCH"' "$SERVE"
|
|
grep -q -- '--spec-ngram-mod-n-min "$SPEC_NGRAM_N_MIN"' "$SERVE"
|
|
grep -q -- '--spec-ngram-mod-n-max "$SPEC_NGRAM_N_MAX"' "$SERVE"
|
|
python3 -c "
|
|
lines = open('$SERVE').read().splitlines()
|
|
guard = next(i for i, l in enumerate(lines) if '*ngram*' in l)
|
|
indent = len(lines[guard]) - len(lines[guard].lstrip())
|
|
end = next(i for i in range(guard + 1, len(lines))
|
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
|
assert '--spec-ngram-mod-n-match' in chr(10).join(lines[guard:end]), 'ngram flags must sit in the ngram branch'
|
|
"
|
|
# DSpark drafts into a block of 7, so at most 6 draft tokens per
|
|
# verification step -- llama.cpp asserts n_max <= block_size - 1.
|
|
local n_max
|
|
n_max="$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
[ -n "$n_max" ] && [ "$n_max" -ge 1 ] && [ "$n_max" -le 6 ]
|
|
}
|
|
|
|
@test "a draft MODEL is passed only for DFlash types, never for MTP" {
|
|
# MTP builds its context against the TARGET model (its head lives in that
|
|
# GGUF). Passing --spec-draft-model alongside it points the draft context at
|
|
# the drafter's file, which has no MTP layers, and the server dies at load:
|
|
# context type MTP requested but model doesn't contain MTP layers
|
|
# failed to create MTP context
|
|
# So the flag must appear exactly once, inside the dflash branch.
|
|
python3 -c "
|
|
lines = open('$SERVE').read().splitlines()
|
|
guard = next(i for i, l in enumerate(lines) if '*dflash*' in l)
|
|
indent = len(lines[guard]) - len(lines[guard].lstrip())
|
|
end = next(i for i in range(guard + 1, len(lines))
|
|
if lines[i].strip() == 'fi' and len(lines[i]) - len(lines[i].lstrip()) == indent)
|
|
block = chr(10).join(lines[guard:end])
|
|
assert '--spec-draft-model' in block, 'the dflash branch must pass the draft model'
|
|
assert sum(l.count('--spec-draft-model') for l in lines) == 1, 'only that branch may pass it'
|
|
print('draft-model wiring ok')
|
|
"
|
|
}
|
|
|
|
@test "the runtime is pinned to a commit and to gfx1201 alone" {
|
|
# A floating ref would silently move the service between engine versions,
|
|
# and an unpinned arch would build for a sibling GPU that loads and then
|
|
# segfaults on these cards.
|
|
grep -Eq '^LLAMA_REF="\$\{LLAMA_REF:-[0-9a-f]{40}\}"$' "$BUILD"
|
|
grep -Eq '^HIP_ARCH="\$\{HIP_ARCH:-gfx1201\}"$' "$BUILD"
|
|
grep -q 'GGML_HIP=ON' "$BUILD"
|
|
grep -q 'frank-medium.env' "$SERVE"
|
|
}
|
|
|
|
@test "the payload is fetched resumably and verified by size" {
|
|
grep -q 'UD-Q8_K_XL\.gguf' "$FETCH"
|
|
grep -q 'Qwen3\.8-27B-DSpark-Q8_0\.gguf' "$FETCH"
|
|
grep -q 'curl -fL -C -' "$FETCH"
|
|
grep -q 'verifying sizes' "$FETCH"
|
|
}
|
|
|
|
@test "one slot's KV still fits a card beside the weights that card holds" {
|
|
local ctx parallel cache_k cache_v split_hi split_lo
|
|
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
|
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
|
split_hi="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
split_lo="$(sed -nE 's/^TENSOR_SPLIT=([0-9]+),([0-9]+)$/\2/p' "$ENV_FILE")"
|
|
[ -n "$ctx" ] && [ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ]
|
|
[ -n "$split_hi" ] && [ -n "$split_lo" ]
|
|
[ "$parallel" -ge 1 ]
|
|
[ "$cache_k" = "f16" ] && [ "$cache_v" = "f16" ]
|
|
# qwen35 spends KV on its 16 full-attention layers only: 2 (K,V) x 4 heads
|
|
# x 256 dim x 2 B = 64 KiB per token at f16, carried by whichever card a
|
|
# layer split puts those layers on. The busier card is the SECOND one: its
|
|
# share of the weights and KV, plus the WHOLE drafter -- whose KV spans the
|
|
# full context, because DSpark declares no sliding window (5 x 2 x 8 x 128
|
|
# x 2 B = 20480 B per position) -- plus the loader's compute buffers.
|
|
#
|
|
# This model is deliberately calibrated against rocm-smi: at 262144 / 38:27
|
|
# it predicts 30.69e9 B against the 30.71e9 B measured on the busier card
|
|
# (2026-09-16). Keep it that way -- a model that is pessimistic beyond a few
|
|
# GiB would reject a context that fits.
|
|
local per_card bufs
|
|
bufs=$(( 3584 * 1024 * 1024 ))
|
|
per_card=$(( MODEL_BYTES * split_lo / (split_hi + split_lo) \
|
|
+ DRAFT_BYTES \
|
|
+ (ctx * 65536) * split_lo / (split_hi + split_lo) \
|
|
+ ctx * 20480 \
|
|
+ bufs ))
|
|
[ "$per_card" -le "$CARD_BYTES" ]
|
|
# ...and the window may not exceed what the checkpoint was trained for.
|
|
[ "$ctx" -le 262144 ]
|
|
}
|
|
|
|
@test "frank-medium is text only: the vision tower stays on frank-small" {
|
|
grep -q '^NO_MMPROJ=1$' "$ENV_FILE"
|
|
grep -q -- '--no-mmproj' "$SERVE"
|
|
}
|
|
|
|
@test "the fleet endpoint and the predecessor's sampler defaults survive" {
|
|
grep -q '^ALIAS=frank-medium$' "$ENV_FILE"
|
|
grep -q '^LISTEN_PORT=7770$' "$ENV_FILE"
|
|
grep -q -- '--alias "$ALIAS"' "$SERVE"
|
|
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
|
|
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
|
|
# Prometheus scrapes this endpoint for LLM fleet metrics.
|
|
grep -q -- '--metrics' "$SERVE"
|
|
# Requests that send no sampler fields get these instead of an error.
|
|
grep -q '^TEMPERATURE=0.7$' "$ENV_FILE"
|
|
grep -q '^TOP_P=0.95$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the deploy handler installs the root unit and refuses a missing payload" {
|
|
grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-medium.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl restart frank-medium.service' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
|
grep -q 'build-llama-rdna4.sh' "$REMOTE_DEPLOY"
|
|
grep -q 'download-frank-medium-gguf.sh' "$REMOTE_DEPLOY"
|
|
grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deploy handler retires the vLLM quadlets and frees the port they held" {
|
|
grep -q 'retire_vllm_medium_quadlets' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
# No image left to tag: the runtime is a native binary now.
|
|
# ...and no image work or quadlet install of the predecessor survives.
|
|
! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
|
|
! grep -q 'podman tag docker.io/stilldeadcode' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the retired vLLM deployment is gone from the repo, not merely unmapped" {
|
|
[ ! -d "$REPO_ROOT/dankserver/frank-medium" ]
|
|
[ ! -f "$REPO_ROOT/tests/frank_medium_quadlet.bats" ]
|
|
# ...and the archive is present, so the two assertions above cannot pass
|
|
# because a path went missing rather than because it stayed missing.
|
|
[ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/frank-medium-a.container" ]
|
|
[ -f "$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance/radiance-entrypoint.sh" ]
|
|
}
|
|
|
|
@test "the serve script refuses to start on an incomplete host" {
|
|
grep -q 'frank-medium: missing\|missing executable runtime\|missing or unreadable' "$SERVE"
|
|
grep -q 'missing or unreadable target model' "$SERVE"
|
|
grep -q 'missing or unreadable draft model' "$SERVE"
|
|
}
|