Frank
d389d09d97
The entrypoint now reads the model's weight quantization from config.json and serves the plain FP8 path (no MXFP4 patches, --quantization=fp8, MTP self-speculation, ROCM_AITER unified attention with the R4D all-reduce) or the MXFP4 patch-and-serve path. A model swap is now: change SNAP (+ SPEC_METHOD/SPEC), restart. - radiance-entrypoint.sh: quantization detection, gated MXFP4 env vars and patch block, --quantization fp8 flag - quadlet: SNAP -> Qwen3.8-27B-Uncensored-FP8, SPEC_METHOD=mtp, SPEC=8, KV_MEM=0 (profile the shape, the documented plain-FP8 default) - deploy-remote.sh: model-aware gates (read SNAP/SPEC_METHOD from the quadlet; drafter gate only for dflash, r4d.so gate only for mxfp4) - tests: quantization-aware drafter/spec and KV-pool tests - hf-fetch.sh: generic, resumable, verified, idempotent HF fetch tool - frank-medium-fp8.manifest: 17 files (10 config + 7 shards) with sizes and LFS sha256
214 lines
10 KiB
Bash
Executable File
214 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-medium vllm-radiance container on dankserver.
|
|
# frank-medium became a CONTAINER again on 2026-09-20, when the board swap put
|
|
# both R9700s on CPU root ports and a measured three-arm comparison retired the
|
|
# mainline-llama.cpp layer split (scripts/.AGENTTRASH/20260920-llama-layersplit/).
|
|
# What this seam pins is what the flags alone do not say: boot persistence for a
|
|
# GENERATED unit (this host's v1 quadlet generator cannot enable units, so the
|
|
# [Install] section is the only thing that survives a reboot), that BOTH cards
|
|
# work on ONE model through tensor parallelism — the thing the pre-2026-09-20
|
|
# hardware could not do — the KV pin's coupling to the batch shape it was
|
|
# calibrated at, and that the pool still holds one full-window request.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container"
|
|
ENTRY="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh"
|
|
HEALTH="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
LLAMA_TRASH="$REPO_ROOT/scripts/.AGENTTRASH/20260920-llama-layersplit"
|
|
REPLICA_TRASH="$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance"
|
|
|
|
# Measured constants, used by the fit arithmetic.
|
|
CARD_BYTES=34208743424 # one R9700 as rocm-smi reports it (32624 MiB)
|
|
KV_BYTES_PER_TOKEN=70656 # --kv-cache-memory 18563072000 => "262,720 tokens"
|
|
env_value() {
|
|
sed -nE "s/^Environment=$1=(-?[0-9]+)$/\1/p" "$UNIT" | head -n1
|
|
}
|
|
# GPU_UTIL is a fraction, not a count; the integer reader would drop it.
|
|
env_ratio() {
|
|
sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1
|
|
}
|
|
# SPEC_METHOD is a string (dflash|mtp), not a count; the integer reader drops it.
|
|
env_str() {
|
|
sed -nE "s/^Environment=$1=(.+)$/\1/p" "$UNIT" | head -n1
|
|
}
|
|
|
|
@test "frank-medium's unit, entrypoint and healthcheck exist" {
|
|
[ -f "$UNIT" ]
|
|
[ -x "$ENTRY" ]
|
|
[ -f "$HEALTH" ]
|
|
}
|
|
|
|
@test "frank-medium is boot-persistent and self-healing" {
|
|
# Generated units cannot be enabled on this host, so WantedBy in the file is
|
|
# the whole boot story (AGENTS.md: Boot Persistence & Self-Healing).
|
|
grep -q '^WantedBy=default.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
# First start compiles Triton/inductor kernels before the engine loads ~9.5
|
|
# GiB per card; a shorter timeout kills a healthy boot.
|
|
grep -q '^TimeoutStartSec=1800$' "$UNIT"
|
|
grep -q '^HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py$' "$UNIT"
|
|
grep -q '^HealthStartPeriod=1800s$' "$UNIT"
|
|
# Boots must not depend on the registry.
|
|
grep -q '^Pull=never$' "$UNIT"
|
|
}
|
|
|
|
@test "both R9700s serve ONE model as one tensor-parallel process" {
|
|
grep -q '^Environment=TP=2$' "$UNIT"
|
|
grep -q '^Environment=ROCR_VISIBLE_DEVICES=0,1$' "$UNIT"
|
|
grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT"
|
|
grep -q -- '--tensor-parallel-size "$TP"' "$ENTRY"
|
|
# The iGPU (index 2, 2 GiB) must stay out of the visible set: a third rank
|
|
# would take layers onto system RAM.
|
|
! grep -q '^Environment=.*VISIBLE_DEVICES=0,1,2' "$UNIT"
|
|
# TP needs the all-reduce to actually run. R4D is the collective-free path
|
|
# this host measured; the entrypoint must still select it, and the RCCL
|
|
# fallback must not be re-disabled by the pre-board-swap workarounds.
|
|
grep -q 'RADIANCE_USE_R4D_AR' "$ENTRY"
|
|
grep -q -- '--attention-backend "$ATTN"' "$ENTRY"
|
|
! grep -q 'NCCL_P2P_DISABLE' "$UNIT"
|
|
! grep -q 'RCCL_MSCCL_ENABLE' "$UNIT"
|
|
}
|
|
|
|
@test "HIP never reads one of these tunables as its own setting" {
|
|
# HIP reads bare TEMP, HOST and GPU_MAX_HEAP_SIZE (2026-08-13 incident:
|
|
# TEMP=1.0 sent the JIT cache to path "1.0" and segfaulted every load).
|
|
# Serve scripts export env files with `set -a`; a container inherits the
|
|
# same hazard from Environment=.
|
|
! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
|
|
}
|
|
|
|
@test "the speculative drafter matches the serving shape" {
|
|
local spec_method spec maxseqs chunk
|
|
spec_method="$(env_str SPEC_METHOD)"
|
|
spec="$(env_value SPEC)"
|
|
maxseqs="$(env_value MAXSEQS)"
|
|
chunk="$(env_value CHUNK)"
|
|
[ -n "$spec_method" ] && [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
|
|
# The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk
|
|
# larger than this row's silently changes which all-reduce path runs.
|
|
[ "$chunk" -le 8192 ]
|
|
case "$spec_method" in
|
|
dflash)
|
|
# DFlash2 drafts at most 7 tokens per step; deeper is a different
|
|
# drafter's contract and costs acceptance, not just VRAM.
|
|
[ "$spec" -eq 7 ]
|
|
# The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the
|
|
# shipped default covers 64. Exceeding it without the entrypoint's >8
|
|
# bump is a launch failure at the worst moment (first concurrent
|
|
# request).
|
|
[ $(( maxseqs * (spec + 1) )) -le 64 ]
|
|
grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
|
|
;;
|
|
mtp)
|
|
# MTP self-speculation at depth 8 (the documented plain-FP8 path; the
|
|
# model's own single-layer MTP head, applied autoregressively).
|
|
[ "$spec" -eq 8 ]
|
|
;;
|
|
*)
|
|
false "unknown SPEC_METHOD: $spec_method"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
@test "the KV pool holds a full window and fits beside the weights" {
|
|
local kv_mem maxseqs chunk maxlen spec
|
|
kv_mem="$(env_value KV_MEM)"
|
|
maxseqs="$(env_value MAXSEQS)"
|
|
chunk="$(env_value CHUNK)"
|
|
maxlen="$(env_value MAXLEN)"
|
|
spec="$(env_value SPEC)"
|
|
[ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
|
|
# KV_MEM=0 = profile the shape (the documented plain-FP8 path's default;
|
|
# the upstream docker-compose leaves the pin commented out). The pool is
|
|
# auto-sized to fit, so there is no pin to verify — only that the shape
|
|
# is set and the model's weights fit beside an auto-sized pool.
|
|
if [ "$kv_mem" -eq 0 ]; then
|
|
[ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$maxlen" -eq 262144 ]
|
|
return 0
|
|
fi
|
|
# Pinned pool: the pin must hold at least ONE full-window request or the
|
|
# endpoint refuses its own --max-model-len, and it must fit on one card
|
|
# beside the weights.
|
|
grep -q 'kv-profiles.tsv' "$UNIT"
|
|
[ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ]
|
|
# Weights per card: MXFP4 ~9.5 GiB (19 GB / 2), FP8 ~15.5 GiB (30.9 GB / 2).
|
|
local weights util util_pct floor
|
|
case "$(env_str SPEC_METHOD)" in
|
|
dflash) weights=10200547328 ;;
|
|
*) weights=16646128640 ;;
|
|
esac
|
|
util="$(env_ratio GPU_UTIL)"
|
|
[ -n "$util" ]
|
|
util_pct="${util#0.}"
|
|
floor=$(( 1024 * 1024 * 1024 )) # 1 GiB per card
|
|
[ $(( weights + kv_mem / 2 )) -le $(( CARD_BYTES * util_pct / 100 - floor )) ]
|
|
}
|
|
|
|
@test "the fleet endpoint contract survives the runtime swap" {
|
|
grep -q '^Environment=PORT=7770$' "$UNIT"
|
|
grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT"
|
|
grep -q -- '--port "$PORT"' "$ENTRY"
|
|
grep -q -- '--served-model-name $SERVED_NAMES' "$ENTRY"
|
|
# Host network: vLLM binds the port itself, so no PublishPort (and no
|
|
# pasta bind, which is what made the retired proxy's port replacement
|
|
# fail at container create).
|
|
grep -q -- '--network=host' "$UNIT"
|
|
! grep -q '^PublishPort=' "$UNIT"
|
|
# Tools and reasoning are part of the harness contract; both parsers must
|
|
# be selected, and thinking must land in its own field.
|
|
grep -q -- '--enable-auto-tool-choice' "$ENTRY"
|
|
grep -q -- '--tool-call-parser qwen3_coder' "$ENTRY"
|
|
grep -q -- '--reasoning-parser qwen3' "$ENTRY"
|
|
grep -q -- '--enable-prefix-caching' "$ENTRY"
|
|
# Requests that send no sampler fields get the predecessor's defaults.
|
|
grep -q '"temperature":0.7' "$ENTRY"
|
|
grep -q '"top_p":0.95' "$ENTRY"
|
|
}
|
|
|
|
@test "the healthcheck proves the engine serves, not merely that a process lives" {
|
|
grep -q '"/health"' "$HEALTH" || grep -q '/health' "$HEALTH"
|
|
grep -q 'status == 200' "$HEALTH"
|
|
# PORT-aware, and it must not hard-code a replica port.
|
|
grep -q 'os.environ.get("PORT"' "$HEALTH"
|
|
! grep -qE '7771|7772' "$HEALTH"
|
|
}
|
|
|
|
@test "the deploy handler gates on host assets and installs the quadlet" {
|
|
grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
|
|
grep -q 'install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_container_healthy frank-medium' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
|
|
# The image is only pulled when absent, and re-tagged to the name the unit
|
|
# actually runs (the quadlet path cannot resolve registry refs at create).
|
|
grep -q 'podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
|
|
grep -q 'podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
|
|
# The patched libr4d is not optional (the image's own build NaNs this model).
|
|
grep -q 'patched libr4d absent' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "a generated unit is never asked to enable, and the llama unit is gone" {
|
|
# podman 5.x v1 generator: generated units cannot be enabled; asking makes
|
|
# the deploy fail on a unit that is already boot-persistent via [Install].
|
|
! grep -q 'systemctl --user enable frank-medium' "$REMOTE_DEPLOY"
|
|
# ...and the root llama unit must be retired, not relinked: relinking would
|
|
# resurrect a service whose repo file no longer exists.
|
|
! grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
|
|
grep -q 'retire_llama_medium_unit' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the llama layer split and the replica trio are archived, not deleted" {
|
|
[ -f "$LLAMA_TRASH/frank-medium.service.llamacpp-layersplit" ]
|
|
[ -f "$LLAMA_TRASH/frank-medium.env.llamacpp-layersplit" ]
|
|
[ -f "$LLAMA_TRASH/serve-frank-medium.sh.llamacpp-layersplit" ]
|
|
[ ! -f "$REPO_ROOT/scripts/frank-medium.env" ]
|
|
[ -f "$REPLICA_TRASH/frank-medium-a.container" ]
|
|
[ -f "$REPLICA_TRASH/frank-medium-proxy.container" ]
|
|
# The replicas stay retired: the handler stops them but never installs them.
|
|
grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
|
|
! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
|
|
}
|