Frank
a3e1c97408
The deployed unit failed its first boot with the engine core OOMing during profile/graph capture: "Tried to allocate 662.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 240.00 MiB is free ... try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True". The expert-offload plan fills VRAM to its modelled headroom by design, so the default allocator's fragmentation leaves the capture phase nothing — every successful bring-up probe (both scratch-port and 7770) carried this env, and the repo recipe dropped it. Also pins the seam: the setting is part of the qualified shape, not an optional tuning.
242 lines
12 KiB
Bash
Executable File
242 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
|
|
#
|
|
# This file asserts the tcclaviger RDNA4 vLLM ROOT-UNIT shape (the 2026-09-30
|
|
# cutover to GLM-5.3-Flash W4A16 int4, which replaced the qwen FP8 + MTP4
|
|
# shape): the serve script drives `podman run` DIRECTLY (the image entrypoint
|
|
# execs `vllm serve`), with every engine flag qualified by boot probe on
|
|
# 2026-09-30 and carried in the repo env file. The seam pins what the flags
|
|
# alone do not say: the qualification law (131,072 x 2 seqs at bf16 KV, util
|
|
# 0.98 is what boots; the window is forced up by the fork's KV reserve model,
|
|
# not by capacity), the offload law (170 GiB of experts cannot fit 64 GiB of
|
|
# VRAM — the engine MUST run with --enable-expert-offload), the self-healing
|
|
# law (the run returns before the engine is up, so the serve script must wait
|
|
# on the container for Restart=on-failure to mean anything), the memlock law
|
|
# that makes the 73.64 GiB/rank expert pin legal, the deployer's image/payload
|
|
# gates with the false-green MainPID check, and the three measured-absent
|
|
# flags: expert prefetch (36% precision, costs ~40-50% decode), MTP spec
|
|
# decode (buys zero decode and collapses prefill on an offload-bound shape),
|
|
# and fp8 KV (disables the only MLA decode path gfx12 has).
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-primary.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-primary.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-primary.env"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
DOWNLOAD="$REPO_ROOT/scripts/download-frank-primary.sh"
|
|
|
|
@test "frank-primary unit, serve script and env exist in scripts/" {
|
|
[ -f "$UNIT" ]
|
|
[ -x "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
}
|
|
|
|
@test "frank-primary is boot-persistent, self-healing and unprivileged" {
|
|
# Root unit parity with frank-large/frank-small: enable is the [Install]
|
|
# section, the restart policy survives a crash, and the tiered-expert
|
|
# pin needs an unbounded memlock (a root unit grants it via PID1; the
|
|
# dank session's own hard limit is 8 MiB and cannot be raised without
|
|
# restarting user@1000.service).
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "frank-primary runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh$' "$UNIT"
|
|
grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT"
|
|
}
|
|
|
|
@test "the unit stops the container child explicitly" {
|
|
# The serve script blocks on podman wait, which cannot stop the
|
|
# container child; systemd stop goes through ExecStop or the engine
|
|
# outlives the unit and holds both GPUs. The container name must match
|
|
# the env file's identity.
|
|
grep -q '^ExecStop=/usr/bin/podman stop' "$UNIT"
|
|
grep -q 'frank-primary-glm' "$UNIT"
|
|
grep -q '^FRANK_PRIMARY_CONTAINER=frank-primary-glm$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve shape is direct podman run, not a launcher or shim" {
|
|
# The tcclaviger stack is fully expressed by flags; the R9V launcher,
|
|
# docker->podman shim and setup-state layering are the RETIRED shape.
|
|
grep -q 'podman run' "$SERVE"
|
|
! grep -q 'launch.sh' "$SERVE"
|
|
! grep -q 'podman-shim' "$SERVE"
|
|
! grep -q 'R9V_STATE_FILE' "$SERVE"
|
|
! grep -q -- '--model-draft' "$SERVE"
|
|
! grep -q -- '--spec-type' "$SERVE"
|
|
}
|
|
|
|
@test "the qualified engine shape is in the env file" {
|
|
# 2026-09-30 boot probe: 131,072 x 2 seqs at bf16 KV, util 0.98 boots and
|
|
# serves (measured "GPU KV cache size: 138,561 tokens"). 0.99 fails the
|
|
# free-memory wall. The window is not a capacity choice: the engine's own
|
|
# reserve model (one max_model_len request) needs a large L for this
|
|
# model's per-sequence KDA state, and below ~92k the boot REFUSES with
|
|
# "needed ... larger than the available KV cache memory". NBATCH pins the
|
|
# prefill chunk that was measured.
|
|
grep -q '^FRANK_PRIMARY_MAXLEN=131072$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_SEQS=2$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_UTIL=0.98$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_NBATCH=2048$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_MODEL_SHARDS=34$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script wires the engine flags from the env" {
|
|
grep -q -- '--max-num-seqs "$FRANK_PRIMARY_SEQS"' "$SERVE"
|
|
grep -q -- '--max-model-len "$FRANK_PRIMARY_MAXLEN"' "$SERVE"
|
|
grep -q -- '--max-num-batched-tokens "$FRANK_PRIMARY_NBATCH"' "$SERVE"
|
|
grep -q -- '--tensor-parallel-size 2' "$SERVE"
|
|
grep -q -- '--enable-prefix-caching' "$SERVE"
|
|
grep -q -- '--trust-remote-code' "$SERVE"
|
|
}
|
|
|
|
@test "the endpoint advertises the GLM tool dialect" {
|
|
# Without --enable-auto-tool-choice the engine rejects tool_choice "auto"
|
|
# with a 400 ("requires --enable-auto-tool-choice and --tool-call-parser"),
|
|
# which breaks every tool-using agent on the endpoint.
|
|
grep -q -- '--enable-auto-tool-choice' "$SERVE"
|
|
grep -q -- '--tool-call-parser "$FRANK_PRIMARY_TOOLPARSER"' "$SERVE"
|
|
grep -q -- '--reasoning-parser "$FRANK_PRIMARY_REASONINGPARSER"' "$SERVE"
|
|
grep -q '^FRANK_PRIMARY_TOOLPARSER=glm47$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_REASONINGPARSER=glm47$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "expandable segments is ON — the cache arena leaves no slack" {
|
|
# The expert-offload plan fills VRAM to the modelled headroom, so the
|
|
# profile/graph-capture phase has ~0.2 GiB to work with and the default
|
|
# allocator's fragmentation fails it: measured 2026-09-30 on the deployed
|
|
# unit, "Tried to allocate 662.00 MiB ... 240.00 MiB is free" from the
|
|
# engine core while the identical shape booted with this env set.
|
|
grep -q 'PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True' "$SERVE"
|
|
}
|
|
|
|
@test "expert offload is ON — the payload cannot fit VRAM without it" {
|
|
# 170 GiB of int4 experts against 64 GiB of VRAM. Without the offload flag
|
|
# the engine has no home for the expert tensors at all; the arena
|
|
# (14.51 GiB = 2,383 slots) plus 73.64 GiB/rank of pinned host rows is the
|
|
# only shape that boots. --expert-precise-memory is the qualified sizing.
|
|
grep -q -- '--enable-expert-offload' "$SERVE"
|
|
grep -q -- '--expert-precise-memory' "$SERVE"
|
|
}
|
|
|
|
@test "MTP spec decode is absent BY MEASUREMENT, not by omission" {
|
|
# 2026-09-30: the MTP head loads and accepts well (mean accepted length
|
|
# 3.76-4.90), but decode was unchanged (9.6-11.2 vs 10.0 tok/s) while
|
|
# prefill collapsed 918 -> 255 tok/s — the verification forward gathers the
|
|
# union of the batch's experts, so spec decode does not amortize a
|
|
# residency-bound gather. It also costs 2.6 GiB of arena.
|
|
! grep -q -- '--speculative-config' "$SERVE"
|
|
! grep -q 'speculative' "$ENV_FILE"
|
|
}
|
|
|
|
@test "fp8 KV is absent — it would disable the only MLA decode path on gfx12" {
|
|
# _use_rocm_sparse_triton() in the ROCM_AITER_MLA_SPARSE backend requires a
|
|
# non-fp8 KV cache, and that rope-free Triton path is the only MLA decode
|
|
# available on gfx12: aiter's MLA decode op is registered under
|
|
# @if_aiter_supported, gated on CDNA>2. With fp8 KV the MTP draft step dies
|
|
# with "_OpNamespace 'vllm' object has no attribute
|
|
# 'rocm_aiter_mla_decode_fwd'". KV dtype therefore stays auto/bf16.
|
|
! grep -q -- '--kv-cache-dtype' "$SERVE"
|
|
! grep -q '^FRANK_PRIMARY_KV_DTYPE=' "$ENV_FILE"
|
|
}
|
|
|
|
@test "expert prefetch is absent BY LAW, not by omission" {
|
|
# Telemetry (2026-09-29): the prefetcher served 13,369 of 36,720
|
|
# prefetched slabs (36% precision) — roughly half the gather traffic was
|
|
# waste, and the "savings" would have been installs anyway. Deleting the
|
|
# flag moved decode 26.8 -> 40.6 tok/s. Any future flag must come with a
|
|
# fresh A/B, so the serve script must NOT wire it unconditionally.
|
|
! grep -q -- '--expert-prefetch' "$SERVE"
|
|
! grep -q 'expert_prefetch' "$ENV_FILE"
|
|
grep -q 'PREFETCH IS ABSENT BY LAW' "$ENV_FILE"
|
|
}
|
|
|
|
@test "TunableOp stays off" {
|
|
# CLAV_TUNABLEOP_SWEEP silently skips under expert offload unless
|
|
# CLAV_TUNABLEOP_WITH_OFFLOAD=1, and forcing it corrupts numerics
|
|
# (repetition loops, early halts, invalid measurements).
|
|
! grep -q 'TUNABLEOP' "$ENV_FILE"
|
|
! grep -q 'TUNABLEOP' "$SERVE"
|
|
}
|
|
|
|
@test "the serve script gates the digest-pinned image and the payload" {
|
|
# The image is pinned by OCI manifest digest (the floating :latest moved
|
|
# once already). The payload gate checks config.json and the exact shard
|
|
# count from the env — a missing shard means a guaranteed long-timeout
|
|
# boot instead of a fast failure.
|
|
grep -q 'FRANK_PRIMARY_IMAGE' "$SERVE"
|
|
grep -q 'RepoDigests' "$SERVE"
|
|
grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$SERVE"
|
|
grep -q 'config.json' "$SERVE"
|
|
grep -q '/dev/kfd' "$SERVE"
|
|
grep -q -- '--device /dev/dri' "$SERVE"
|
|
! grep -q 'tcclaviger/vllm:latest' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script clears stale containers and self-heals via podman wait" {
|
|
# A crashed unit can leave the container name registered forever after;
|
|
# the legacy identities (the retired R9V container and the ad hoc
|
|
# experiment container) hold the same GPUs and die here too. Then block
|
|
# on the container so a crash is a nonzero ExecStart.
|
|
grep -q 'FRANK_PRIMARY_LEGACY_CONTAINERS' "$ENV_FILE"
|
|
grep -q 'frank-primary-r9v' "$ENV_FILE"
|
|
grep -q 'qwen-fp8' "$ENV_FILE"
|
|
grep -q 'exec podman wait "$FRANK_PRIMARY_CONTAINER"' "$SERVE"
|
|
}
|
|
|
|
@test "frank-primary holds the single fleet port under its own alias" {
|
|
# The scratch identity (qwen3.8-flash-next on :7773) must never leak
|
|
# into production: LAN publish of :7770, the harness's model id.
|
|
grep -q '^FRANK_PRIMARY_PUBLISH=7770:7770$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_SERVED_NAME=frank-primary$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the model id matches the payload's served alias wiring" {
|
|
# The payload dir and its shard count are the fetch script's output
|
|
# layout; the fetch script must write where the env points.
|
|
grep -q '^FRANK_PRIMARY_MODEL_DIR=/home/dank/models/GLM-5.3-Flash-W4A16-AutoRound$' "$ENV_FILE"
|
|
grep -q '^FRANK_PRIMARY_MODEL_SHARDS=34$' "$ENV_FILE"
|
|
grep -q 'GLM-5.3-Flash-W4A16-AutoRound' "$DOWNLOAD"
|
|
grep -q 'Intel/GLM-5.3-Flash-W4A16-AutoRound' "$DOWNLOAD"
|
|
}
|
|
|
|
@test "no bare HIP-reserved env var is defined" {
|
|
# HIP reads bare TEMP for its JIT cache path (2026-08-13: TEMP=1.0
|
|
# segfaulted every load), and HOST/GPU_MAX_HEAP_SIZE are the same
|
|
# class. Every key here is FRANK_PRIMARY_-prefixed.
|
|
! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
|
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
|
}
|
|
|
|
@test "the deployer gates the image digest and payload, then health-checks" {
|
|
# The deployer refuses when the pinned image is absent from rootless
|
|
# storage or the payload is missing/wrong-sharded — then clears stale
|
|
# containers (including the legacy identities), waits out the port,
|
|
# relinks/enables/restarts the root unit and health-gates it.
|
|
grep -q 'LLM_PRIMARY_CONTAINER=frank-primary-glm' "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_PRIMARY_IMAGE' "$REMOTE_DEPLOY"
|
|
grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_PRIMARY_MODEL_SHARDS' "$REMOTE_DEPLOY"
|
|
grep -q 'config.json' "$REMOTE_DEPLOY"
|
|
grep -q 'qwen-fp8' "$REMOTE_DEPLOY"
|
|
grep -q 'glm-probe' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deployer will not report green on a stray port-holder" {
|
|
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
|
|
# the health curl while the real unit was stopped. The deploy is only
|
|
# green when the unit's own MainPID is live behind the health answer.
|
|
grep -q 'systemctl show -p MainPID --value frank-primary.service' "$REMOTE_DEPLOY"
|
|
grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
|
|
}
|