serverAppConfigs/tests/frank_medium_unit.bats

#!/usr/bin/env bats
# Seam: the frank-medium vllm-radiance container on dankserver.
# frank-medium became a CONTAINER again on 2026-09-20, when the board swap put
# both R9700s on CPU root ports and a measured three-arm comparison retired the
# mainline-llama.cpp layer split (scripts/.AGENTTRASH/20260920-llama-layersplit/).
# What this seam pins is what the flags alone do not say: boot persistence for a
# GENERATED unit (this host's v1 quadlet generator cannot enable units, so the
# [Install] section is the only thing that survives a reboot), that BOTH cards
# work on ONE model through tensor parallelism — the thing the pre-2026-09-20
# hardware could not do — the KV pin's coupling to the batch shape it was
# calibrated at, and that the pool still holds one full-window request.

load helpers

UNIT="$REPO_ROOT/dankserver/frank-medium/quadlet/frank-medium.container"
ENTRY="$REPO_ROOT/dankserver/frank-medium/radiance-entrypoint.sh"
HEALTH="$REPO_ROOT/dankserver/frank-medium/frank-medium-healthcheck.py"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
LLAMA_TRASH="$REPO_ROOT/scripts/.AGENTTRASH/20260920-llama-layersplit"
REPLICA_TRASH="$REPO_ROOT/dankserver/.AGENTTRASH/20260916-vllm-radiance"

# Measured constants, used by the fit arithmetic.
CARD_BYTES=34208743424      # one R9700 as rocm-smi reports it (32624 MiB)
KV_BYTES_PER_TOKEN=70656    # --kv-cache-memory 18563072000 => "262,720 tokens"
env_value() {
    sed -nE "s/^Environment=$1=(-?[0-9]+)$/\1/p" "$UNIT" | head -n1
}
# GPU_UTIL is a fraction, not a count; the integer reader would drop it.
env_ratio() {
    sed -nE "s/^Environment=$1=([0-9.]+)$/\1/p" "$UNIT" | head -n1
}

@test "frank-medium's unit, entrypoint and healthcheck exist" {
    [ -f "$UNIT" ]
    [ -x "$ENTRY" ]
    [ -f "$HEALTH" ]
}

@test "frank-medium is boot-persistent and self-healing" {
    # Generated units cannot be enabled on this host, so WantedBy in the file is
    # the whole boot story (AGENTS.md: Boot Persistence & Self-Healing).
    grep -q '^WantedBy=default.target$' "$UNIT"
    grep -q '^Restart=on-failure$' "$UNIT"
    grep -q '^RestartSec=' "$UNIT"
    # First start compiles Triton/inductor kernels before the engine loads ~9.5
    # GiB per card; a shorter timeout kills a healthy boot.
    grep -q '^TimeoutStartSec=1800$' "$UNIT"
    grep -q '^HealthCmd=python3 /usr/local/bin/frank-medium-healthcheck.py$' "$UNIT"
    grep -q '^HealthStartPeriod=1800s$' "$UNIT"
    # Boots must not depend on the registry.
    grep -q '^Pull=never$' "$UNIT"
}

@test "both R9700s serve ONE model as one tensor-parallel process" {
    grep -q '^Environment=TP=2$' "$UNIT"
    grep -q '^Environment=ROCR_VISIBLE_DEVICES=0,1$' "$UNIT"
    grep -q '^Environment=HIP_VISIBLE_DEVICES=0,1$' "$UNIT"
    grep -q -- '--tensor-parallel-size "$TP"' "$ENTRY"
    # The iGPU (index 2, 2 GiB) must stay out of the visible set: a third rank
    # would take layers onto system RAM.
    ! grep -q '^Environment=.*VISIBLE_DEVICES=0,1,2' "$UNIT"
    # TP needs the all-reduce to actually run. R4D is the collective-free path
    # this host measured; the entrypoint must still select it, and the RCCL
    # fallback must not be re-disabled by the pre-board-swap workarounds.
    grep -q 'RADIANCE_USE_R4D_AR' "$ENTRY"
    grep -q -- '--attention-backend "$ATTN"' "$ENTRY"
    ! grep -q 'NCCL_P2P_DISABLE' "$UNIT"
    ! grep -q 'RCCL_MSCCL_ENABLE' "$UNIT"
}

@test "HIP never reads one of these tunables as its own setting" {
    # HIP reads bare TEMP, HOST and GPU_MAX_HEAP_SIZE (2026-08-13 incident:
    # TEMP=1.0 sent the JIT cache to path "1.0" and segfaulted every load).
    # Serve scripts export env files with `set -a`; a container inherits the
    # same hazard from Environment=.
    ! grep -qE '^Environment=(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$UNIT"
}

@test "the drafter is DFlash2 at the measured depth and the kernel band covers the batch" {
    grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT"
    local maxseqs spec chunk
    spec="$(env_value SPEC)"
    maxseqs="$(env_value MAXSEQS)"
    chunk="$(env_value CHUNK)"
    [ -n "$spec" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ]
    # DFlash2 drafts at most 7 tokens per step; deeper is a different drafter's
    # contract and costs acceptance, not just VRAM.
    [ "$spec" -eq 7 ]
    # The MXFP4 decode kernel band covers MAXSEQS x (SPEC+1) rows, and the
    # shipped default covers 64. Exceeding it without the entrypoint's >8 bump
    # is a launch failure at the worst moment (first concurrent request).
    [ $(( maxseqs * (spec + 1) )) -le 64 ]
    grep -q 'RADIANCE_MXFP4_DECODE_MAX_M' "$ENTRY"
    # The chunk is the AR size gate's input (CHUNK x hidden x 2 bytes); a chunk
    # larger than this row's silently changes which all-reduce path runs.
    [ "$chunk" -le 8192 ]
}

@test "the KV pin is valid only at the shape it was calibrated for" {
    # The pin is the upstream kv-profiles.tsv row for sig 2x7551-32624 at
    # maxseqs=8 chunk=8192 maxlen=262144 spec=dflash — exactly this host. It is
    # not a general-purpose budget: changing MAXSEQS, CHUNK or SPEC_METHOD
    # invalidates it, so the shape has to move with it or the pin goes to 0
    # (let the image profile the shape).
    local kv_mem maxseqs chunk maxlen spec
    kv_mem="$(env_value KV_MEM)"
    maxseqs="$(env_value MAXSEQS)"
    chunk="$(env_value CHUNK)"
    maxlen="$(env_value MAXLEN)"
    spec="$(env_value SPEC)"
    [ -n "$kv_mem" ] && [ -n "$maxseqs" ] && [ -n "$chunk" ] && [ -n "$maxlen" ]
    grep -q 'kv-profiles.tsv' "$UNIT"
    [ "$maxseqs" -eq 8 ] && [ "$chunk" -eq 8192 ] && [ "$spec" -eq 7 ]
    grep -q '^Environment=SPEC_METHOD=dflash$' "$UNIT"
    # Pool arithmetic, from the engine's own boot line: 18563072000 bytes
    # reserved => "GPU KV cache size: 262,720 tokens". The pool must hold at
    # least ONE full-window request or the endpoint refuses its own
    # --max-model-len, and it must fit on one card beside the weights.
    [ $(( kv_mem / KV_BYTES_PER_TOKEN )) -ge "$maxlen" ]
    # 9.5 GiB of MXFP4 weights per card + the KV pool, under the 0.98 util cap.
    local weights=10200547328   # ~9.5 GiB per card of the 19 GB MXFP4 checkpoint
    local util util_pct floor
    util="$(env_ratio GPU_UTIL)"
    [ -n "$util" ]
    util_pct="${util#0.}"
    floor=$(( 1024 * 1024 * 1024 ))    # 1 GiB per card
    [ $(( weights + kv_mem / 2 )) -le $(( CARD_BYTES * util_pct / 100 - floor )) ]
}

@test "the fleet endpoint contract survives the runtime swap" {
    grep -q '^Environment=PORT=7770$' "$UNIT"
    grep -q '^Environment=SERVED_NAMES=frank-medium$' "$UNIT"
    grep -q -- '--port "$PORT"' "$ENTRY"
    grep -q -- '--served-model-name $SERVED_NAMES' "$ENTRY"
    # Host network: vLLM binds the port itself, so no PublishPort (and no
    # pasta bind, which is what made the retired proxy's port replacement
    # fail at container create).
    grep -q -- '--network=host' "$UNIT"
    ! grep -q '^PublishPort=' "$UNIT"
    # Tools and reasoning are part of the harness contract; both parsers must
    # be selected, and thinking must land in its own field.
    grep -q -- '--enable-auto-tool-choice' "$ENTRY"
    grep -q -- '--tool-call-parser qwen3_coder' "$ENTRY"
    grep -q -- '--reasoning-parser qwen3' "$ENTRY"
    grep -q -- '--enable-prefix-caching' "$ENTRY"
    # Requests that send no sampler fields get the predecessor's defaults.
    grep -q '"temperature":0.7' "$ENTRY"
    grep -q '"top_p":0.95' "$ENTRY"
}

@test "the healthcheck proves the engine serves, not merely that a process lives" {
    grep -q '"/health"' "$HEALTH" || grep -q '/health' "$HEALTH"
    grep -q 'status == 200' "$HEALTH"
    # PORT-aware, and it must not hard-code a replica port.
    grep -q 'os.environ.get("PORT"' "$HEALTH"
    ! grep -qE '7771|7772' "$HEALTH"
}

@test "the deploy handler gates on host assets and installs the quadlet" {
    grep -q 'deploy_llm_medium' "$REMOTE_DEPLOY"
    grep -q 'install_quadlet frank-medium dankserver/frank-medium/quadlet frank-medium' "$REMOTE_DEPLOY"
    grep -q 'wait_container_healthy frank-medium' "$REMOTE_DEPLOY"
    grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
    grep -q '\[\[ "$missing" -eq 0 \]\] || return 1' "$REMOTE_DEPLOY"
    # The image is only pulled when absent, and re-tagged to the name the unit
    # actually runs (the quadlet path cannot resolve registry refs at create).
    grep -q 'podman image exists "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
    grep -q 'podman tag "$LLM_MEDIUM_IMAGE" "$LLM_MEDIUM_IMAGE_LOCAL"' "$REMOTE_DEPLOY"
    # The patched libr4d is not optional (the image's own build NaNs this model).
    grep -q 'patched libr4d absent' "$REMOTE_DEPLOY"
}

@test "a generated unit is never asked to enable, and the llama unit is gone" {
    # podman 5.x v1 generator: generated units cannot be enabled; asking makes
    # the deploy fail on a unit that is already boot-persistent via [Install].
    ! grep -q 'systemctl --user enable frank-medium' "$REMOTE_DEPLOY"
    # ...and the root llama unit must be retired, not relinked: relinking would
    # resurrect a service whose repo file no longer exists.
    ! grep -q 'relink_unit frank-medium' "$REMOTE_DEPLOY"
    grep -q 'retire_llama_medium_unit' "$REMOTE_DEPLOY"
}

@test "the llama layer split and the replica trio are archived, not deleted" {
    [ -f "$LLAMA_TRASH/frank-medium.service.llamacpp-layersplit" ]
    [ -f "$LLAMA_TRASH/frank-medium.env.llamacpp-layersplit" ]
    [ -f "$LLAMA_TRASH/serve-frank-medium.sh.llamacpp-layersplit" ]
    [ ! -f "$REPO_ROOT/scripts/frank-medium.env" ]
    [ -f "$REPLICA_TRASH/frank-medium-a.container" ]
    [ -f "$REPLICA_TRASH/frank-medium-proxy.container" ]
    # The replicas stay retired: the handler stops them but never installs them.
    grep -q 'systemctl --user stop frank-medium-a.service frank-medium-b.service' "$REMOTE_DEPLOY"
    ! grep -q 'install_quadlet frank-medium-a' "$REMOTE_DEPLOY"
}