Frank
dab483c79f
Engine: digest-pinned docker.io/kyuz0/vllm-therock-gfx1151 (vLLM 0.30, TheRock ROCm 10, Triton GDN kernels); the serve script gates the image by REPO DIGEST, the payload (config + index + exact shard count), the spec draft (BOTH weight layouts: single model.safetensors for dflash, sharded index for the Qwen3.5-9B MTP head), and whole /dev/dri, sweeps the bench identity vllm-mimo, then exec podman wait so a container death trips Restart=always. Shape: 4 x 262144 at util 0.45 (KV grows only on the 8 full-attention layers), MTP 2 draft tokens. Measured 2026-10-01 window (frank-large stopped): base 13.1 t/s decode (~94% of the LPDDR5x BF16 roof), DFlash-8 17.4, MTP-2 19.8 with 71%/44% per-position acceptance - the cross-model head works because vLLM reads the draft from a separate path while MiMo ships zero mtp.* tensors. Spec arms do not preserve greedy bf16 text (batched-verify reduction order). The seam was rewritten around the new laws (18 tests): unit self-healing, digest gating, engine wiring, the measured spec table, the retired gufo gates, the deployer MainPID check. frank_small port test follows the PORT -> FRANK_LARGE_PUBLISH rename. Rollback: tag frank-large-gufo-q4kxl-20261001.
198 lines
9.7 KiB
Bash
Executable File
198 lines
9.7 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: frank-large — the Dankclaw iGPU endpoint on :7770.
|
|
#
|
|
# This file asserts the vLLM ROOT-UNIT shape of the 2026-10-01 cutover: the
|
|
# BF16 MiMo-V2.6-Distill-Qwen-9B payload (the same arch frank-small serves
|
|
# INT8 on the 3090) under the digest-pinned kyuz0 TheRock gfx1151 image,
|
|
# with CROSS-MODEL MTP spec decode from the Qwen3.5-9B head. The gufo-era
|
|
# laws (built-from-source runtime, GGUF schema gates, DFlash2-on-27B) retired
|
|
# with that runtime; rollback is the tag frank-large-gufo-q4kxl-20261001 and
|
|
# scripts/build-gufo.sh stays on disk for it.
|
|
#
|
|
# What the seam pins that the flags alone do not say: the measured laws of
|
|
# the 2026-10-01 window (BF16 decode 13.1 t/s is ~94% of the LPDDR5x roof,
|
|
# so spec is the only way past the wall; MTP-2 measured +51% decode against
|
|
# DFlash-8's +33%; every spec arm pays ~55% of prefill; spec arms do NOT
|
|
# reproduce the no-spec greedy text on vLLM), the roof-free shape (KV grows
|
|
# only on the 8 full-attention layers, so 4 x 262,144 fits the GTT
|
|
# aperture), the client contract difference (vLLM emits `reasoning`, not
|
|
# llama.cpp's `reasoning_content`), and the deployer's image/payload/spec
|
|
# gates with the false-green MainPID check.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
FETCH_TARGET="$REPO_ROOT/scripts/download-frank-large-mimo.sh"
|
|
FETCH_DRAFT="$REPO_ROOT/scripts/download-frank-large-dflash.sh"
|
|
FETCH_MTPSRC="$REPO_ROOT/scripts/download-frank-large-qwen35.sh"
|
|
|
|
@test "frank-large unit, serve script and env exist in scripts/" {
|
|
[ -f "$UNIT" ]
|
|
[ -x "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
}
|
|
|
|
@test "frank-large is boot-persistent and self-healing" {
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
# Restart=always, not on-failure: vLLM's API server can drain and exit 0
|
|
# after an engine fault (the frank-primary 2026-10-01 lesson), so
|
|
# on-failure would leave the endpoint dark until a human notices.
|
|
grep -q '^Restart=always$' "$UNIT"
|
|
! grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
# ExecStop runs `podman stop -t 120`; TimeoutStopSec MUST exceed it or
|
|
# every stop records 'Failed with result timeout' (frank-primary law).
|
|
grep -q '^ExecStop=.*podman stop -t 120 frank-large-mimo' "$UNIT"
|
|
stop_t=$(awk -F= '/^TimeoutStopSec=/{print $2}' "$UNIT")
|
|
[ -n "$stop_t" ] && [ "$stop_t" -gt 120 ]
|
|
# StartLimitIntervalSec belongs to [Unit]; a [Service] copy is an
|
|
# 'Unknown key' warning on every load.
|
|
[ "$(grep -c '^StartLimitIntervalSec=' "$UNIT")" = "1" ]
|
|
awk '/^\[Service\]/{in_svc=1} /^StartLimitIntervalSec=/{if(in_svc) exit 1}' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
grep -q '^OOMScoreAdjust=-500$' "$UNIT"
|
|
}
|
|
|
|
@test "the unit runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh$' "$UNIT"
|
|
grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT"
|
|
}
|
|
|
|
@test "the serve shape is direct podman run, not the retired gufo or llama shapes" {
|
|
grep -q 'podman run' "$SERVE"
|
|
grep -q 'exec podman wait "$FRANK_LARGE_CONTAINER"' "$SERVE"
|
|
# The gufo runtime and the llama.cpp prebuilt are both retired engines;
|
|
# nothing in this trio may reach for them.
|
|
! grep -q 'gufo serve' "$SERVE"
|
|
! grep -q 'llama-server' "$SERVE"
|
|
! grep -q -- '--sessions' "$SERVE"
|
|
! grep -q -- '--model-draft' "$SERVE"
|
|
! grep -q -- '--dflash-model' "$SERVE"
|
|
}
|
|
|
|
@test "the serve script gates the digest-pinned image and the complete payload" {
|
|
grep -q 'FRANK_LARGE_IMAGE' "$SERVE"
|
|
grep -q 'RepoDigests' "$SERVE"
|
|
grep -q 'FRANK_LARGE_MODEL_SHARDS' "$SERVE"
|
|
grep -q 'config.json' "$SERVE"
|
|
grep -q 'model.safetensors.index.json' "$SERVE"
|
|
# Whole /dev/dri, never individual render nodes (Dankserver 2026-09-29:
|
|
# renderD-only mounts enumerate the GPU but cannot create a context).
|
|
grep -q -- '--device /dev/dri' "$SERVE"
|
|
! grep -q 'renderD' "$SERVE"
|
|
# The env file pins by digest, not by the floating tag.
|
|
! grep -qE '^FRANK_LARGE_IMAGE=.*:latest$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the qualified engine shape is in the env file" {
|
|
# 2026-10-01 window: 262144 is the model's native window and the GTT
|
|
# aperture swallows it — KV grows ONLY on the 8 full-attention layers
|
|
# (32 KiB/token bf16); the 24 GDN layers hold fixed ~25 MiB state per
|
|
# running sequence. SEQS 4 keeps the four-session contract; the pool is
|
|
# shared by the scheduler instead of carved per session like gufo.
|
|
grep -q '^FRANK_LARGE_MAXLEN=262144$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_SEQS=4$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_UTIL=0.45$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_MODEL_SHARDS=4$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script wires the engine flags from the env" {
|
|
grep -q -- '--max-model-len "$FRANK_LARGE_MAXLEN"' "$SERVE"
|
|
grep -q -- '--max-num-seqs "$FRANK_LARGE_SEQS"' "$SERVE"
|
|
grep -q -- '--gpu-memory-utilization "$FRANK_LARGE_UTIL"' "$SERVE"
|
|
grep -q -- '--dtype bfloat16' "$SERVE"
|
|
grep -q -- '--trust-remote-code' "$SERVE"
|
|
grep -q -- '--ulimit memlock=-1:-1' "$SERVE"
|
|
}
|
|
|
|
@test "speculative decode is cross-model MTP BY MEASUREMENT" {
|
|
# The 2026-10-01 window on this host: base 13.1 t/s, DFlash-8 17.4,
|
|
# MTP-2 19.8 (+51%). MTP acceptance is 71%/44% per position (token rate
|
|
# 57%) because the head trained on the distill's SOURCE (Qwen/Qwen3.5-9B,
|
|
# the only place mtp.* tensors exist for this family) still reads MiMo's
|
|
# hidden states. Beyond 2 draft tokens is UNMEASURED; the DFlash drafter
|
|
# stays on disk as the alternative method.
|
|
grep -q '^FRANK_LARGE_SPEC=mtp$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_SPEC_TOKENS=2$' "$ENV_FILE"
|
|
# The JSON is built ONLY when SPEC is set, and the draft path handed to
|
|
# the engine is the CONTAINER mount path, never the host path.
|
|
grep -q -- '--speculative-config' "$SERVE"
|
|
grep -q 'FRANK_LARGE_SPEC_CONTAINER_DIR="/models/' "$SERVE"
|
|
# The draft gate accepts BOTH weight layouts: a single
|
|
# model.safetensors (dflash) and a sharded index (the Qwen3.5-9B MTP
|
|
# head). A gate keyed only to the single file rejects the deployed arm.
|
|
grep -q 'FRANK_LARGE_SPEC_DIR/model.safetensors.index.json' "$SERVE"
|
|
# itself carries ZERO mtp.* tensors (that absence is what makes SGLang's
|
|
# NEXTN on frank-small a 0.00-acceptance ghost).
|
|
grep -q 'ZERO mtp' "$ENV_FILE"
|
|
}
|
|
|
|
@test "tools and reasoning are wired — the client contract is served" {
|
|
grep -q '^FRANK_LARGE_REASONING_PARSER=qwen3$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_TOOL_PARSER=qwen3_coder$' "$ENV_FILE"
|
|
grep -q -- '--reasoning-parser "$FRANK_LARGE_REASONING_PARSER"' "$SERVE"
|
|
grep -q -- '--enable-auto-tool-choice' "$SERVE"
|
|
grep -q -- '--tool-call-parser "$FRANK_LARGE_TOOL_PARSER"' "$SERVE"
|
|
# vLLM's field name is `reasoning`, NOT llama.cpp/gufo's
|
|
# `reasoning_content` — the harness models.yml compat block must match.
|
|
grep -q 'reasoning' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script clears stale containers including the bench identity" {
|
|
grep -q '^FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo"$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_CONTAINER=frank-large-mimo$' "$ENV_FILE"
|
|
grep -q 'FRANK_LARGE_LEGACY_CONTAINERS' "$SERVE"
|
|
}
|
|
|
|
@test "frank-large holds the single fleet port under its own alias" {
|
|
grep -q '^FRANK_LARGE_PUBLISH=7770:7770$' "$ENV_FILE"
|
|
grep -q '^FRANK_LARGE_SERVED_NAME=frank-large$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "no bare HIP-reserved env var is defined" {
|
|
# HIP reads bare TEMP/HOST/GPU_MAX_HEAP_SIZE; the serve script sources
|
|
# this file with set -a, so an unprefixed key leaks into the engine.
|
|
! grep -qE '^(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
|
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
|
}
|
|
|
|
@test "the fetch scripts carry the verified payload and draft manifests" {
|
|
grep -q 'XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B' "$FETCH_TARGET"
|
|
[ "$(grep -c 'safetensors.*[0-9a-f]\{64\}' "$FETCH_TARGET")" = "4" ]
|
|
grep -q -- '--verify-only' "$FETCH_TARGET"
|
|
grep -q 'z-lab/Qwen3.5-9B-DFlash' "$FETCH_DRAFT"
|
|
grep -q -- '--verify-only' "$FETCH_DRAFT"
|
|
# The env file's payload/draft dirs must be where the fetch scripts write.
|
|
grep -q 'MiMo-V2.6-Distill-Qwen-9B-BF16' "$FETCH_TARGET"
|
|
grep -q 'Qwen3.5-9B-DFlash' "$FETCH_DRAFT"
|
|
grep -q 'Qwen/Qwen3.5-9B' "$FETCH_MTPSRC"
|
|
grep -q '/home/dank/models/Qwen3.5-9B' "$FETCH_MTPSRC"
|
|
}
|
|
|
|
@test "the gufo deploy gates are retired with the gufo runtime" {
|
|
! grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY"
|
|
! grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY"
|
|
! grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY"
|
|
! grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY"
|
|
# build-gufo.sh stays on disk as the rollback path to the tagged engine.
|
|
[ -f "$REPO_ROOT/scripts/build-gufo.sh" ]
|
|
}
|
|
|
|
@test "the deployer gates image payload and spec then health-checks with MainPID" {
|
|
grep -q 'LLM_LARGE_CONTAINER=frank-large-mimo' "$REMOTE_DEPLOY"
|
|
grep -q 'LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"' "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_LARGE_IMAGE' "$REMOTE_DEPLOY"
|
|
grep -q "podman images --format '{{.RepoDigests}}'" "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_LARGE_MODEL_SHARDS' "$REMOTE_DEPLOY"
|
|
grep -q 'FRANK_LARGE_SPEC=' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
grep -q 'MainPID --value frank-large.service' "$REMOTE_DEPLOY"
|
|
}
|