Frank
5e581666f2
int8_per_token_head keeps a separate fp32 scale tensor per (token, head); Gemma4's hybrid head dims (SWA 256 / full 512) make the per-token page bytes 4160 vs 8256, not an integer ratio, so unify_kv_cache_spec_page_size raises NotImplementedError — nine crash-loop restarts measured. TurboQuant k8v4 packs its block scales inside the tensor, keeps the 2:1 page ratio, and has a dedicated backend with an explicit Ampere path. 289,647 -> ~772k-token pool.
161 lines
8.2 KiB
Bash
Executable File
161 lines
8.2 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-small vLLM quadlet on dankclaw (since 2026-09-18).
|
|
# frank-small is the RTX 3090 endpoint. It now runs vLLM (the pinned
|
|
# gemma4-unified image) serving google/gemma-4-12B-it-qat-w4a16-ct with the
|
|
# google bf16 MTP assistant, replacing the mainline-llama.cpp root
|
|
# unit (retired into scripts/.AGENTTRASH/). This seam pins what the flags
|
|
# alone do not say: boot persistence for the generated quadlet, the CUDA
|
|
# wiring this host needs (no nvidia-container-toolkit), the repo-relative
|
|
# entrypoint and env, the MTP contract (a separate QAT assistant checkpoint
|
|
# in --speculative-config, method "mtp"), the fleet's image path (the model
|
|
# is natively multimodal — no projector file), the full-window sizing, the
|
|
# separate client port, and the deploy handler's cutover order.
|
|
|
|
load helpers
|
|
|
|
QUADLET="$REPO_ROOT/dankclaw/frank-small/quadlet/frank-small.container"
|
|
DOCKERFILE="$REPO_ROOT/dankclaw/frank-small/Dockerfile"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
|
|
FETCH="$REPO_ROOT/scripts/download-frank-small-vllm.sh"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
|
|
|
@test "frank-small quadlet, Dockerfile, entrypoint, env and fetch script exist" {
|
|
[ -f "$QUADLET" ]
|
|
[ -f "$DOCKERFILE" ]
|
|
[ -f "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
[ -f "$FETCH" ]
|
|
# The old root unit is retired, not deleted: it must live in the trash,
|
|
# not in scripts/, so the planner and relink_unit never see it.
|
|
[ ! -e "$REPO_ROOT/scripts/frank-small.service" ]
|
|
[ -e "$REPO_ROOT/scripts/.AGENTTRASH/frank-small.service.llama.cpp" ]
|
|
}
|
|
|
|
@test "frank-small is boot-persistent, self-healing and registry-pinned" {
|
|
grep -q '^WantedBy=default.target$' "$QUADLET"
|
|
grep -q '^Restart=on-failure$' "$QUADLET"
|
|
grep -q '^RestartSec=' "$QUADLET"
|
|
grep -q '^Pull=never$' "$QUADLET"
|
|
# The unit runs the localhost/-aliased LOCAL BUILD (base + vllm[audio]
|
|
# extra); the digest-pinned docker.io base lives only in the deploy
|
|
# handler, which pulls, re-tags and builds before restart.
|
|
grep -q '^Image=localhost/vllm/vllm-openai:frank-small$' "$QUADLET"
|
|
! grep -q '^Image=docker.io' "$QUADLET"
|
|
}
|
|
|
|
@test "frank-small wires CUDA without the nvidia-container-toolkit" {
|
|
# The four device nodes (0666 on this host, so rootless can open them)...
|
|
grep -q '^AddDevice=/dev/nvidia0$' "$QUADLET"
|
|
grep -q '^AddDevice=/dev/nvidiactl$' "$QUADLET"
|
|
grep -q '^AddDevice=/dev/nvidia-uvm$' "$QUADLET"
|
|
grep -q '^AddDevice=/dev/nvidia-uvm-tools$' "$QUADLET"
|
|
# ...plus the host driver's libcuda and NVML stubs. Without the NVML
|
|
# mount vllm serve dies at parser construction with "Failed to infer
|
|
# device type" (measured 2026-09-18).
|
|
grep -q 'libcuda.so.1.*:ro' "$QUADLET"
|
|
grep -q 'libnvidia-ml.so.1.*:ro' "$QUADLET"
|
|
grep -q '^Environment=NVIDIA_VISIBLE_DEVICES=all$' "$QUADLET"
|
|
# CUDA pins its staging buffers; memlock must be unlimited.
|
|
grep -q '^Ulimit=memlock=-1:-1$' "$QUADLET"
|
|
}
|
|
|
|
@test "frank-small runs the repo's own entrypoint with the repo's env" {
|
|
grep -q '^Entrypoint=/scripts/serve-frank-small.sh$' "$QUADLET"
|
|
grep -q '^Environment=FRANK_SMALL_ENV_FILE=/scripts/frank-small.env$' "$QUADLET"
|
|
grep -q 'Volume=/home/dank/serverAppConfigs/scripts:/scripts:ro' "$QUADLET"
|
|
}
|
|
|
|
@test "frank-small mounts the sha256-verified payloads read-only" {
|
|
grep -q 'Volume=/home/dank/models/gemma4-12b-qat-w4a16-ct:/models/main:ro' "$QUADLET"
|
|
grep -q 'Volume=/home/dank/models/gemma4-12b-assistant-bf16:/models/assistant:ro' "$QUADLET"
|
|
# The fetch script verifies both LFS payloads against the Hub oids.
|
|
grep -q '60b6e3989502969d8ae04185d72ecbbc7db63978d5af747a493d53895aa6bfa3' "$FETCH"
|
|
grep -q '3279c173daddd7186e79d652ad94022415736d3a1370625696c898429b06d6df' "$FETCH"
|
|
}
|
|
|
|
@test "frank-small drafts with the QAT MTP assistant via method mtp" {
|
|
# The drafter is a separate 4-layer assistant checkpoint in the same
|
|
# int4 w4a16 compressed-tensors format (the card's compatibility rule:
|
|
# a QAT target drafts with a QAT drafter at the same precision), passed
|
|
# in --speculative-config as method "mtp" — vLLM's Gemma 4 MTP path.
|
|
grep -qF -- '\"method\":\"mtp\"' "$SERVE"
|
|
grep -q -- '--speculative-config "$SPEC_CONFIG"' "$SERVE"
|
|
grep -q '^SPEC_NUM_TOKENS=4$' "$ENV_FILE"
|
|
# The gemma4 reasoning and tool-call parsers keep the fleet contract:
|
|
# reasoning_content on the wire and tools answered.
|
|
grep -q -- '--reasoning-parser gemma4' "$SERVE"
|
|
grep -q -- '--tool-call-parser gemma4' "$SERVE"
|
|
grep -q -- '--enable-auto-tool-choice' "$SERVE"
|
|
}
|
|
|
|
@test "frank-small carries the fleet's image path, and only it" {
|
|
# The model is natively multimodal (encoder-free unified): image content
|
|
# is ingested by the checkpoint itself, so there is no projector file
|
|
# any more — the seam that used to pin MMPROJ_PATH now pins its absence.
|
|
! grep -q '^MMPROJ' "$ENV_FILE"
|
|
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE"
|
|
}
|
|
|
|
@test "frank-small serves the full window with turboquant k8v4 KV on a dedicated card" {
|
|
local maxlen kvdtype util seqs
|
|
maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
|
|
util="$(sed -nE 's/^GPU_MEM_UTIL=([0-9.]+)$/\1/p' "$ENV_FILE")"
|
|
seqs="$(sed -nE 's/^MAX_NUM_SEQS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
# The model's FULL trained window (262144), not a divided one.
|
|
[ "$maxlen" = "262144" ]
|
|
# TurboQuant k8v4 KV (K 8-bit, V 4-bit, in-tensor block scales,
|
|
# dedicated TURBOQUANT backend with an Ampere path). The int8/fp8
|
|
# per-token-head dtypes cannot boot this model: their separate fp32
|
|
# scale tensor breaks the page-size ratio between Gemma4's SWA
|
|
# (head_dim 256) and full-attention (head_dim 512) layers, and fp8 is
|
|
# hardware-dead on sm_86 anyway.
|
|
[ "$kvdtype" = "turboquant_k8v4" ]
|
|
# The 3090 is dedicated to this service, so the engine may take most of
|
|
# it; the KV pool, not the seq ceiling, caps full-context concurrency.
|
|
[ "$util" = "0.95" ]
|
|
[ "$seqs" -ge 4 ]
|
|
}
|
|
|
|
@test "frank-small runs on its own port beside frank-large's" {
|
|
# 8000 is the in-container port; the quadlet publishes it as host 7771,
|
|
# the endpoint Hermes, the harness and the homepage probe use.
|
|
grep -q '^LISTEN_PORT=8000$' "$ENV_FILE"
|
|
grep -q '^ALIAS=frank-small$' "$ENV_FILE"
|
|
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
|
|
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
|
|
grep -q '^PublishPort=7771:8000$' "$QUADLET"
|
|
# The two endpoints cannot share a port, and only the halogen unit
|
|
# publishes 7770.
|
|
grep -q '^PublishPort=7770:8731$' "$QUADLET_LARGE"
|
|
! grep -q 'PublishPort=.*7771' "$QUADLET_LARGE"
|
|
}
|
|
|
|
@test "the deploy handler cuts over from the root unit and gates on health" {
|
|
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
|
|
# The digest-pinned docker.io base is pulled and re-tagged to the FROM
|
|
# ref, then the Dockerfile is built into the localhost tag the quadlet
|
|
# runs (the vllm[audio] extra the base ships without).
|
|
grep -q 'VLLM_SMALL_BASE_IMAGE="docker.io/vllm/vllm-openai@sha256:' "$REMOTE_DEPLOY"
|
|
grep -q 'VLLM_SMALL_IMAGE="localhost/vllm/vllm-openai:frank-small"' "$REMOTE_DEPLOY"
|
|
grep -q 'podman tag "\$VLLM_SMALL_BASE_IMAGE" "\$VLLM_SMALL_BASE_ALIAS"' "$REMOTE_DEPLOY"
|
|
grep -q 'podman build --pull=never -t "\$VLLM_SMALL_IMAGE"' "$REMOTE_DEPLOY"
|
|
grep -q 'dankclaw/frank-small/Dockerfile' "$REMOTE_DEPLOY"
|
|
# The old ROOT unit is retired (stop, disable, trash, reload) before the
|
|
# user unit takes the port — two managers must never own frank-small.
|
|
grep -q 'retire_root_llm_small' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_port_free 7771' "$REMOTE_DEPLOY"
|
|
# Missing payloads refuse the start, and the health gate is the real
|
|
# endpoint.
|
|
grep -q 'gemma4-12b-qat-w4a16-ct/model.safetensors' "$REMOTE_DEPLOY"
|
|
grep -q 'gemma4-12b-assistant-bf16/model.safetensors' "$REMOTE_DEPLOY"
|
|
grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "frank-small's entrypoint refuses to start on a half-finished download" {
|
|
grep -q 'frank-small: missing' "$SERVE"
|
|
}
|