Runtime swap on the RTX 3090 (port 7771): - vLLM gemma-x86_64-cu129 dev build, digest-pinned, audio extra pip-installed into localhost/vllm/vllm-openai:frank-small (Dockerfile, no COPY lines) - google/gemma-4-12B-it-qat-w4a16-ct (10.26 GiB int4) + Google's bf16 MTP assistant (845 MB) — the w4a16-ct assistant cannot draft in vLLM (drafter path skips Marlin; acceptance 1.00), bf16 measures 2.80-3.56 - root systemd unit -> rootless Quadlet (dankclaw/frank-small/quadlet/); manual CUDA device+lib mounts (no nvidia-container-toolkit on host) - bf16 KV (fp8 hardware-dead on sm_86), 289,647-token pool, 2 concurrent full-context sessions, CUDA graphs on - native text+image+audio: the fleet's multimodal endpoint - measured: pp@67k 1293 t/s, pp@213k 608 t/s, tg@213k ~13.6 t/s, tg@85k ~17.6 t/s - host driver 610.57.04 (Ubuntu 26.04); sm_86 is correct, no arch override - planner/scoping: frank-small now selected by dankclaw/frank-small/ + env/serve/download trio; old unit retired to scripts/.AGENTTRASH/ - docs: new runbook frank-small-vllm-gemma4.md; AGENTS.md, LLM index, top docs index updated; old runbook moved to retired
117 lines
5.7 KiB
Bash
Executable File
117 lines
5.7 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-large halogen-flash-server quadlet on dankclaw.
|
|
# halogen-flash-server replaced ik_llama.cpp again on 2026-09-16, so this
|
|
# service is once more a rootless Quadlet container on the gfx1151 iGPU. This
|
|
# seam pins what a flag list cannot show: the boot-persistent podman posture
|
|
# (both device nodes, memlock, the create-time image contract), the
|
|
# matched ctx/pool/slots set that IS the two-conversation promise, the
|
|
# co-tenancy budget the arena and reserve are chosen for, the image handoff to
|
|
# frank-small, and a deploy handler that retires the previous ROOT unit without
|
|
# deleting it.
|
|
|
|
load helpers
|
|
|
|
QUADLET="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
|
|
@test "the frank-large quadlet exists where the deployer installs it from" {
|
|
[ -f "$QUADLET" ]
|
|
}
|
|
|
|
@test "frank-large is boot-persistent, self-healing and container-shaped" {
|
|
grep -q '^WantedBy=default.target$' "$QUADLET"
|
|
grep -q '^Restart=on-failure$' "$QUADLET"
|
|
grep -q '^RestartSec=' "$QUADLET"
|
|
grep -q '^Pull=never$' "$QUADLET"
|
|
# The localhost/-aliased tag is what create-time resolution needs on this
|
|
# host; the deployer pulls the ghcr.io ref and re-tags it.
|
|
grep -q '^Image=localhost/peonist-ai/halogen-flash-server:' "$QUADLET"
|
|
}
|
|
|
|
@test "frank-large has the gfx1151 devices and the image's documented posture" {
|
|
grep -q '^AddDevice=/dev/kfd$' "$QUADLET"
|
|
grep -q '^AddDevice=/dev/dri$' "$QUADLET"
|
|
grep -q '^PodmanArgs=--ipc=host --security-opt seccomp=unconfined$' "$QUADLET"
|
|
grep -q '^Ulimit=memlock=-1:-1$' "$QUADLET"
|
|
# The weights mount must stay read-write: the first start downloads into it.
|
|
grep -q '^Volume=/home/dank/models/halogen:/models$' "$QUADLET"
|
|
}
|
|
|
|
@test "frank-large keeps the fleet alias and publishes only the API port" {
|
|
grep -q '^Environment=HALOGEN_MODEL_ID=frank-large$' "$QUADLET"
|
|
grep -q '^PublishPort=7770:8731$' "$QUADLET"
|
|
# The engine's own listener stays inside the container: it is unauthenticated.
|
|
! grep -q 'PublishPort=.*8730' "$QUADLET"
|
|
}
|
|
|
|
@test "the KV pool is exactly two conversations of the per-request ceiling" {
|
|
local ctx pool slots
|
|
ctx="$(sed -nE 's/^Environment=HALOGEN_CTX=([0-9]+)$/\1/p' "$QUADLET")"
|
|
pool="$(sed -nE 's/^Environment=HALOGEN_KV_POOL_POSITIONS=([0-9]+)$/\1/p' "$QUADLET")"
|
|
slots="$(sed -nE 's/^Environment=HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$QUADLET")"
|
|
[ -n "$ctx" ] && [ -n "$pool" ] && [ -n "$slots" ]
|
|
# The deployed promise is two resident conversations: the pool is twice the
|
|
# ceiling, one slot per conversation. Slots still oversubscribe the pool for
|
|
# any real request, because admission reserves prompt + max_tokens.
|
|
[ "$pool" -eq $((ctx * 2)) ]
|
|
[ "$slots" -eq 2 ]
|
|
}
|
|
|
|
@test "the prefill arena stays inside the range the vendor tested" {
|
|
local max_tok reserve
|
|
max_tok="$(sed -nE 's/^Environment=HALOGEN_MAX_TOK=([0-9]+)$/\1/p' "$QUADLET")"
|
|
reserve="$(sed -nE 's/^Environment=HALOGEN_HOST_RESERVE_GIB=([0-9]+)$/\1/p' "$QUADLET")"
|
|
[ -n "$max_tok" ] && [ -n "$reserve" ]
|
|
# 32768 is the vendor default; 65536 cannot boot on this host because the
|
|
# arena alone outgrows the device budget. The arena is a per-call prefill
|
|
# chunk, so a smaller value costs prefill speed, not context.
|
|
[ "$max_tok" -le 32768 ]
|
|
[ "$max_tok" -ge 16384 ]
|
|
[ "$reserve" -ge 15 ]
|
|
}
|
|
|
|
@test "frank-large loads no vision tower: the fleet's image path is frank-small" {
|
|
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET"
|
|
# frank-small's vLLM checkpoint (2026-09-18) is natively multimodal —
|
|
# image content is ingested by the model itself, so the seam that used
|
|
# to pin --mmproj now pins the projector's absence.
|
|
! grep -q 'mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh"
|
|
grep -q '^MODEL_DIR=/models/main$' "$REPO_ROOT/scripts/frank-small.env"
|
|
}
|
|
|
|
@test "the deploy handler retires the previous ROOT unit without deleting it" {
|
|
grep -q 'retire_root_llm_large' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl stop frank-large.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl disable frank-large.service' "$REMOTE_DEPLOY"
|
|
grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY"
|
|
! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the deploy handler installs the quadlet and gates the first start on weights" {
|
|
grep -q 'install_quadlet frank-large dankclaw/frank-large/quadlet frank-large' "$REMOTE_DEPLOY"
|
|
grep -q 'wait_for_llm_large_ready' "$REMOTE_DEPLOY"
|
|
grep -q 'qwen38-flash-next-w4b.hgn' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "the ik_llama.cpp runtime that held this service is archived, not live" {
|
|
# Its files must not sit in scripts/ where a stale root unit could still be
|
|
# pointed at them, and must not be gone either: .AGENTTRASH is the rollback.
|
|
local retired
|
|
for retired in frank-large.env.ik-llama.cpp frank-large.service.ik-llama.cpp \
|
|
serve-frank-large.sh.ik-llama.cpp; do
|
|
[ -f "$REPO_ROOT/scripts/.AGENTTRASH/$retired" ]
|
|
done
|
|
[ ! -e "$REPO_ROOT/scripts/frank-large.env" ]
|
|
[ ! -e "$REPO_ROOT/scripts/frank-large.service" ]
|
|
[ ! -e "$REPO_ROOT/scripts/serve-frank-large.sh" ]
|
|
}
|
|
|
|
@test "the deploy handler waits for the predecessor to release the port" {
|
|
# The 2026-09-16 cutover failed at exactly this: the ik server's socket was
|
|
# still on 7770 when pasta tried to bind it ("Address already in use"), and
|
|
# the container died on it. Restart=on-failure recovered minutes later, but
|
|
# a deploy that starts a doomed container has not deployed anything.
|
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
grep -q '^wait_port_free() {' "$REMOTE_DEPLOY"
|
|
} |