dank/serverAppConfigs
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

frank-small: llama.cpp -> vLLM (gemma-4-12B-it-qat-w4a16-ct, w4a16 Marlin)

Browse Source 
Runtime swap on the RTX 3090 (port 7771):
- vLLM gemma-x86_64-cu129 dev build, digest-pinned, audio extra pip-installed
  into localhost/vllm/vllm-openai:frank-small (Dockerfile, no COPY lines)
- google/gemma-4-12B-it-qat-w4a16-ct (10.26 GiB int4) + Google's bf16 MTP
  assistant (845 MB) — the w4a16-ct assistant cannot draft in vLLM (drafter
  path skips Marlin; acceptance 1.00), bf16 measures 2.80-3.56
- root systemd unit -> rootless Quadlet (dankclaw/frank-small/quadlet/);
  manual CUDA device+lib mounts (no nvidia-container-toolkit on host)
- bf16 KV (fp8 hardware-dead on sm_86), 289,647-token pool, 2 concurrent
  full-context sessions, CUDA graphs on
- native text+image+audio: the fleet's multimodal endpoint
- measured: pp@67k 1293 t/s, pp@213k 608 t/s, tg@213k ~13.6 t/s,
  tg@85k ~17.6 t/s
- host driver 610.57.04 (Ubuntu 26.04); sm_86 is correct, no arch override
- planner/scoping: frank-small now selected by dankclaw/frank-small/ +
  env/serve/download trio; old unit retired to scripts/.AGENTTRASH/
- docs: new runbook frank-small-vllm-gemma4.md; AGENTS.md, LLM index,
  top docs index updated; old runbook moved to retired
This commit is contained in:
Frank
2026-09-18 21:06:46 -07:00
parent cf06c4eab8
commit a2b39b179a
15 changed files with 821 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/deployment_scoping.bats
+4 -2
View File
@@ -64,7 +64,8 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
@test "the small endpoint is Dankclaw's again, and Dankserver's copy stays retired" {
run python3 "$PLANNER" --target dankclaw --paths \
scripts/frank-small.env scripts/frank-small.service scripts/serve-frank-small.sh
dankclaw/frank-small/quadlet/frank-small.container scripts/frank-small.env \
scripts/serve-frank-small.sh scripts/download-frank-small-vllm.sh
[ "$status" -eq 0 ]
[ "$output" = "frank-small" ]
@@ -72,7 +73,8 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
# that name was retired with the FreeToken runtime on 2026-09-15 and must
# not come back through a planner mapping.
run python3 "$PLANNER" --target dankserver --paths \
scripts/frank-small.env scripts/frank-small.service scripts/serve-frank-small.sh
dankclaw/frank-small/quadlet/frank-small.container scripts/frank-small.env \
scripts/serve-frank-small.sh scripts/download-frank-small-vllm.sh
[ "$status" -eq 0 ]
[ -z "$output" ]
}
tests/frank_large_unit.bats
+5 -1
View File
@@ -72,7 +72,11 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
@test "frank-large loads no vision tower: the fleet's image path is frank-small" {
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET"
grep -q -- '--mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh"
# frank-small's vLLM checkpoint (2026-09-18) is natively multimodal —
# image content is ingested by the model itself, so the seam that used
# to pin --mmproj now pins the projector's absence.
! grep -q 'mmproj' "$REPO_ROOT/scripts/serve-frank-small.sh"
grep -q '^MODEL_DIR=/models/main$' "$REPO_ROOT/scripts/frank-small.env"
}
@test "the deploy handler retires the previous ROOT unit without deleting it" {
tests/frank_small_unit.bats
+115 -73
View File
@@ -1,115 +1,157 @@
#!/usr/bin/env bats
# Seam: the frank-small mainline-llama.cpp root unit on dankclaw.
# frank-small is the RTX 3090 endpoint created on 2026-09-16, when frank-large
# went back to the halogen engine on the iGPU. This seam pins what the flags
# alone do not say: boot persistence for a NATIVE runtime (no generated quadlet
# is involved), the repo-relative paths the unit executes, the separate client
# port, the MTP contract (the head is a separate sidecar file loaded as the
# draft model, chained behind ngram-mod), the vision handoff from frank-large,
# and the KV budget: q8_0 quantized KV whose context --fit sizes to the card.
# Seam: the frank-small vLLM quadlet on dankclaw (since 2026-09-18).
# frank-small is the RTX 3090 endpoint. It now runs vLLM (the pinned
# gemma4-unified image) serving google/gemma-4-12B-it-qat-w4a16-ct with the
# google bf16 MTP assistant, replacing the mainline-llama.cpp root
# unit (retired into scripts/.AGENTTRASH/). This seam pins what the flags
# alone do not say: boot persistence for the generated quadlet, the CUDA
# wiring this host needs (no nvidia-container-toolkit), the repo-relative
# entrypoint and env, the MTP contract (a separate QAT assistant checkpoint
# in --speculative-config, method "mtp"), the fleet's image path (the model
# is natively multimodal — no projector file), the full-window sizing, the
# separate client port, and the deploy handler's cutover order.
load helpers
UNIT="$REPO_ROOT/scripts/frank-small.service"
QUADLET="$REPO_ROOT/dankclaw/frank-small/quadlet/frank-small.container"
DOCKERFILE="$REPO_ROOT/dankclaw/frank-small/Dockerfile"
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
FETCH="$REPO_ROOT/scripts/download-frank-small-vllm.sh"
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
QUADLET_LARGE="$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container"
@test "frank-small unit, serve script and env exist in scripts/" {
[ -f "$UNIT" ]
@test "frank-small quadlet, Dockerfile, entrypoint, env and fetch script exist" {
[ -f "$QUADLET" ]
[ -f "$DOCKERFILE" ]
[ -f "$SERVE" ]
[ -f "$ENV_FILE" ]
[ -f "$FETCH" ]
# The old root unit is retired, not deleted: it must live in the trash,
# not in scripts/, so the planner and relink_unit never see it.
[ ! -e "$REPO_ROOT/scripts/frank-small.service" ]
[ -e "$REPO_ROOT/scripts/.AGENTTRASH/frank-small.service.llama.cpp" ]
}
@test "frank-small is boot-persistent, self-healing and unprivileged" {
grep -q '^WantedBy=multi-user.target$' "$UNIT"
grep -q '^Restart=on-failure$' "$UNIT"
grep -q '^RestartSec=' "$UNIT"
grep -q '^User=dank$' "$UNIT"
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
@test "frank-small is boot-persistent, self-healing and registry-pinned" {
grep -q '^WantedBy=default.target$' "$QUADLET"
grep -q '^Restart=on-failure$' "$QUADLET"
grep -q '^RestartSec=' "$QUADLET"
grep -q '^Pull=never$' "$QUADLET"
# The unit runs the localhost/-aliased LOCAL BUILD (base + vllm[audio]
# extra); the digest-pinned docker.io base lives only in the deploy
# handler, which pulls, re-tags and builds before restart.
grep -q '^Image=localhost/vllm/vllm-openai:frank-small$' "$QUADLET"
! grep -q '^Image=docker.io' "$QUADLET"
}
@test "frank-small runs the repo's own serve script with the repo's env" {
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh$' "$UNIT"
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
@test "frank-small wires CUDA without the nvidia-container-toolkit" {
# The four device nodes (0666 on this host, so rootless can open them)...
grep -q '^AddDevice=/dev/nvidia0$' "$QUADLET"
grep -q '^AddDevice=/dev/nvidiactl$' "$QUADLET"
grep -q '^AddDevice=/dev/nvidia-uvm$' "$QUADLET"
grep -q '^AddDevice=/dev/nvidia-uvm-tools$' "$QUADLET"
# ...plus the host driver's libcuda and NVML stubs. Without the NVML
# mount vllm serve dies at parser construction with "Failed to infer
# device type" (measured 2026-09-18).
grep -q 'libcuda.so.1.*:ro' "$QUADLET"
grep -q 'libnvidia-ml.so.1.*:ro' "$QUADLET"
grep -q '^Environment=NVIDIA_VISIBLE_DEVICES=all$' "$QUADLET"
# CUDA pins its staging buffers; memlock must be unlimited.
grep -q '^Ulimit=memlock=-1:-1$' "$QUADLET"
}
@test "frank-small offloads every layer to the 3090 with flash attention" {
grep -q 'llama.cpp-mainline/build-cuda/bin/llama-server' "$ENV_FILE"
grep -q '^NGL=999$' "$ENV_FILE"
grep -q -- '--gpu-layers "$NGL"' "$SERVE"
grep -q '^FLASH_ATTN=on$' "$ENV_FILE"
grep -q -- '--flash-attn "$FLASH_ATTN"' "$SERVE"
# Prometheus scrapes this endpoint; without --metrics there is no /metrics.
grep -q -- '--metrics' "$SERVE"
@test "frank-small runs the repo's own entrypoint with the repo's env" {
grep -q '^ContainerEntrypoint=/scripts/serve-frank-small.sh$' "$QUADLET"
grep -q '^Environment=FRANK_SMALL_ENV_FILE=/scripts/frank-small.env$' "$QUADLET"
grep -q 'Volume=/home/dank/serverAppConfigs/scripts:/scripts:ro' "$QUADLET"
}
@test "frank-small drafts with the MTP sidecar chained behind ngram-mod" {
grep -q '^SPEC_TYPE=draft-mtp,ngram-mod$' "$ENV_FILE"
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
# The Gemma4 MTP head is a separate assistant file (gemma4-assistant, 4
# nextn layers); it loads as --model-draft and shares the target's last
# layers' KV. ngram-mod is draftless, so the chain is legal.
grep -q '^DRAFT_FILE=mtp-gemma-4-12B-it\.gguf$' "$ENV_FILE"
grep -q -- '--model-draft "$MODEL_DIR/$DRAFT_FILE"' "$SERVE"
@test "frank-small mounts the sha256-verified payloads read-only" {
grep -q 'Volume=/home/dank/models/gemma4-12b-qat-w4a16-ct:/models/main:ro' "$QUADLET"
grep -q 'Volume=/home/dank/models/gemma4-12b-assistant-bf16:/models/assistant:ro' "$QUADLET"
# The fetch script verifies both LFS payloads against the Hub oids.
grep -q '60b6e3989502969d8ae04185d72ecbbc7db63978d5af747a493d53895aa6bfa3' "$FETCH"
grep -q '3279c173daddd7186e79d652ad94022415736d3a1370625696c898429b06d6df' "$FETCH"
}
@test "frank-small drafts with the QAT MTP assistant via method mtp" {
# The drafter is a separate 4-layer assistant checkpoint in the same
# int4 w4a16 compressed-tensors format (the card's compatibility rule:
# a QAT target drafts with a QAT drafter at the same precision), passed
# in --speculative-config as method "mtp" — vLLM's Gemma 4 MTP path.
grep -qF -- '\"method\":\"mtp\"' "$SERVE"
grep -q -- '--speculative-config "$SPEC_CONFIG"' "$SERVE"
grep -q '^SPEC_NUM_TOKENS=4$' "$ENV_FILE"
# The gemma4 reasoning and tool-call parsers keep the fleet contract:
# reasoning_content on the wire and tools answered.
grep -q -- '--reasoning-parser gemma4' "$SERVE"
grep -q -- '--tool-call-parser gemma4' "$SERVE"
grep -q -- '--enable-auto-tool-choice' "$SERVE"
}
@test "frank-small carries the fleet's image path, and only it" {
grep -q '^MMPROJ_PATH=.*mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16\.gguf$' "$ENV_FILE"
grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE"
# The model is natively multimodal (encoder-free unified): image content
# is ingested by the checkpoint itself, so there is no projector file
# any more — the seam that used to pin MMPROJ_PATH now pins its absence.
! grep -q '^MMPROJ' "$ENV_FILE"
! grep -q 'HALOGEN_VISION_TOWER' "$QUADLET_LARGE"
}
@test "frank-small's slot grid is full-window and fits the 24 GiB card" {
local parallel cache_k cache_v ctx fixed per_slot
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
[ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$ctx" ]
# Every slot must be the model's FULL trained window (262144), not a
# divided one -- CTX_SIZE is the total pool and --parallel divides it.
[ $((ctx / parallel)) -eq 262144 ]
[ $((ctx % parallel)) -eq 0 ]
[ "$cache_k" = "q8_0" ] && [ "$cache_v" = "q8_0" ]
# Fit budget from the measured boot arithmetic on the 3090 (24576
# MiB): one full-context slot idles at 11,996 MiB, four at 22,346,
# so each slot costs (22346-11996)/3 = 3450 MiB and the fixed part
# (weights, projector, sidecar draft, compute buffers, CUDA context)
# is 11996-3450 = 8546 MiB. If the config asks for more slots than
# the card holds, this inequality fails before deploy instead of the
# KV allocation failing at boot.
fixed=8546
per_slot=3450
[ $((fixed + parallel * per_slot)) -le 24576 ]
# Quantized KV on CUDA needs the GGML_CUDA_FA_ALL_QUANTS build option,
# without which prefill collapses about 12x (see the CUDA lesson in
# AGENTS.md). The build-cuda tree must carry it.
grep -q '^FLASH_ATTN=on$' "$ENV_FILE"
@test "frank-small serves the full window with bf16 KV on a dedicated card" {
local maxlen kvdtype util seqs
maxlen="$(sed -nE 's/^MAX_MODEL_LEN=([0-9]+)$/\1/p' "$ENV_FILE")"
kvdtype="$(sed -nE 's/^KV_CACHE_DTYPE=(.+)$/\1/p' "$ENV_FILE")"
util="$(sed -nE 's/^GPU_MEM_UTIL=([0-9.]+)$/\1/p' "$ENV_FILE")"
seqs="$(sed -nE 's/^MAX_NUM_SEQS=([0-9]+)$/\1/p' "$ENV_FILE")"
# The model's FULL trained window (262144), not a divided one.
[ "$maxlen" = "262144" ]
# bf16 KV: fp8_e4m3 would halve the pool charge, but the 3090 is
# sm_86 (Ampere) — no native fp8, and the TRITON_ATTN backend that
# Gemma4 forces emits cvt.e4m3x2 instructions that exist only from sm_89.
[ "$kvdtype" = "auto" ]
# The 3090 is dedicated to this service, so the engine may take most of
# it; the KV pool, not the seq ceiling, caps full-context concurrency.
[ "$util" = "0.95" ]
[ "$seqs" -ge 4 ]
}
@test "frank-small runs on its own port beside frank-large's" {
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
# 8000 is the in-container port; the quadlet publishes it as host 7771,
# the endpoint Hermes, the harness and the homepage probe use.
grep -q '^LISTEN_PORT=8000$' "$ENV_FILE"
grep -q '^ALIAS=frank-small$' "$ENV_FILE"
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
# The two endpoints cannot share a port, and only the halogen unit publishes
# 7770.
grep -q '^PublishPort=7771:8000$' "$QUADLET"
# The two endpoints cannot share a port, and only the halogen unit
# publishes 7770.
grep -q '^PublishPort=7770:8731$' "$QUADLET_LARGE"
! grep -q 'PublishPort=.*7771' "$QUADLET_LARGE"
}
@test "the deploy handler installs the root unit and refuses a missing model" {
@test "the deploy handler cuts over from the root unit and gates on health" {
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY"
grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY"
grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY"
grep -q 'Gemma4-12B-QAT' "$REMOTE_DEPLOY"
# The digest-pinned docker.io base is pulled and re-tagged to the FROM
# ref, then the Dockerfile is built into the localhost tag the quadlet
# runs (the vllm[audio] extra the base ships without).
grep -q 'VLLM_SMALL_BASE_IMAGE="docker.io/vllm/vllm-openai@sha256:' "$REMOTE_DEPLOY"
grep -q 'VLLM_SMALL_IMAGE="localhost/vllm/vllm-openai:frank-small"' "$REMOTE_DEPLOY"
grep -q 'podman tag "\$VLLM_SMALL_BASE_IMAGE" "\$VLLM_SMALL_BASE_ALIAS"' "$REMOTE_DEPLOY"
grep -q 'podman build --pull=never -t "\$VLLM_SMALL_IMAGE"' "$REMOTE_DEPLOY"
grep -q 'dankclaw/frank-small/Dockerfile' "$REMOTE_DEPLOY"
# The old ROOT unit is retired (stop, disable, trash, reload) before the
# user unit takes the port — two managers must never own frank-small.
grep -q 'retire_root_llm_small' "$REMOTE_DEPLOY"
grep -q 'wait_port_free 7771' "$REMOTE_DEPLOY"
# Missing payloads refuse the start, and the health gate is the real
# endpoint.
grep -q 'gemma4-12b-qat-w4a16-ct/model.safetensors' "$REMOTE_DEPLOY"
grep -q 'gemma4-12b-assistant-bf16/model.safetensors' "$REMOTE_DEPLOY"
grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY"
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
}
@test "frank-small's serve script refuses to start on a half-finished download" {
@test "frank-small's entrypoint refuses to start on a half-finished download" {
grep -q 'frank-small: missing' "$SERVE"
}