Frank
e23df8b229
The build default is 8192 MiB. dankclaw is memory-tight (frank-large's 70 GiB GTT plus the rest of the stack) and Gemma4 runs fully on the 3090 (-ngl 999), so the CPU-side cache is not the hot path. Cap it at 2048 to give back ~6 GiB of LPDDR. Gemma4 is not a DeltaNet model, so the >12.8 GiB CACHE_RAM floor does not apply. Wired through frank-small.env (CACHE_RAM) -> serve-frank-small.sh (--cache-ram), with a wiring seam so the cap cannot be silently unwired.
122 lines
5.5 KiB
Bash
Executable File
122 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: the frank-small mainline-llama.cpp root unit on dankclaw.
|
|
# frank-small is the RTX 3090 endpoint created on 2026-09-16, when frank-large
|
|
# went back to the halogen engine on the iGPU. This seam pins what the flags
|
|
# alone do not say: boot persistence for a NATIVE runtime (no generated quadlet
|
|
# is involved), the repo-relative paths the unit executes, the separate client
|
|
# port, the MTP contract (the head is a separate sidecar file loaded as the
|
|
# draft model, chained behind ngram-mod), the vision handoff from frank-large,
|
|
# and the KV budget: q8_0 quantized KV whose context --fit sizes to the card.
|
|
|
|
load helpers
|
|
|
|
UNIT="$REPO_ROOT/scripts/frank-small.service"
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-small.sh"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-small.env"
|
|
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
ENV_LARGE="$REPO_ROOT/scripts/frank-large.env"
|
|
|
|
@test "frank-small unit, serve script and env exist in scripts/" {
|
|
[ -f "$UNIT" ]
|
|
[ -f "$SERVE" ]
|
|
[ -f "$ENV_FILE" ]
|
|
}
|
|
|
|
@test "frank-small is boot-persistent, self-healing and unprivileged" {
|
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
|
grep -q '^Restart=on-failure$' "$UNIT"
|
|
grep -q '^RestartSec=' "$UNIT"
|
|
grep -q '^User=dank$' "$UNIT"
|
|
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "frank-small runs the repo's own serve script with the repo's env" {
|
|
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-small.sh$' "$UNIT"
|
|
grep -q 'FRANK_SMALL_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-small.env' "$UNIT"
|
|
}
|
|
|
|
@test "frank-small offloads every layer to the 3090 with flash attention" {
|
|
grep -q 'llama.cpp-mainline/build-cuda/bin/llama-server' "$ENV_FILE"
|
|
grep -q '^NGL=999$' "$ENV_FILE"
|
|
grep -q -- '--gpu-layers "$NGL"' "$SERVE"
|
|
grep -q '^FLASH_ATTN=on$' "$ENV_FILE"
|
|
grep -q -- '--flash-attn "$FLASH_ATTN"' "$SERVE"
|
|
# Prometheus scrapes this endpoint; without --metrics there is no /metrics.
|
|
grep -q -- '--metrics' "$SERVE"
|
|
}
|
|
|
|
@test "frank-small caps its CPU RAM cache below the build default" {
|
|
# Without the flag the 8192 MiB default returns and the env cap is a
|
|
# silent no-op; this memory-tight host needs the cap to hold.
|
|
grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE"
|
|
}
|
|
|
|
@test "frank-small drafts with the MTP sidecar chained behind ngram-mod" {
|
|
grep -q '^SPEC_TYPE=draft-mtp,ngram-mod$' "$ENV_FILE"
|
|
grep -q -- '--spec-type "$SPEC_TYPE"' "$SERVE"
|
|
# The Gemma4 MTP head is a separate assistant file (gemma4-assistant, 4
|
|
# nextn layers); it loads as --model-draft and shares the target's last
|
|
# layers' KV. ngram-mod is draftless, so the chain is legal.
|
|
grep -q '^DRAFT_FILE=mtp-gemma-4-12B-it\.gguf$' "$ENV_FILE"
|
|
grep -q -- '--model-draft "$MODEL_DIR/$DRAFT_FILE"' "$SERVE"
|
|
}
|
|
|
|
@test "frank-small carries the fleet's image path, and only it" {
|
|
grep -q '^MMPROJ_PATH=.*mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16\.gguf$' "$ENV_FILE"
|
|
grep -q -- '--mmproj "$MMPROJ_PATH"' "$SERVE"
|
|
! grep -q '^ENABLE_VISION=1$' "$ENV_LARGE"
|
|
}
|
|
|
|
@test "frank-small's slot grid is full-window and fits the 24 GiB card" {
|
|
local parallel cache_k cache_v ctx fixed per_slot
|
|
parallel="$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
cache_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
|
cache_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
|
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
[ -n "$parallel" ] && [ -n "$cache_k" ] && [ -n "$cache_v" ] && [ -n "$ctx" ]
|
|
# Every slot must be the model's FULL trained window (262144), not a
|
|
# divided one -- CTX_SIZE is the total pool and --parallel divides it.
|
|
[ $((ctx / parallel)) -eq 262144 ]
|
|
[ $((ctx % parallel)) -eq 0 ]
|
|
[ "$cache_k" = "q8_0" ] && [ "$cache_v" = "q8_0" ]
|
|
# Fit budget from the measured boot arithmetic on the 3090 (24576
|
|
# MiB): one full-context slot idles at 11,996 MiB, four at 22,346,
|
|
# so each slot costs (22346-11996)/3 = 3450 MiB and the fixed part
|
|
# (weights, projector, sidecar draft, compute buffers, CUDA context)
|
|
# is 11996-3450 = 8546 MiB. If the config asks for more slots than
|
|
# the card holds, this inequality fails before deploy instead of the
|
|
# KV allocation failing at boot.
|
|
fixed=8546
|
|
per_slot=3450
|
|
[ $((fixed + parallel * per_slot)) -le 24576 ]
|
|
# Quantized KV on CUDA needs the GGML_CUDA_FA_ALL_QUANTS build option,
|
|
# without which prefill collapses about 12x (see the CUDA lesson in
|
|
# AGENTS.md). The build-cuda tree must carry it.
|
|
grep -q '^FLASH_ATTN=on$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "frank-small runs on its own port beside frank-large's" {
|
|
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
|
|
grep -q '^ALIAS=frank-small$' "$ENV_FILE"
|
|
grep -q -- '--port "$LISTEN_PORT"' "$SERVE"
|
|
grep -q -- '--host "$BIND_ADDR"' "$SERVE"
|
|
# The two endpoints cannot share a port: frank-large holds 7770, this one
|
|
# 7771.
|
|
grep -q '^PORT=7770$' "$ENV_LARGE"
|
|
! grep -q '^PORT=7771$' "$ENV_LARGE"
|
|
}
|
|
|
|
@test "the deploy handler installs the root unit and refuses a missing model" {
|
|
grep -q 'deploy_llm_small' "$REMOTE_DEPLOY"
|
|
grep -q 'relink_unit frank-small' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl enable frank-small.service' "$REMOTE_DEPLOY"
|
|
grep -q 'systemctl restart frank-small.service' "$REMOTE_DEPLOY"
|
|
grep -q 'Gemma4-12B-QAT' "$REMOTE_DEPLOY"
|
|
grep -q 'refusing to start a unit that cannot load' "$REMOTE_DEPLOY"
|
|
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
|
}
|
|
|
|
@test "frank-small's serve script refuses to start on a half-finished download" {
|
|
grep -q 'frank-small: missing' "$SERVE"
|
|
}
|