Frank
64185d63b9
- env: Bonsai2 payload pair, CACHE_TYPE_K/V=q8_0 (f16 OOMs at 2x200,704: 34 KiB vs 64 KiB/token over the 16 full-attention layers), measured boot ledger 22,664 used / 1,461 free incl. the 873 MiB mmproj that lands AFTER the engine fit check, q4_0 fallback rung, CRACK one-line MODEL_FILE swap - deployer: runtime/model/mmproj gates re-keyed to the PrismML vendor build (the only loader for GGML tensor type 142 + prism.hadamard.* metadata) - seam: payload trio pinned; KV argv assertions now derived from CACHE_TYPE_* in the env file — the wrong-var-name f16 fallback that produced the first failed trial window can no longer pass silently
296 lines
13 KiB
Bash
Executable File
296 lines
13 KiB
Bash
Executable File
# Seam: the frank-zealot llama.cpp root unit on dankclaw.
|
||
# frank-zealot is the RTX 3090 endpoint; since 2026-10-17 it serves
|
||
# prism-ml/Ternary-Bonsai-2-27B-gguf at PQ2_0 — the ternary Bonsai-2-27B, a
|
||
# qwen35 hybrid with 64 blocks (16 full-attention + 48 Gated DeltaNet, no
|
||
# nextn/mtp tensors in the file) — via the PrismML fork build of llama.cpp
|
||
# (vendor build 10754), the ONLY loader that registers the payload's custom
|
||
# tensor type 142 and honours its prism.hadamard.* metadata: TWO 200,704-token
|
||
# conversations (--ctx-size 401408 --parallel 2), q8_0 K + q8_0 V under flash
|
||
# attention (f16 KV does not fit this card at this pool — the boot dies on a
|
||
# 25,088 MiB cudaMalloc), the publisher's own Q8_0 vision projector, and MTP
|
||
# off. Log verbosity 4 ships so the VRAM-fit forensics land in the journal.
|
||
# This seam runs the serve script against a stub engine and asserts on the
|
||
# argv it actually execs, so what is pinned is the command the operator gets.
|
||
|
||
load helpers
|
||
|
||
UNIT="$REPO_ROOT/scripts/frank-zealot.service"
|
||
SERVE="$REPO_ROOT/scripts/serve-frank-zealot.sh"
|
||
ENV_FILE="$REPO_ROOT/scripts/frank-zealot.env"
|
||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||
ENV_LARGE="$REPO_ROOT/scripts/frank-dragoon.env"
|
||
|
||
setup() {
|
||
STAGE="$(mktemp -d)"
|
||
RUNTIME_DIR="$STAGE/llama"
|
||
MODEL_DIR="$STAGE/models"
|
||
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR"
|
||
cat > "$RUNTIME_DIR/bin/llama-server" <<'STUB'
|
||
#!/usr/bin/env bash
|
||
printf '%s\n' "$@"
|
||
STUB
|
||
chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server"
|
||
# The stub payload files carry the SAME basenames as the shipped env, so
|
||
# the seam exercises the real paths, and a payload rename moves one file
|
||
# (the env) instead of silently forking the test's filename constants.
|
||
MODEL_STUB="$(basename "$(sed -nE 's/^MODEL_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||
MM_STUB="$(basename "$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||
DRAFT_STUB="draft-assistant.gguf"
|
||
touch "$MODEL_DIR/$MODEL_STUB" "$MODEL_DIR/$MM_STUB" "$MODEL_DIR/$DRAFT_STUB"
|
||
}
|
||
|
||
teardown() {
|
||
rm -rf "$STAGE"
|
||
}
|
||
|
||
# serve_argv [env-file] — run the serve script and capture the engine's
|
||
# argv, one token per line, in $output.
|
||
serve_argv() {
|
||
local envf="${1:-$STAGE/test.env}"
|
||
cat > "$envf" <<EOF
|
||
LLAMA_BIN=$RUNTIME_DIR/bin/llama-server
|
||
MODEL_FILE=$MODEL_DIR/$MODEL_STUB
|
||
MMPROJ_FILE=$MODEL_DIR/$MM_STUB
|
||
ALIAS=frank-zealot
|
||
BIND_ADDR=0.0.0.0
|
||
LISTEN_PORT=7771
|
||
CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
BATCH_SIZE=$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
UBATCH_SIZE=$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")
|
||
CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
|
||
SPEC_TYPE=$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")
|
||
SPEC_DRAFT_N_MAX=$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
VERBOSITY=$(sed -nE 's/^VERBOSITY=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
EOF
|
||
run env FRANK_ZEALOT_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
|
||
}
|
||
|
||
argv_value() {
|
||
grep -A1 -xF -- "$1" <<<"$output" | tail -1
|
||
}
|
||
|
||
# run_stage — exec the CURRENT $STAGE/test.env as-is. serve_argv REWRITES
|
||
# that file from the shipped env, so any test that mutates it (drop a line,
|
||
# append a drafter) must exec through this, not serve_argv — which silently
|
||
# undoes the mutation and makes the assertion a tautology.
|
||
run_stage() {
|
||
run env FRANK_ZEALOT_ENV_FILE="$STAGE/test.env" nvidia-modprobe=true bash "$SERVE"
|
||
}
|
||
|
||
@test "frank-zealot unit, serve script and env exist in scripts/" {
|
||
[ -f "$UNIT" ]
|
||
[ -f "$SERVE" ]
|
||
[ -f "$ENV_FILE" ]
|
||
}
|
||
|
||
@test "frank-zealot is boot-persistent, self-healing and unprivileged" {
|
||
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
||
grep -q '^Restart=' "$UNIT"
|
||
grep -q '^RestartSec=' "$UNIT"
|
||
grep -q '^User=dank$' "$UNIT"
|
||
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
||
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-zealot.sh$' "$UNIT"
|
||
grep -q 'FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env' "$UNIT"
|
||
}
|
||
|
||
@test "the shipped env points at the fork runtime and the Bonsai-2 PQ2_0 pair" {
|
||
# The payload path IS the endpoint's identity, and the runtime is part of
|
||
# it here: the file's custom tensor type 142 + prism.hadamard.* metadata
|
||
# make the PrismML build the only loader, so a retarget that moves the
|
||
# model but leaves the master build behind ships an endpoint that cannot
|
||
# load at all.
|
||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-prism/llama-server$' "$ENV_FILE"
|
||
grep -qE '^MODEL_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-PQ2_0\.gguf$' "$ENV_FILE"
|
||
grep -qE '^MMPROJ_FILE=/home/dank/models/Bonsai2-PQ2_0/Ternary-Bonsai-2-27B-mmproj-Q8_0\.gguf$' "$ENV_FILE"
|
||
}
|
||
|
||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||
# Anti-drift: the deploy handler checks files by its own defaults. If
|
||
# those ever differ from what the unit actually loads, a green deploy
|
||
# would be gating on a file nobody reads — and a missing projector would
|
||
# ship a blind endpoint.
|
||
local env_v dep_v path pair
|
||
for pair in "MODEL_FILE LLM_ZEALOT_MODEL_FILE" "MMPROJ_FILE LLM_ZEALOT_MMPROJ"; do
|
||
env_v="${pair%% *}"
|
||
dep_v="${pair##* }"
|
||
path="$(sed -nE "s/^${env_v}=(.+)$/\1/p" "$ENV_FILE")"
|
||
grep -qF -- "${dep_v}=\"\${${dep_v}:-${path}}\"" "$REMOTE_DEPLOY"
|
||
done
|
||
}
|
||
|
||
@test "two conversations share the pool at a 200704 window each" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --alias)" = "frank-zealot" ]
|
||
[ "$(argv_value --port)" = "7771" ]
|
||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||
[ "$(argv_value --model)" = "$MODEL_DIR/$MODEL_STUB" ]
|
||
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
||
# llama.cpp divides --ctx-size across slots, so the observable contract
|
||
# is the per-slot window. Frank's 200k cap is a per-conversation cap, so
|
||
# the pair is 401,408 / 2 — and the harness contextWindow must quote
|
||
# 200704, not the pool and not the model's native 262,144.
|
||
[ "$(argv_value --parallel)" -eq 2 ]
|
||
[ "$(argv_value --ctx-size)" -eq 401408 ]
|
||
local pool slots
|
||
pool="$(argv_value --ctx-size)"; slots="$(argv_value --parallel)"
|
||
[ $((pool % slots)) -eq 0 ]
|
||
[ $((pool / slots)) -eq 200704 ]
|
||
}
|
||
|
||
@test "the KV dtype pair follows the env file and flash-attn stays pinned on" {
|
||
# Asserted against the env rather than a literal so a retune moves one
|
||
# file, not two. The shipped pair is q8_0/q8_0: this payload bills
|
||
# 16 full-attention layers × 4 KV heads × 256 dim = 16,384 elements per
|
||
# token per cache, so f16/f16 would need 25,088 MiB of the 24,123 MiB card
|
||
# (measured: cudaMalloc OOM, engine fit 33,198 vs 23,558 free) while q8_0
|
||
# bills 34,816 B/token = 13,328.00 MiB at the 401,408 pool. The boot
|
||
# ledger, not this file, is the referee. flash-attn is not decoration:
|
||
# quantized V is only legal with it, and it must stay explicit so a
|
||
# future quant swap cannot silently fall back.
|
||
local env_k env_v
|
||
env_k="$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")"
|
||
env_v="$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")"
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --cache-type-k)" = "$env_k" ]
|
||
[ "$(argv_value --cache-type-v)" = "$env_v" ]
|
||
[ "$(argv_value --flash-attn)" = "on" ]
|
||
}
|
||
|
||
@test "log verbosity follows the env file" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
# The shipped 4 (debug) is what makes VRAM-fit forensics possible from
|
||
# the journal (KV layout, buffer sizes, graph capture).
|
||
[ "$(argv_value --log-verbosity)" = "4" ]
|
||
|
||
# And an env that names no verbosity still ships the engine default
|
||
# explicitly — the script's written-down 3, never a blank.
|
||
sed -i '/^VERBOSITY=/d' "$STAGE/test.env"
|
||
run_stage
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --log-verbosity)" = "3" ]
|
||
}
|
||
|
||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||
# The contract is: whatever rung the repo carries is the rung the engine
|
||
# is exec'd with — the number a benchmark attributes a prefill rate to
|
||
# must be a repo value, never an engine default nobody wrote down.
|
||
# Asserted against the env rather than a literal so a retune moves one
|
||
# file, not two.
|
||
local env_batch env_ubatch
|
||
env_batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
env_ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --batch-size)" = "$env_batch" ]
|
||
[ "$(argv_value --ubatch-size)" = "$env_ubatch" ]
|
||
# llama.cpp refuses ubatch > batch, so a retune that inverts the pair has
|
||
# to fail here instead of at boot on a live endpoint.
|
||
[ "$env_batch" -ge "$env_ubatch" ]
|
||
|
||
# And an env that names neither flag still ships an explicit pair: the
|
||
# serve script's fallbacks are written-down values, not blanks the engine
|
||
# fills in silently.
|
||
sed -i '/^BATCH_SIZE=/d;/^UBATCH_SIZE=/d' "$STAGE/test.env"
|
||
run_stage
|
||
[ "$status" -eq 0 ]
|
||
argv_value --batch-size | grep -qE '^[0-9]+$'
|
||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||
}
|
||
|
||
@test "speculation is off: no drafter flag reaches the argv" {
|
||
# Frank, 2026-10-07: MTP is off because it CANNOT be on — the Q6_K file
|
||
# carries no nextn/mtp tensors (verified from its tensor-name list), and
|
||
# MTP is a property of the file, not of the flags. The flags move as a
|
||
# set: SPEC_TYPE=none strips --spec-type, --spec-draft-n-max AND
|
||
# --spec-draft-model together, so a no-spec boot cannot carry half a
|
||
# spec pair (the ROCmFPX-era baseline arm segfaulted at load doing that).
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
! grep -qxF -- '--spec-type' <<<"$output"
|
||
! grep -qxF -- '--spec-draft-n-max' <<<"$output"
|
||
! grep -qxF -- '--spec-draft-model' <<<"$output"
|
||
grep -qE '^SPEC_TYPE=none$' "$ENV_FILE"
|
||
}
|
||
|
||
@test "naming a drafter ships the full MTP trio" {
|
||
# The opt-back-in path (e.g. a future Qwen3.5 -mtp build): SPEC_TYPE +
|
||
# SPEC_DRAFT_FILE must reach the engine as a complete set — a draft-mtp
|
||
# flag without its model file boots a drafter against the main weights and
|
||
# drafts nothing.
|
||
serve_argv
|
||
{
|
||
echo 'SPEC_TYPE=draft-mtp'
|
||
echo 'SPEC_DRAFT_N_MAX=3'
|
||
echo "SPEC_DRAFT_FILE=$MODEL_DIR/$DRAFT_STUB"
|
||
} >> "$STAGE/test.env"
|
||
run_stage
|
||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||
[ "$(argv_value --spec-draft-n-max)" = "3" ]
|
||
[ "$(argv_value --spec-draft-model)" = "$MODEL_DIR/$DRAFT_STUB" ]
|
||
}
|
||
|
||
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
||
# Qwen3.5-9B is natively multimodal, but the Q6_K file carries no vision
|
||
# tensors — the publisher's projector argument IS the vision capability:
|
||
# no flag, no eyes.
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --mmproj)" = "$MODEL_DIR/$MM_STUB" ]
|
||
|
||
# The documented opt-out must reach the engine as ABSENCE of the flag,
|
||
# not as an argument pointing at an empty string.
|
||
sed -i 's|^MMPROJ_FILE=.*|MMPROJ_FILE=|' "$STAGE/test.env"
|
||
run env FRANK_ZEALOT_ENV_FILE="$STAGE/test.env" nvidia-modprobe=true bash "$SERVE"
|
||
[ "$status" -eq 0 ]
|
||
! grep -qxF -- '--mmproj' <<<"$output"
|
||
}
|
||
|
||
@test "the fleet reasoning contract reaches the argv" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
# deepseek format = thinking lands in reasoning_content, the field the
|
||
# harness reads on this endpoint; --jinja is what makes
|
||
# chat_template_kwargs reach the template at all.
|
||
[ "$(argv_value --reasoning-format)" = "deepseek" ]
|
||
argv_value --jinja >/dev/null
|
||
grep -qxF -- '--jinja' <<<"$output"
|
||
}
|
||
|
||
@test "the serve script refuses a missing runtime, GGUF or projector" {
|
||
rm -f "$MODEL_DIR/$MM_STUB"
|
||
serve_argv
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"frank-zealot: missing"* ]]
|
||
|
||
rm -f "$RUNTIME_DIR/bin/llama-server"
|
||
serve_argv
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"frank-zealot: missing"* ]]
|
||
}
|
||
|
||
@test "the deploy handler gates on binary, GGUF and projector, then MainPID-hardens health" {
|
||
grep -q 'deploy_llm_zealot' "$REMOTE_DEPLOY"
|
||
grep -q 'LLM_ZEALOT_RUNTIME' "$REMOTE_DEPLOY"
|
||
grep -q 'LLM_ZEALOT_MODEL_FILE' "$REMOTE_DEPLOY"
|
||
grep -q 'LLM_ZEALOT_MMPROJ' "$REMOTE_DEPLOY"
|
||
grep -q 'relink_unit frank-zealot' "$REMOTE_DEPLOY"
|
||
grep -q 'systemctl enable frank-zealot.service' "$REMOTE_DEPLOY"
|
||
grep -q '127.0.0.1:7771/health' "$REMOTE_DEPLOY"
|
||
grep -q 'frank-zealot.service has no MainPID' "$REMOTE_DEPLOY"
|
||
}
|
||
|
||
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
|
||
# frank-dragoon's sweeps must not take down frank-zealot, whose
|
||
# process is llama-server on the SAME host.
|
||
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
|
||
local large
|
||
large="$(sed -n '/^deploy_llm_dragoon() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||
! grep -q 'pkill -x llama-server' <<<"$large"
|
||
}
|