Frank
156ea71879
frank-zealot shipped Restart=on-failure while its two siblings shipped always. exit 0 is a CLEAN stop to systemd: on-failure leaves the endpoint dark and the unit GREEN until the next boot — and frank-zealot is the compaction hub for all three endpoints, so its quiet death degrades every conversation in the fleet. tests/llm_units.bats gains the fleet parity seam (Restart=always, RestartSec 1..20, WantedBy=multi-user.target across scripts/frank-*.service); the per-unit seams drop their weaker Restart greps so the policy lives in one place. Runbook unit line updated.
282 lines
12 KiB
Bash
Executable File
282 lines
12 KiB
Bash
Executable File
# Seam: the frank-zealot llama.cpp root unit on dankclaw.
|
||
# frank-zealot is the RTX 3090 endpoint; since 2026-10-07 it serves
|
||
# HauhauCS's Qwen3.5-9B-Uncensored-HauhauCS-Aggressive (Q6_K, qwen35 hybrid:
|
||
# 32 layers, 8 full-attention + 24 Gated DeltaNet, no MTP tensors in the file)
|
||
# via the CUDA (sm_86) build of llama.cpp master: TWO 200,704-token
|
||
# conversations (--ctx-size 401408 --parallel 2), exact f16 K + f16 V under
|
||
# flash attention, the publisher's own BF16 vision projector, and MTP off —
|
||
# the file carries no nextn/mtp tensors, so there is no in-file head to draft
|
||
# with. Log verbosity 4 ships so the VRAM-fit forensics (KV layout, buffer
|
||
# sizes, graph capture) land in the journal.
|
||
# This seam runs the serve script against a stub engine and asserts on the
|
||
# argv it actually execs, so what is pinned is the command the operator gets.
|
||
|
||
load helpers
|
||
|
||
UNIT="$REPO_ROOT/scripts/frank-zealot.service"
|
||
SERVE="$REPO_ROOT/scripts/serve-frank-zealot.sh"
|
||
ENV_FILE="$REPO_ROOT/scripts/frank-zealot.env"
|
||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||
ENV_LARGE="$REPO_ROOT/scripts/frank-dragoon.env"
|
||
|
||
setup() {
|
||
STAGE="$(mktemp -d)"
|
||
RUNTIME_DIR="$STAGE/llama"
|
||
MODEL_DIR="$STAGE/models"
|
||
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR"
|
||
cat > "$RUNTIME_DIR/bin/llama-server" <<'STUB'
|
||
#!/usr/bin/env bash
|
||
printf '%s\n' "$@"
|
||
STUB
|
||
chmod +x "$RUNTIME_DIR/bin/llama-server" "$RUNTIME_DIR/bin" 2>/dev/null || chmod +x "$RUNTIME_DIR/bin/llama-server"
|
||
# The stub payload files carry the SAME basenames as the shipped env, so
|
||
# the seam exercises the real paths, and a payload rename moves one file
|
||
# (the env) instead of silently forking the test's filename constants.
|
||
MODEL_STUB="$(basename "$(sed -nE 's/^MODEL_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||
MM_STUB="$(basename "$(sed -nE 's/^MMPROJ_FILE=(.+)$/\1/p' "$ENV_FILE")")"
|
||
DRAFT_STUB="draft-assistant.gguf"
|
||
touch "$MODEL_DIR/$MODEL_STUB" "$MODEL_DIR/$MM_STUB" "$MODEL_DIR/$DRAFT_STUB"
|
||
}
|
||
|
||
teardown() {
|
||
rm -rf "$STAGE"
|
||
}
|
||
|
||
# serve_argv [env-file] — run the serve script and capture the engine's
|
||
# argv, one token per line, in $output.
|
||
serve_argv() {
|
||
local envf="${1:-$STAGE/test.env}"
|
||
cat > "$envf" <<EOF
|
||
LLAMA_BIN=$RUNTIME_DIR/bin/llama-server
|
||
MODEL_FILE=$MODEL_DIR/$MODEL_STUB
|
||
MMPROJ_FILE=$MODEL_DIR/$MM_STUB
|
||
ALIAS=frank-zealot
|
||
BIND_ADDR=0.0.0.0
|
||
LISTEN_PORT=7771
|
||
CTX_SIZE=$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
PARALLEL=$(sed -nE 's/^PARALLEL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
BATCH_SIZE=$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
UBATCH_SIZE=$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
CACHE_TYPE_K=$(sed -nE 's/^CACHE_TYPE_K=(.+)$/\1/p' "$ENV_FILE")
|
||
CACHE_TYPE_V=$(sed -nE 's/^CACHE_TYPE_V=(.+)$/\1/p' "$ENV_FILE")
|
||
SPEC_TYPE=$(sed -nE 's/^SPEC_TYPE=(.+)$/\1/p' "$ENV_FILE")
|
||
SPEC_DRAFT_N_MAX=$(sed -nE 's/^SPEC_DRAFT_N_MAX=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
VERBOSITY=$(sed -nE 's/^VERBOSITY=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
NGL=$(sed -nE 's/^NGL=([0-9]+)$/\1/p' "$ENV_FILE")
|
||
EOF
|
||
run env FRANK_ZEALOT_ENV_FILE="$envf" nvidia-modprobe=true bash "$SERVE"
|
||
}
|
||
|
||
argv_value() {
|
||
grep -A1 -xF -- "$1" <<<"$output" | tail -1
|
||
}
|
||
|
||
# run_stage — exec the CURRENT $STAGE/test.env as-is. serve_argv REWRITES
|
||
# that file from the shipped env, so any test that mutates it (drop a line,
|
||
# append a drafter) must exec through this, not serve_argv — which silently
|
||
# undoes the mutation and makes the assertion a tautology.
|
||
run_stage() {
|
||
run env FRANK_ZEALOT_ENV_FILE="$STAGE/test.env" nvidia-modprobe=true bash "$SERVE"
|
||
}
|
||
|
||
@test "frank-zealot unit, serve script and env exist in scripts/" {
|
||
[ -f "$UNIT" ]
|
||
[ -f "$SERVE" ]
|
||
[ -f "$ENV_FILE" ]
|
||
}
|
||
|
||
@test "frank-zealot runs unprivileged with the memlock budget and its env file" {
|
||
# Restart policy, RestartSec and WantedBy are the FLEET parity seam
|
||
# (tests/llm_units.bats). What is endpoint-specific: CUDA pins its
|
||
# staging buffers, so the memlock budget is load-bearing even though the
|
||
# weights are copied into VRAM, and the unit must name the serve script
|
||
# and the env file that sizes it.
|
||
grep -q '^User=dank$' "$UNIT"
|
||
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
||
grep -q '^ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-zealot.sh$' "$UNIT"
|
||
grep -q 'FRANK_ZEALOT_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-zealot.env' "$UNIT"
|
||
}
|
||
|
||
@test "the shipped env points at the llama.cpp runtime and the Qwen3.5-9B Q6_K GGUF" {
|
||
# The payload path IS the endpoint's identity.
|
||
grep -qE '^LLAMA_BIN=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server$' "$ENV_FILE"
|
||
grep -qE '^MODEL_FILE=/home/dank/models/Qwen35-9B-HauhauCS-Q6K/Qwen3\.5-9B-Uncensored-HauhauCS-Aggressive-Q6_K\.gguf$' "$ENV_FILE"
|
||
}
|
||
|
||
@test "the deployer gates on exactly the payload paths the env file ships" {
|
||
# Anti-drift: the deploy handler checks files by its own defaults. If
|
||
# those ever differ from what the unit actually loads, a green deploy
|
||
# would be gating on a file nobody reads — and a missing projector would
|
||
# ship a blind endpoint.
|
||
local env_v dep_v path pair
|
||
for pair in "MODEL_FILE LLM_ZEALOT_MODEL_FILE" "MMPROJ_FILE LLM_ZEALOT_MMPROJ"; do
|
||
env_v="${pair%% *}"
|
||
dep_v="${pair##* }"
|
||
path="$(sed -nE "s/^${env_v}=(.+)$/\1/p" "$ENV_FILE")"
|
||
grep -qF -- "${dep_v}=\"\${${dep_v}:-${path}}\"" "$REMOTE_DEPLOY"
|
||
done
|
||
}
|
||
|
||
@test "two conversations share the pool at a 200704 window each" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --alias)" = "frank-zealot" ]
|
||
[ "$(argv_value --port)" = "7771" ]
|
||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||
[ "$(argv_value --model)" = "$MODEL_DIR/$MODEL_STUB" ]
|
||
[ "$(argv_value --n-gpu-layers)" -gt 0 ]
|
||
# llama.cpp divides --ctx-size across slots, so the observable contract
|
||
# is the per-slot window. Frank's 200k cap is a per-conversation cap, so
|
||
# the pair is 401,408 / 2 — and the harness contextWindow must quote
|
||
# 200704, not the pool and not the model's native 262,144.
|
||
[ "$(argv_value --parallel)" -eq 2 ]
|
||
[ "$(argv_value --ctx-size)" -eq 401408 ]
|
||
local pool slots
|
||
pool="$(argv_value --ctx-size)"; slots="$(argv_value --parallel)"
|
||
[ $((pool % slots)) -eq 0 ]
|
||
[ $((pool / slots)) -eq 200704 ]
|
||
}
|
||
|
||
@test "the KV contract is exact f16 K + f16 V, flash-attn on" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
# Frank, 2026-10-07: the 9B rung funds unquantized KV (the 26B MoE era
|
||
# shipped q8_0 V because the card was full). qwen35 bills the pool on the
|
||
# 8 full-attention layers only (8 × 4 KV heads × 256 dim = 8,192
|
||
# elements/token ⇒ 32 KiB/token at f16/f16, 12.25 GiB at the 401,408
|
||
# pool); the 24 DeltaNet layers cost a fixed recurrent state per slot.
|
||
# The boot ledger, not this file, is the referee. flash-attn stays pinned:
|
||
# any quantized-V fallback (q8_0 = this build's FP8 class) requires it, and
|
||
# the build carries GGML_CUDA_FA_ALL_QUANTS=ON.
|
||
[ "$(argv_value --cache-type-k)" = "f16" ]
|
||
[ "$(argv_value --cache-type-v)" = "f16" ]
|
||
[ "$(argv_value --flash-attn)" = "on" ]
|
||
}
|
||
|
||
@test "log verbosity follows the env file" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
# The shipped 4 (debug) is what makes VRAM-fit forensics possible from
|
||
# the journal (KV layout, buffer sizes, graph capture).
|
||
[ "$(argv_value --log-verbosity)" = "4" ]
|
||
|
||
# And an env that names no verbosity still ships the engine default
|
||
# explicitly — the script's written-down 3, never a blank.
|
||
sed -i '/^VERBOSITY=/d' "$STAGE/test.env"
|
||
run_stage
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --log-verbosity)" = "3" ]
|
||
}
|
||
|
||
@test "the compute-buffer pair follows the env file and stays self-consistent" {
|
||
# The contract is: whatever rung the repo carries is the rung the engine
|
||
# is exec'd with — the number a benchmark attributes a prefill rate to
|
||
# must be a repo value, never an engine default nobody wrote down.
|
||
# Asserted against the env rather than a literal so a retune moves one
|
||
# file, not two.
|
||
local env_batch env_ubatch
|
||
env_batch="$(sed -nE 's/^BATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
env_ubatch="$(sed -nE 's/^UBATCH_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --batch-size)" = "$env_batch" ]
|
||
[ "$(argv_value --ubatch-size)" = "$env_ubatch" ]
|
||
# llama.cpp refuses ubatch > batch, so a retune that inverts the pair has
|
||
# to fail here instead of at boot on a live endpoint.
|
||
[ "$env_batch" -ge "$env_ubatch" ]
|
||
|
||
# And an env that names neither flag still ships an explicit pair: the
|
||
# serve script's fallbacks are written-down values, not blanks the engine
|
||
# fills in silently.
|
||
sed -i '/^BATCH_SIZE=/d;/^UBATCH_SIZE=/d' "$STAGE/test.env"
|
||
run_stage
|
||
[ "$status" -eq 0 ]
|
||
argv_value --batch-size | grep -qE '^[0-9]+$'
|
||
argv_value --ubatch-size | grep -qE '^[0-9]+$'
|
||
}
|
||
|
||
@test "speculation is off: no drafter flag reaches the argv" {
|
||
# Frank, 2026-10-07: MTP is off because it CANNOT be on — the Q6_K file
|
||
# carries no nextn/mtp tensors (verified from its tensor-name list), and
|
||
# MTP is a property of the file, not of the flags. The flags move as a
|
||
# set: SPEC_TYPE=none strips --spec-type, --spec-draft-n-max AND
|
||
# --spec-draft-model together, so a no-spec boot cannot carry half a
|
||
# spec pair (the ROCmFPX-era baseline arm segfaulted at load doing that).
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
! grep -qxF -- '--spec-type' <<<"$output"
|
||
! grep -qxF -- '--spec-draft-n-max' <<<"$output"
|
||
! grep -qxF -- '--spec-draft-model' <<<"$output"
|
||
grep -qE '^SPEC_TYPE=none$' "$ENV_FILE"
|
||
}
|
||
|
||
@test "naming a drafter ships the full MTP trio" {
|
||
# The opt-back-in path (e.g. a future Qwen3.5 -mtp build): SPEC_TYPE +
|
||
# SPEC_DRAFT_FILE must reach the engine as a complete set — a draft-mtp
|
||
# flag without its model file boots a drafter against the main weights and
|
||
# drafts nothing.
|
||
serve_argv
|
||
{
|
||
echo 'SPEC_TYPE=draft-mtp'
|
||
echo 'SPEC_DRAFT_N_MAX=3'
|
||
echo "SPEC_DRAFT_FILE=$MODEL_DIR/$DRAFT_STUB"
|
||
} >> "$STAGE/test.env"
|
||
run_stage
|
||
[ "$(argv_value --spec-type)" = "draft-mtp" ]
|
||
[ "$(argv_value --spec-draft-n-max)" = "3" ]
|
||
[ "$(argv_value --spec-draft-model)" = "$MODEL_DIR/$DRAFT_STUB" ]
|
||
}
|
||
|
||
@test "vision ships by default and MMPROJ_FILE empty opts out of the image path" {
|
||
# Qwen3.5-9B is natively multimodal, but the Q6_K file carries no vision
|
||
# tensors — the publisher's projector argument IS the vision capability:
|
||
# no flag, no eyes.
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
[ "$(argv_value --mmproj)" = "$MODEL_DIR/$MM_STUB" ]
|
||
|
||
# The documented opt-out must reach the engine as ABSENCE of the flag,
|
||
# not as an argument pointing at an empty string.
|
||
sed -i 's|^MMPROJ_FILE=.*|MMPROJ_FILE=|' "$STAGE/test.env"
|
||
run env FRANK_ZEALOT_ENV_FILE="$STAGE/test.env" nvidia-modprobe=true bash "$SERVE"
|
||
[ "$status" -eq 0 ]
|
||
! grep -qxF -- '--mmproj' <<<"$output"
|
||
}
|
||
|
||
@test "the fleet reasoning contract reaches the argv" {
|
||
serve_argv
|
||
[ "$status" -eq 0 ]
|
||
# deepseek format = thinking lands in reasoning_content, the field the
|
||
# harness reads on this endpoint; --jinja is what makes
|
||
# chat_template_kwargs reach the template at all.
|
||
[ "$(argv_value --reasoning-format)" = "deepseek" ]
|
||
argv_value --jinja >/dev/null
|
||
grep -qxF -- '--jinja' <<<"$output"
|
||
}
|
||
|
||
@test "the serve script refuses a missing runtime, GGUF or projector" {
|
||
# The refusal is the shared gate library's: it names the artifact class,
|
||
# the exact path, and the script that produces it.
|
||
rm -f "$MODEL_DIR/$MM_STUB"
|
||
serve_argv
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"frank-zealot:"* ]]
|
||
[[ "$output" == *"mmproj"*"missing"*"$MM_STUB"* ]]
|
||
[[ "$output" == *"download-frank-zealot.sh"* ]]
|
||
|
||
rm -f "$RUNTIME_DIR/bin/llama-server"
|
||
serve_argv
|
||
[ "$status" -ne 0 ]
|
||
[[ "$output" == *"runtime"*"missing"*"llama-server"* ]]
|
||
[[ "$output" == *"build-llama-master-cuda.sh"* ]]
|
||
}
|
||
|
||
@test "the two endpoints keep separate ports and the large handler never pkills llama-server" {
|
||
# frank-dragoon's sweeps must not take down frank-zealot, whose
|
||
# process is llama-server on the SAME host.
|
||
grep -q '^LISTEN_PORT=7771$' "$ENV_FILE"
|
||
local large
|
||
large="$(sed -n '/^deploy_llm_dragoon() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||
! grep -q 'pkill -x llama-server' <<<"$large"
|
||
}
|