Frank
7d3dd33de0
Frank's order: replace frank-large with kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF's Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf, following that card. The file is ftype 114 — 339 GGML_TYPE_Q6_0_ROCMFPX (102) + 159 GGML_TYPE_Q8_0_ROCMFPX (103) + 353 F32 over 851 tensors, qwen35/64 layers/full_attention_interval 4/4 KV heads/262,144 window — read out of its own header. The pinned ROCmFPX build already knows both types (ggml.h 102/103 plus convert/cpy/getrows/mmq/fattn device paths for 102), so no rebuild: the fork stays at 36e9acd and the payload directory is the only new host state. Card compliance, all already our flags: Q4_0 draft head (not Q8_0), --spec-draft-n-max 4 (the knee on both cards), --spec-draft-ngl 99, --jinja, --fit off, -fa on, -dio, one slot at full window, thinking with the card's own sampling triple (which the file also embeds as its general.sampling defaults). Kept from the deployed shape: the BF16 projector (this repo ships only a Q8_0 one; BF16 is the file verified 4/4 on spatial ground truth here) and the byte-identical MTP head, which the new script hardlinks instead of re-fetching 1.6 GiB. Franksdank-Runbook: serverAppConfigs-docs/LLM/frank-large-rocmfpx-q6agent-uncensored-mtp.md
282 lines
13 KiB
Bash
Executable File
282 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bats
|
|
# Seam: frank-large serves kingjones777's UNCENSORED Qwen3.8-27B
|
|
# Q6_0_ROCMFPX_AGENT (a ftype-114 ROCmFPX file) with the repo's MTP draft head
|
|
# on the Strix Halo iGPU, via the ROCmFPX fork of llama.cpp — one session at
|
|
# the native 262,144 window.
|
|
#
|
|
# This seam exists because the format is the constraint: GGML_TYPE_Q6_0_ROCMFPX
|
|
# (tensor type 102) and GGML_TYPE_Q8_0_ROCMFPX (103) are understood by the
|
|
# ROCmFPX fork and by nothing else on this host, so the runtime, the payload
|
|
# and the deployer's gate have to agree
|
|
# on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and
|
|
# its deployer twin disagreed during a cutover and the endpoint crash-looped;
|
|
# both sides are pinned here.
|
|
#
|
|
# Everything is tested from what an operator or a supervisor can observe — the
|
|
# command line, the exit code, the deployer's error text — never from anything
|
|
# inside llama.cpp.
|
|
setup() {
|
|
load helpers
|
|
|
|
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
|
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
|
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
|
DEPLOYER="$REPO_ROOT/scripts/deploy-remote.sh"
|
|
BUILD="$REPO_ROOT/scripts/build-rocmfpx.sh"
|
|
RUNTIME_STUB="/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server"
|
|
MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf"
|
|
DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf"
|
|
MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf"
|
|
|
|
TEST_TMP="$(mktemp -d)"
|
|
SRC="$TEST_TMP/src"
|
|
mkdir -p "$SRC"
|
|
git -C "$SRC" init -q
|
|
git -C "$SRC" -c user.email=t -c user.name=t commit -q --allow-empty -m x
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TEST_TMP"
|
|
}
|
|
|
|
# make_stub — an executable that prints its whole argv on one line, so
|
|
# "what command line would the supervisor run" is observable output.
|
|
make_stub() {
|
|
cat > "$TEST_TMP/bin" <<'STUB'
|
|
#!/bin/sh
|
|
printf '%s\n' "$*"
|
|
STUB
|
|
chmod +x "$TEST_TMP/bin"
|
|
}
|
|
|
|
@test "the env file defines the deployment values" {
|
|
grep -qE '^LLAMA_BIN=/' "$ENV_FILE"
|
|
grep -qE '^MODEL_FILE=/' "$ENV_FILE"
|
|
grep -qE '^DRAFT_FILE=/' "$ENV_FILE"
|
|
grep -qE '^ALIAS=[a-z0-9-]+$' "$ENV_FILE"
|
|
grep -qE '^LISTEN_PORT=[0-9]+$' "$ENV_FILE"
|
|
grep -qE '^CTX_SIZE=[0-9]+$' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the serve script fails when the runtime is missing" {
|
|
# Only the model, the head and the projector exist; the fork was never
|
|
# built. llama.cpp is not reached, so this fails fast and says what to
|
|
# run instead of failing at bind time.
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/nope/llama-server" \
|
|
"MODEL_FILE=$TEST_TMP/model.gguf" "DRAFT_FILE=$TEST_TMP/draft.gguf" \
|
|
"MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"build it with scripts/build-rocmfpx.sh"* ]]
|
|
}
|
|
|
|
@test "the serve script refuses to start without the model file" {
|
|
make_stub
|
|
touch "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/absent.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"is not readable"* ]]
|
|
[[ "$output" == *"download-frank-large-kingjones-uncensored.sh"* ]]
|
|
}
|
|
|
|
@test "the serve script refuses to start without the MTP draft head" {
|
|
make_stub
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/absent.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"MTP draft head"* ]]
|
|
}
|
|
|
|
@test "the serve script refuses a missing mmproj unless vision is opted out" {
|
|
make_stub
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/absent.gguf" "$SERVE"
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"set MMPROJ_FILE='' for text-only"* ]]
|
|
# The documented opt-out must actually start, without a --mmproj argument.
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=" "$SERVE"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" != *"--mmproj"* ]]
|
|
}
|
|
|
|
@test "the serve script fails when LLAMA_BIN is empty" {
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env LLAMA_BIN= "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"LLAMA_BIN is empty"* ]]
|
|
}
|
|
|
|
@test "the shipped env points at the ROCmFPX runtime and the AGENT payload" {
|
|
grep -qE "^LLAMA_BIN=$RUNTIME_STUB$" "$ENV_FILE"
|
|
grep -qE "^MODEL_FILE=$MODEL_STUB$" "$ENV_FILE"
|
|
grep -qE "^DRAFT_FILE=$DRAFT_STUB$" "$ENV_FILE"
|
|
grep -qE "^MMPROJ_FILE=$MMPROJ_STUB$" "$ENV_FILE"
|
|
# HIP reads bare HOST/TEMP/GPU_MAX_HEAP_SIZE out of the environment that
|
|
# `set -a` creates; a tunable named HOST leaks into the runtime.
|
|
! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
|
}
|
|
|
|
@test "the argv contract keeps the fleet's port and alias" {
|
|
make_stub
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" \
|
|
ALIAS=frank-large BIND_ADDR=127.0.0.1 LISTEN_PORT=7770 "$SERVE"
|
|
[ "$status" -eq 0 ]
|
|
# The caller's values must survive the env file, or a single-run operator
|
|
# override silently does nothing.
|
|
[[ "$output" == *"--host 127.0.0.1"* ]]
|
|
[[ "$output" == *"--port 7770"* ]]
|
|
[[ "$output" == *"--alias frank-large"* ]]
|
|
# Vision is wired: the projector is the whole vision tower.
|
|
[[ "$output" == *"--mmproj $TEST_TMP/mmproj.gguf"* ]]
|
|
}
|
|
|
|
@test "the shipped argv is one full native window on one session" {
|
|
grep -qE '^CTX_SIZE=262144$' "$ENV_FILE"
|
|
grep -qE '^PARALLEL=1$' "$ENV_FILE"
|
|
# 262144 IS the model's trained window (qwen35.context_length in the GGUF),
|
|
# so 1 slot gets the whole thing — no truncation, no yarn scaling.
|
|
#
|
|
# Budget, from the GGUF header: 23.15 GiB of weights + 1.57 GiB draft head
|
|
# + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x
|
|
# 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB
|
|
# of DeltaNet state + a couple GiB of compute buffers, against the iGPU's
|
|
# ~114 GiB GTT aperture. Change CTX_SIZE and this arithmetic, not a boot
|
|
# log, is what tells you it no longer fits.
|
|
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
|
kv_gib=$((ctx * 65536 / 1073741824))
|
|
total_gib=$((24 + 2 + 1 + kv_gib + 3))
|
|
[ "$total_gib" -le 114 ]
|
|
}
|
|
|
|
@test "the MTP draft contract matches the model card's measured optimum" {
|
|
make_stub
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -eq 0 ]
|
|
# The head is the model's own nextn layer, loaded as a separate draft
|
|
# model because this checkpoint ships it standalone (no blk.<n>.nextn.*
|
|
# tensors in the target file).
|
|
[[ "$output" == *"--spec-type draft-mtp"* ]]
|
|
[[ "$output" == *"--model-draft $TEST_TMP/draft.gguf"* ]]
|
|
# n-max 4 is the measured optimum; n-max 8 measured WORSE (acceptance
|
|
# fell to 0.757 and the extra verification cost outran the savings), so
|
|
# the flag is a measurement, not a wish.
|
|
[[ "$output" == *"--spec-draft-n-max 4"* ]]
|
|
[[ "$output" == *"--spec-draft-n-min 0"* ]]
|
|
[[ "$output" == *"--spec-draft-p-min 0.0"* ]]
|
|
# The draft runs on the iGPU beside the target: no device-to-device hop
|
|
# on a link that measures single-digit GB/s.
|
|
[[ "$output" == *"--spec-draft-device ROCm0"* ]]
|
|
[[ "$output" == *"--spec-draft-ngl 99"* ]]
|
|
}
|
|
|
|
@test "KV and placement stay unquantized with flash attention on" {
|
|
make_stub
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -eq 0 ]
|
|
[[ "$output" == *"--cache-type-k f16"* ]]
|
|
[[ "$output" == *"--cache-type-v f16"* ]]
|
|
# ROCmFPx quantized KV has no kernel on this branch, and the endpoint has
|
|
# the aperture to keep KV unquantized anyway.
|
|
[[ "$output" == *"--flash-attn on"* ]]
|
|
# --fit's placement heuristics do not know that 311 tensors of type 103
|
|
# belong on the iGPU, so the whole stack is placed explicitly instead.
|
|
[[ "$output" == *"--fit off"* ]]
|
|
[[ "$output" == *"--n-gpu-layers 999"* ]]
|
|
}
|
|
|
|
@test "the client contract wires thinking and template kwargs" {
|
|
make_stub
|
|
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
|
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
|
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
|
[ "$status" -eq 0 ]
|
|
# Without --jinja, llama.cpp silently ignores the harness's
|
|
# chat_template_kwargs, so thinking control disappears from the API.
|
|
[[ "$output" == *"--jinja"* ]]
|
|
# The harness reads `reasoning_content` for this endpoint, which is what
|
|
# the deepseek emission format produces (vLLM's `reasoning` is not it).
|
|
[[ "$output" == *"--reasoning-format deepseek"* ]]
|
|
# The template needs an unbounded thinking budget.
|
|
[[ "$output" == *"--reasoning-budget -1"* ]]
|
|
}
|
|
|
|
@test "the deployer gates exactly the paths the serve script opens" {
|
|
# A gate that checks a different file than the engine opens is the
|
|
# 2026-10-01 crash-loop: deploy passed, the unit restart-looped. Read the
|
|
# engine's own paths out of the env file and require the deployer's
|
|
# defaults to name the same ones.
|
|
for pair in "LLAMA_BIN:LLM_LARGE_RUNTIME" \
|
|
"MODEL_FILE:LLM_LARGE_MODEL" \
|
|
"DRAFT_FILE:LLM_LARGE_DRAFT" \
|
|
"MMPROJ_FILE:LLM_LARGE_MMPROJ"; do
|
|
env_key="${pair%%:*}"
|
|
dep_key="${pair##*:}"
|
|
want="$(sed -nE "s|^${env_key}=(.*)\$|\1|p" "$ENV_FILE")"
|
|
[ -n "$want" ]
|
|
got="$(grep -oE "${dep_key}:-[^}]+" "$DEPLOYER" | head -1 | sed "s|^${dep_key}:-||")"
|
|
[ "$got" = "$want" ]
|
|
done
|
|
}
|
|
|
|
@test "the deployer points a missing runtime at the fork build script" {
|
|
grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER"
|
|
grep -qE 'fetch it: bash scripts/download-frank-large-kingjones-uncensored\.sh' "$DEPLOYER"
|
|
}
|
|
|
|
@test "the stray-process sweep cannot take frank-small down with frank-large" {
|
|
# frank-small serves a different GGUF over llama-server on the SAME host.
|
|
# A sweep keyed on the binary name (`pkill -f llama-server`) would kill
|
|
# the 3090 endpoint too, so it is keyed on frank-large's own payload file.
|
|
grep -q "pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT'" "$DEPLOYER"
|
|
! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
|
|
}
|
|
|
|
@test "the build script refuses a source tree that is not the pinned commit" {
|
|
# The pin is load-bearing: the older builds of this fork line segfault in
|
|
# ggml_compute_forward_set_rows on this exact payload. A runtime that
|
|
# differs from what was qualified must be heard about at deploy time.
|
|
wrong="$(git -C "$SRC" rev-parse HEAD)"
|
|
run env ROCMFPX_SRC_DIR="$SRC" ROCMFPX_COMMIT="ffff$wrong" \
|
|
"$BUILD" --verify
|
|
[ "$status" -ne 0 ]
|
|
[[ "$output" == *"ROCMFPX_COMMIT"*mismatch* ]]
|
|
}
|
|
|
|
@test "the build script pins a real commit and the gfx1151 target" {
|
|
grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
|
|
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
|
grep -qE 'DGPU_TARGETS=' "$BUILD"
|
|
# The webui asset fetch half-failed on this box and aborted the link at
|
|
# 98%; the build must not depend on it.
|
|
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
|
}
|
|
|
|
@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
|
|
grep -qE '^Restart=(always|on-failure)$' "$UNIT"
|
|
grep -qE '^WantedBy=' "$UNIT"
|
|
grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT"
|
|
grep -qE '^User=dank$' "$UNIT"
|
|
# The payload is 26 GiB of mmap'd weights plus pinned staging buffers; an
|
|
# RLIMIT smaller than the pinned set fails the load in a way that looks
|
|
# like an engine crash.
|
|
grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
|
|
}
|
|
|
|
@test "the unit points at no retired frank-large runtime path" {
|
|
# The host keeps every retired payload and build on disk, so a stale path
|
|
# in a unit still resolves — to a runtime nobody has measured in weeks.
|
|
! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
|
|
! grep -q 'halogen' "$UNIT"
|
|
}
|