frank-large: ROCmFPX llama.cpp on the Q8_0_ROCMFPX_AGENT file, MTP drafting
The repo's highest-quality rung is ftype 115: 311 tensors of GGML_TYPE_Q8_0_ROCMFPX, a type only the ROCmFPX fork reads, so the runtime moves from gufo to scripts/build-rocmfpx.sh's pinned fork build (36e9acd) — the first build on this host measured to load this file without the 2026-08-25 set_rows segfault. One session at the native 262,144 window, f16 KV (64 KiB/token, 16 GiB), the repo's Q4_0 MTP head drafting 4 tokens, plus the bundled BF16 projector so the endpoint has vision again. The serve script now honors caller environment overrides over the env file (a plain source let the file win, which made single-run overrides no-ops) and defaults apply only when a value is unset, so an explicit LLAMA_BIN='' is the error it reads as.
This commit is contained in:
Frank committed
1 parent
d82c5c03b7
commit
125d04967f
6 files changed
+608
-495
No files matched your search
+242
-300
@@ -1,338 +1,280 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: the frank-large root unit on dankclaw.
|
||||
# Seam: frank-large serves kingjones777's Qwen3.8-27B Q8_0_ROCMFPX_AGENT (a
|
||||
# ftype-115 ROCmFPX file) with the repo's MTP draft head on the Strix Halo
|
||||
# iGPU, via the ROCmFPX fork of llama.cpp — one session at the native
|
||||
# 262,144 window.
|
||||
#
|
||||
# Since 2026-09-28 this endpoint is served by gufo (gufo-org/gufo), the
|
||||
# Strix Halo inference engine built from the pinned v0.1.1 tag, serving
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) UD-Q8_K_XL with the z-lab
|
||||
# DFlash2 Q4_K_M drafter. Two changes landed on this endpoint that day: the
|
||||
# engine (from the stock llama.cpp prebuilt) and the payload (from
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL). Port, alias and client contract did
|
||||
# not move.
|
||||
# This seam exists because the format is the constraint: GGML_TYPE_Q8_0_ROCMFPX
|
||||
# (tensor type 103) is understood by the ROCmFPX fork and by nothing else on
|
||||
# this host, so the runtime, the payload and the deployer's gate have to agree
|
||||
# on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and
|
||||
# its deployer twin disagreed during a cutover and the endpoint crash-looped;
|
||||
# both sides are pinned here.
|
||||
#
|
||||
# Most of this seam runs the serve script against a stub engine and asserts
|
||||
# on the argv it actually execs, so what is pinned is the command the
|
||||
# operator gets rather than the text of the script: a tunable renamed,
|
||||
# dropped, or silently defaulted away fails here the way it would fail a
|
||||
# deployment.
|
||||
load helpers
|
||||
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
||||
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
||||
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
||||
FETCH="$REPO_ROOT/scripts/download-frank-large-q4.sh"
|
||||
DRAFTER_FETCH="$REPO_ROOT/scripts/download-frank-large.sh"
|
||||
BUILD="$REPO_ROOT/scripts/build-gufo.sh"
|
||||
REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
# The deployed target's filename, kept here so a rename in the env file and a
|
||||
# rename in this stub cannot drift apart unnoticed.
|
||||
TARGET_STUB="Qwen3.8-27B-UD-Q4_K_XL.gguf"
|
||||
# Everything is tested from what an operator or a supervisor can observe — the
|
||||
# command line, the exit code, the deployer's error text — never from anything
|
||||
# inside llama.cpp.
|
||||
setup() {
|
||||
STAGE="$(mktemp -d)"
|
||||
RUNTIME_DIR="$STAGE/gufo"
|
||||
MODEL_DIR="$STAGE/models"
|
||||
DRAFTER_DIR="$STAGE/draft"
|
||||
mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR" "$DRAFTER_DIR"
|
||||
# A stub engine: the serve script execs it, so $@ is exactly what gufo
|
||||
# would be invoked with.
|
||||
cat > "$RUNTIME_DIR/bin/gufo" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$@"
|
||||
STUB
|
||||
chmod +x "$RUNTIME_DIR/bin/gufo"
|
||||
# The target and the drafter live in SEPARATE directories since the
|
||||
# 2026-09-28 payload swap: the target came from a different repository,
|
||||
# the DFlash2 drafter did not move. A stub that colocated them would
|
||||
# never notice DRAFTER_DIR going stale.
|
||||
: > "$MODEL_DIR/$TARGET_STUB"
|
||||
: > "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
STUB_ENV="$STAGE/stub.env"
|
||||
cat > "$STUB_ENV" <<EOF
|
||||
RUNTIME_DIR=$RUNTIME_DIR
|
||||
MODEL_DIR=$MODEL_DIR
|
||||
DRAFTER_DIR=$DRAFTER_DIR
|
||||
EOF
|
||||
load helpers
|
||||
|
||||
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
||||
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
||||
DEPLOYER="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
BUILD="$REPO_ROOT/scripts/build-rocmfpx.sh"
|
||||
RUNTIME_STUB="/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server"
|
||||
MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf"
|
||||
DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf"
|
||||
MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf"
|
||||
|
||||
TEST_TMP="$(mktemp -d)"
|
||||
SRC="$TEST_TMP/src"
|
||||
mkdir -p "$SRC"
|
||||
git -C "$SRC" init -q
|
||||
git -C "$SRC" -c user.email=t -c user.name=t commit -q --allow-empty -m x
|
||||
}
|
||||
|
||||
teardown() {
|
||||
if [ -n "${STAGE:-}" ]; then
|
||||
rm -rf "$STAGE"
|
||||
fi
|
||||
rm -rf "$TEST_TMP"
|
||||
}
|
||||
|
||||
# serve_argv [env-file] — run the serve script and capture the engine's argv,
|
||||
# one token per line, in $output. No argument exercises the built-in defaults
|
||||
# with the stub's paths.
|
||||
serve_argv() {
|
||||
local envfile="${1:-$STUB_ENV}"
|
||||
if [ -n "$envfile" ]; then
|
||||
FRANK_LARGE_ENV_FILE="$envfile" run bash "$SERVE"
|
||||
else
|
||||
run env FRANK_LARGE_ENV_FILE="$STAGE/absent.env" bash "$SERVE"
|
||||
fi
|
||||
# make_stub — an executable that prints its whole argv on one line, so
|
||||
# "what command line would the supervisor run" is observable output.
|
||||
make_stub() {
|
||||
cat > "$TEST_TMP/bin" <<'STUB'
|
||||
#!/bin/sh
|
||||
printf '%s\n' "$*"
|
||||
STUB
|
||||
chmod +x "$TEST_TMP/bin"
|
||||
}
|
||||
|
||||
# argv_has <token> — exact token match, so --port never satisfies a test
|
||||
# looking for a different flag.
|
||||
argv_has() {
|
||||
grep -qxF -- "$1" <<<"$output"
|
||||
@test "the env file defines the deployment values" {
|
||||
grep -qE '^LLAMA_BIN=/' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/' "$ENV_FILE"
|
||||
grep -qE '^DRAFT_FILE=/' "$ENV_FILE"
|
||||
grep -qE '^ALIAS=[a-z0-9-]+$' "$ENV_FILE"
|
||||
grep -qE '^LISTEN_PORT=[0-9]+$' "$ENV_FILE"
|
||||
grep -qE '^CTX_SIZE=[0-9]+$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
# argv_value <flag> — the token following <flag>, empty if the flag is absent.
|
||||
argv_value() {
|
||||
local i
|
||||
i="$(grep -nx -- "$1" <<<"$output" | cut -d: -f1)"
|
||||
[ -n "$i" ] || return 0
|
||||
sed -n "$((i + 1))p" <<<"$output"
|
||||
}
|
||||
|
||||
@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" {
|
||||
[ -f "$ENV_FILE" ]
|
||||
[ -f "$UNIT" ]
|
||||
[ -x "$SERVE" ]
|
||||
[ -x "$BUILD" ]
|
||||
}
|
||||
|
||||
@test "frank-large is boot-persistent, self-healing and unprivileged" {
|
||||
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
||||
grep -q '^Restart=on-failure$' "$UNIT"
|
||||
grep -q '^RestartSec=' "$UNIT"
|
||||
grep -q '^User=dank$' "$UNIT"
|
||||
# The engine pins buffers; the memlock budget must be unbounded or the
|
||||
# load fails on the RLIMIT.
|
||||
grep -q '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
# The unit runs the repo's own serve script with the repo's env.
|
||||
grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT"
|
||||
grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the fleet endpoint and alias survive the cutover" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
# frank-large at :7770, whatever engine answers it.
|
||||
[ "$(argv_value --served-model-name)" = "frank-large" ]
|
||||
[ "$(argv_value --port)" = "7770" ]
|
||||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||||
}
|
||||
|
||||
@test "the built-in defaults are one session of the full 262144 window with DFlash2" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
# 262144 is the model's native window (the operator's call, 2026-09-21).
|
||||
[ "$(argv_value --context)" -eq 262144 ]
|
||||
# One session is the single stream every prior frank-large served.
|
||||
[ "$(argv_value --sessions)" -eq 1 ]
|
||||
[ "$(argv_value --speculative)" = "dflash2" ]
|
||||
# A draft ceiling of 0 would silently serve the autoregressive rate.
|
||||
[ "$(argv_value --draft-tokens)" -gt 0 ]
|
||||
}
|
||||
|
||||
@test "the shipped env file, not just the defaults, serves the deployed shape" {
|
||||
# The defaults are a floor; systemd sources the env file. Only the
|
||||
# on-host paths are redirected at the stub, so every fleet value in the
|
||||
# shipped file is the one under test. MODEL_FILE and DRAFTER_FILE are
|
||||
# absolute in the shipped file, so they need redirecting too — a sed that
|
||||
# only rewrote MODEL_DIR would leave them pointing at /home/dank and the
|
||||
# test would pass for the wrong reason (or fail on a box that has them).
|
||||
sed -e "s#^RUNTIME_DIR=.*#RUNTIME_DIR=$RUNTIME_DIR#" \
|
||||
-e "s#^MODEL_DIR=.*#MODEL_DIR=$MODEL_DIR#" \
|
||||
-e "s#^MODEL_FILE=.*#MODEL_FILE=$MODEL_DIR/$TARGET_STUB#" \
|
||||
-e "s#^DRAFTER_DIR=.*#DRAFTER_DIR=$DRAFTER_DIR#" \
|
||||
-e "s#^DRAFTER_FILE=.*#DRAFTER_FILE=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf#" \
|
||||
"$ENV_FILE" > "$STAGE/shipped.env"
|
||||
serve_argv "$STAGE/shipped.env"
|
||||
[ "$status" -eq 0 ]
|
||||
[ "$(argv_value --served-model-name)" = "frank-large" ]
|
||||
[ "$(argv_value --host)" = "0.0.0.0" ]
|
||||
[ "$(argv_value --port)" = "7770" ]
|
||||
# 2026-10-01 night: Frank rolled frank-large back to ONE session at the
|
||||
# full native window (gufo Q4_K_XL, f16 KV — gufo has no KV quant lever).
|
||||
[ "$(argv_value --context)" -eq 262144 ]
|
||||
[ "$(argv_value --sessions)" -eq 1 ]
|
||||
[ "$(argv_value --speculative)" = "dflash2" ]
|
||||
[ "$(argv_value --draft-tokens)" -gt 0 ]
|
||||
[ "$(argv_value --draft-policy)" = "adaptive" ]
|
||||
# The shipped file must name the target the fleet actually serves, not a
|
||||
# path that merely happens to exist.
|
||||
[ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ]
|
||||
[ "$(argv_value --dflash-model)" = "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" ]
|
||||
}
|
||||
|
||||
@test "the engine is pointed at the payload that is actually on disk" {
|
||||
serve_argv
|
||||
[ "$status" -eq 0 ]
|
||||
[ -f "$(argv_value --model)" ]
|
||||
# A drafter path that does not exist would load the target and decode at
|
||||
# the autoregressive rate, which reads as "the new runtime is slow".
|
||||
[ -f "$(argv_value --dflash-model)" ]
|
||||
[ "$(argv_value --model)" != "$(argv_value --dflash-model)" ]
|
||||
}
|
||||
|
||||
@test "frank-large stays text-only" {
|
||||
# The payload has no mmproj, so an --mmproj would point at nothing. The
|
||||
# fleet's image path is frank-small: its SGLang MiMo payload carries the
|
||||
# vision tower inside the checkpoint (no projector file at all), and
|
||||
# models.yml declares frank-large text-only.
|
||||
serve_argv
|
||||
! argv_has --mmproj
|
||||
grep -q 'MiMo-Q6K' "$REPO_ROOT/scripts/frank-small.env"
|
||||
}
|
||||
|
||||
@test "the engine is asked for greedy decoding and for preserved prior reasoning" {
|
||||
# Greedy is what this endpoint has always served; a silent switch to
|
||||
# sampled output would be a model-behaviour change disguised as a
|
||||
# runtime swap. Prior thinking blocks must survive across turns or a
|
||||
# multi-turn agent loses its own reasoning between tool calls.
|
||||
serve_argv
|
||||
[ "$(argv_value --temperature)" = "0" ]
|
||||
[ "$(argv_value --preserve-thinking)" = "on" ]
|
||||
}
|
||||
|
||||
@test "the serve script refuses a missing runtime with the build pointer" {
|
||||
rm -f "$RUNTIME_DIR/bin/gufo"
|
||||
serve_argv
|
||||
@test "the serve script fails when the runtime is missing" {
|
||||
# Only the model, the head and the projector exist; the fork was never
|
||||
# built. llama.cpp is not reached, so this fails fast and says what to
|
||||
# run instead of failing at bind time.
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/nope/llama-server" \
|
||||
"MODEL_FILE=$TEST_TMP/model.gguf" "DRAFT_FILE=$TEST_TMP/draft.gguf" \
|
||||
"MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"runtime not found"* ]]
|
||||
[[ "$output" == *"build-gufo.sh"* ]]
|
||||
# A restart loop would be the alternative; nothing may be exec'd.
|
||||
! argv_has serve
|
||||
[[ "$output" == *"build it with scripts/build-rocmfpx.sh"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses a missing drafter with a fetch pointer" {
|
||||
# The drafter lives in its own directory now, so the target can be present
|
||||
# and this still has to be caught.
|
||||
rm -f "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf"
|
||||
serve_argv
|
||||
@test "the serve script refuses to start without the model file" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/absent.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"missing asset"* ]]
|
||||
[[ "$output" == *"Qwen3.8-27B-DFlash2-Q4_K_M.gguf"* ]]
|
||||
[[ "$output" == *"download-frank-large"* ]]
|
||||
! argv_has serve
|
||||
[[ "$output" == *"is not readable"* ]]
|
||||
[[ "$output" == *"download-frank-large-kingjones.sh"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses a missing target and names the file" {
|
||||
rm -f "$MODEL_DIR/$TARGET_STUB"
|
||||
serve_argv
|
||||
@test "the serve script refuses to start without the MTP draft head" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/absent.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"missing asset"* ]]
|
||||
[[ "$output" == *"$TARGET_STUB"* ]]
|
||||
[[ "$output" == *"MTP draft head"* ]]
|
||||
}
|
||||
|
||||
@test "both payload files are fetched resumably and verified against declared sizes + sha256" {
|
||||
# The target and the drafter come from DIFFERENT repositories, fetched by
|
||||
# two scripts. Anchor the repo checks to the URL ASSIGNMENT, not to the
|
||||
# file: the same repo string also appears in each script's header comment,
|
||||
# so a plain grep stayed green after the URL was repointed at another
|
||||
# repository -- the script would then download something the serve script
|
||||
# never asked for.
|
||||
grep -qE '^URL="https://huggingface\.co/unsloth/Qwen3\.8-27B-GGUF/resolve/main/' "$FETCH"
|
||||
grep -qE 'https://huggingface\.co/z-lab/Qwen3\.8-27B-DFlash2-GGUF/resolve/main/' "$DRAFTER_FETCH"
|
||||
for f in "$FETCH" "$DRAFTER_FETCH"; do
|
||||
# -f so an HTTP error is a failure rather than a small "artifact";
|
||||
# -C - so an interrupted 29 GiB run resumes.
|
||||
grep -q -- '-fL -C -' "$f"
|
||||
# The size and the sha256 are compared after the transfer, not trusted.
|
||||
grep -q 'SHA256 MISMATCH' "$f"
|
||||
grep -q 'SIZE MISMATCH' "$f"
|
||||
@test "the serve script refuses a missing mmproj unless vision is opted out" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/absent.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"set MMPROJ_FILE='' for text-only"* ]]
|
||||
# The documented opt-out must actually start, without a --mmproj argument.
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"--mmproj"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script fails when LLAMA_BIN is empty" {
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env LLAMA_BIN= "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"LLAMA_BIN is empty"* ]]
|
||||
}
|
||||
|
||||
@test "the shipped env points at the ROCmFPX runtime and the AGENT payload" {
|
||||
grep -qE "^LLAMA_BIN=$RUNTIME_STUB$" "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=$MODEL_STUB$" "$ENV_FILE"
|
||||
grep -qE "^DRAFT_FILE=$DRAFT_STUB$" "$ENV_FILE"
|
||||
grep -qE "^MMPROJ_FILE=$MMPROJ_STUB$" "$ENV_FILE"
|
||||
# HIP reads bare HOST/TEMP/GPU_MAX_HEAP_SIZE out of the environment that
|
||||
# `set -a` creates; a tunable named HOST leaks into the runtime.
|
||||
! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the argv contract keeps the fleet's port and alias" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" \
|
||||
ALIAS=frank-large BIND_ADDR=127.0.0.1 LISTEN_PORT=7770 "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
# The caller's values must survive the env file, or a single-run operator
|
||||
# override silently does nothing.
|
||||
[[ "$output" == *"--host 127.0.0.1"* ]]
|
||||
[[ "$output" == *"--port 7770"* ]]
|
||||
[[ "$output" == *"--alias frank-large"* ]]
|
||||
# Vision is wired: the projector is the whole vision tower.
|
||||
[[ "$output" == *"--mmproj $TEST_TMP/mmproj.gguf"* ]]
|
||||
}
|
||||
|
||||
@test "the shipped argv is one full native window on one session" {
|
||||
grep -qE '^CTX_SIZE=262144$' "$ENV_FILE"
|
||||
grep -qE '^PARALLEL=1$' "$ENV_FILE"
|
||||
# 262144 IS the model's trained window (general.context_length in the
|
||||
# GGUF), so 1 slot gets the whole thing — no truncation, no yarn scaling.
|
||||
#
|
||||
# Budget, from the GGUF header: 26.28 GiB of weights + 1.57 GiB draft head
|
||||
# + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x
|
||||
# 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB
|
||||
# of DeltaNet state + a couple GiB of compute buffers, against the iGPU's
|
||||
# ~114 GiB GTT aperture. Change CTX_SIZE and this arithmetic, not a boot
|
||||
# log, is what tells you it no longer fits.
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv_gib=$((ctx * 65536 / 1073741824))
|
||||
total_gib=$((27 + 2 + 1 + kv_gib + 3))
|
||||
[ "$total_gib" -le 114 ]
|
||||
}
|
||||
|
||||
@test "the MTP draft contract matches the model card's measured optimum" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
# The head is the model's own nextn layer, loaded as a separate draft
|
||||
# model because this checkpoint ships it standalone (no blk.<n>.nextn.*
|
||||
# tensors in the target file).
|
||||
[[ "$output" == *"--spec-type draft-mtp"* ]]
|
||||
[[ "$output" == *"--model-draft $TEST_TMP/draft.gguf"* ]]
|
||||
# n-max 4 is the measured optimum; n-max 8 measured WORSE (acceptance
|
||||
# fell to 0.757 and the extra verification cost outran the savings), so
|
||||
# the flag is a measurement, not a wish.
|
||||
[[ "$output" == *"--spec-draft-n-max 4"* ]]
|
||||
[[ "$output" == *"--spec-draft-n-min 0"* ]]
|
||||
[[ "$output" == *"--spec-draft-p-min 0.0"* ]]
|
||||
# The draft runs on the iGPU beside the target: no device-to-device hop
|
||||
# on a link that measures single-digit GB/s.
|
||||
[[ "$output" == *"--spec-draft-device ROCm0"* ]]
|
||||
[[ "$output" == *"--spec-draft-ngl 99"* ]]
|
||||
}
|
||||
|
||||
@test "KV and placement stay unquantized with flash attention on" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"--cache-type-k f16"* ]]
|
||||
[[ "$output" == *"--cache-type-v f16"* ]]
|
||||
# ROCmFPx quantized KV has no kernel on this branch, and the endpoint has
|
||||
# the aperture to keep KV unquantized anyway.
|
||||
[[ "$output" == *"--flash-attn on"* ]]
|
||||
# --fit's placement heuristics do not know that 311 tensors of type 103
|
||||
# belong on the iGPU, so the whole stack is placed explicitly instead.
|
||||
[[ "$output" == *"--fit off"* ]]
|
||||
[[ "$output" == *"--n-gpu-layers 999"* ]]
|
||||
}
|
||||
|
||||
@test "the client contract wires thinking and template kwargs" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
# Without --jinja, llama.cpp silently ignores the harness's
|
||||
# chat_template_kwargs, so thinking control disappears from the API.
|
||||
[[ "$output" == *"--jinja"* ]]
|
||||
# The harness reads `reasoning_content` for this endpoint, which is what
|
||||
# the deepseek emission format produces (vLLM's `reasoning` is not it).
|
||||
[[ "$output" == *"--reasoning-format deepseek"* ]]
|
||||
# The template needs an unbounded thinking budget.
|
||||
[[ "$output" == *"--reasoning-budget -1"* ]]
|
||||
}
|
||||
|
||||
@test "the deployer gates exactly the paths the serve script opens" {
|
||||
# A gate that checks a different file than the engine opens is the
|
||||
# 2026-10-01 crash-loop: deploy passed, the unit restart-looped. Read the
|
||||
# engine's own paths out of the env file and require the deployer's
|
||||
# defaults to name the same ones.
|
||||
for pair in "LLAMA_BIN:LLM_LARGE_RUNTIME" \
|
||||
"MODEL_FILE:LLM_LARGE_MODEL" \
|
||||
"DRAFT_FILE:LLM_LARGE_DRAFT" \
|
||||
"MMPROJ_FILE:LLM_LARGE_MMPROJ"; do
|
||||
env_key="${pair%%:*}"
|
||||
dep_key="${pair##*:}"
|
||||
want="$(sed -nE "s|^${env_key}=(.*)\$|\1|p" "$ENV_FILE")"
|
||||
[ -n "$want" ]
|
||||
got="$(grep -oE "${dep_key}:-[^}]+" "$DEPLOYER" | head -1 | sed "s|^${dep_key}:-||")"
|
||||
[ "$got" = "$want" ]
|
||||
done
|
||||
}
|
||||
|
||||
@test "the serve script and the fetch scripts agree on the payload filenames" {
|
||||
# The serve script's MODEL_FILE and the fetch script's REL are two
|
||||
# independent spellings of one filename, and a repo is not required to
|
||||
# keep them in step: a re-upload under a new name, or an env edited
|
||||
# without the fetch, leaves the endpoint unable to start with a
|
||||
# "missing asset" that no test would have predicted. Bind them here.
|
||||
local served fetched drafter_served drafter_fetched
|
||||
served="$(basename "$(grep '^MODEL_FILE=' "$ENV_FILE" | cut -d= -f2-)")"
|
||||
fetched="$(sed -n 's/^REL="\(.*\)"$/\1/p' "$FETCH")"
|
||||
[ -n "$served" ]
|
||||
[ "$served" = "$fetched" ]
|
||||
drafter_served="$(basename "$(grep '^DRAFTER_FILE=' "$ENV_FILE" | cut -d= -f2-)")"
|
||||
drafter_fetched="$(sed -n 's/^ *"\(Qwen3\.8-27B-DFlash2[^ ]*\.gguf\) .*/\1/p' "$DRAFTER_FETCH")"
|
||||
[ -n "$drafter_served" ]
|
||||
[ "$drafter_served" = "$drafter_fetched" ]
|
||||
@test "the deployer points a missing runtime at the fork build script" {
|
||||
grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER"
|
||||
grep -qE 'fetch it: bash scripts/download-frank-large-kingjones\.sh' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the drafter is still fetched by the script that carries its Hub repo" {
|
||||
# Guards a future "simplification" that folds the drafter into the
|
||||
# payload-fetch script and silently orphans the artifact the serve script
|
||||
# still points at via DRAFTER_DIR. The literal drafter name must not
|
||||
# appear in that script AT ALL, comments included.
|
||||
grep -q 'Qwen3.8-27B-UD-Q8_K_XL' "$DRAFTER_FETCH"
|
||||
! grep -q 'DFlash2' "$FETCH"
|
||||
@test "the stray-process sweep cannot take frank-small down with frank-large" {
|
||||
# frank-small serves a different GGUF over llama-server on the SAME host.
|
||||
# A sweep keyed on the binary name (`pkill -f llama-server`) would kill
|
||||
# the 3090 endpoint too, so it is keyed on frank-large's own payload file.
|
||||
grep -q "pkill -f 'Qwen3.8-27B-Q8_0_ROCMFPX_AGENT'" "$DEPLOYER"
|
||||
! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the build script refuses a source tree that is not the pinned commit" {
|
||||
# gufo ships no prebuilt Linux binaries, so the runtime is built here.
|
||||
# A tag that moves under us would silently install different kernels
|
||||
# under the same RUNTIME_DIR, so the pin is enforced, not documented.
|
||||
local src="$STAGE/not-the-pinned-commit"
|
||||
mkdir -p "$src"
|
||||
git -C "$src" init -q
|
||||
git -C "$src" -c user.email=t@t -c user.name=t commit -q --allow-empty -m drift
|
||||
run env GUFO_SRC_DIR="$src" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify
|
||||
# The pin is load-bearing: the older builds of this fork line segfault in
|
||||
# ggml_compute_forward_set_rows on this exact payload. A runtime that
|
||||
# differs from what was qualified must be heard about at deploy time.
|
||||
wrong="$(git -C "$SRC" rev-parse HEAD)"
|
||||
run env ROCMFPX_SRC_DIR="$SRC" ROCMFPX_COMMIT="ffff$wrong" \
|
||||
"$BUILD" --verify
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"GUFO_COMMIT mismatch"* ]]
|
||||
[[ "$output" == *"ROCMFPX_COMMIT"*mismatch* ]]
|
||||
}
|
||||
|
||||
@test "the build script reports a runtime that was never built" {
|
||||
# --verify on a host with no install must say so rather than exit 0 on
|
||||
# an empty check.
|
||||
run env GUFO_SRC_DIR="$STAGE/no-source-tree" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"no gufo at"* ]]
|
||||
@test "the build script pins a real commit and the gfx1151 target" {
|
||||
grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
|
||||
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
||||
grep -qE 'DGPU_TARGETS=' "$BUILD"
|
||||
# The webui asset fetch half-failed on this box and aborted the link at
|
||||
# 98%; the build must not depend on it.
|
||||
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
||||
}
|
||||
|
||||
@test "the deploy handler gates on the runtime, the payload and the port" {
|
||||
grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY"
|
||||
grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY"
|
||||
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
||||
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY"
|
||||
grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY"
|
||||
@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
|
||||
grep -qE '^Restart=(always|on-failure)$' "$UNIT"
|
||||
grep -qE '^WantedBy=' "$UNIT"
|
||||
grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT"
|
||||
grep -qE '^User=dank$' "$UNIT"
|
||||
# The payload is 26 GiB of mmap'd weights plus pinned staging buffers; an
|
||||
# RLIMIT smaller than the pinned set fails the load in a way that looks
|
||||
# like an engine crash.
|
||||
grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the deploy handler will not stop a serving endpoint it cannot replace" {
|
||||
# The runtime gate must be live (a real conditional on an executable
|
||||
# path, not a branch that never fires) and it must return BEFORE the
|
||||
# stop, or a host whose gufo was never built takes frank-large down on
|
||||
# its way to reporting the failure.
|
||||
local large gate bail stop
|
||||
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||||
[[ "$large" == *'build-gufo.sh'* ]]
|
||||
gate="$(grep -n 'if \[\[ ! -x "\$LLM_LARGE_RUNTIME" \]\]' <<<"$large" | head -1 | cut -d: -f1)"
|
||||
stop="$(grep -n 'systemctl stop frank-large' <<<"$large" | head -1 | cut -d: -f1)"
|
||||
[ -n "$gate" ]
|
||||
[ -n "$stop" ]
|
||||
bail="$(awk -v g="$gate" 'NR > g && /return 1/ { print NR; exit }' <<<"$large")"
|
||||
[ -n "$bail" ]
|
||||
[ "$bail" -lt "$stop" ]
|
||||
}
|
||||
|
||||
@test "the stray-run sweep keys on the drafter and cannot take frank-small down" {
|
||||
# The sweep used to key on the TARGET filename, which the 2026-09-28
|
||||
# payload swap moved to a different repository. It would have stopped
|
||||
# matching the live process, and a stray run would then hold 7770 past the
|
||||
# port gate and fail the deploy. The drafter is the part that did NOT
|
||||
# move, and every frank-large command line carries it via --dflash-model.
|
||||
grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY"
|
||||
# frank-small serves a different GGUF on the 3090 with no DFlash2 drafter,
|
||||
# so a binary-name sweep is the only thing that would endanger it.
|
||||
local large
|
||||
large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")"
|
||||
! grep -q 'pkill -x llama-server' <<<"$large"
|
||||
! grep -q 'pkill -x gufo' <<<"$large"
|
||||
}
|
||||
|
||||
@test "the deploy handler retires the halogen quadlet without deleting it" {
|
||||
grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY"
|
||||
grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY"
|
||||
! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY"
|
||||
# Nothing live ships for this endpoint as a container any more. The
|
||||
# archived copy is host state in an ignored .AGENTTRASH — deliberately
|
||||
# untracked, and present only in a primary checkout — so search the
|
||||
# tracked tree, not the filesystem.
|
||||
[ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ]
|
||||
! git -C "$REPO_ROOT" grep -q 'Image=.*halogen' -- dankclaw
|
||||
@test "the unit points at no retired frank-large runtime path" {
|
||||
# The host keeps every retired payload and build on disk, so a stale path
|
||||
# in a unit still resolves — to a runtime nobody has measured in weeks.
|
||||
! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
|
||||
! grep -q 'halogen' "$UNIT"
|
||||
}
|
||||
Reference in new issue
Block a user