diff --git a/scripts/build-rocmfpx.sh b/scripts/build-rocmfpx.sh new file mode 100755 index 0000000..0644304 --- /dev/null +++ b/scripts/build-rocmfpx.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Build the ROCmFPX runtime that serves frank-large on Dankclaw. +# +# bash scripts/build-rocmfpx.sh # clone/fetch + build the pin +# bash scripts/build-rocmfpx.sh --verify # only check the installed tree +# +# Runtime: kingjones30/ROCmFPX — charlie12345/ROCmFPX plus the model-card +# patches, i.e. the fork that owns the ROCmFPx tensor formats. frank-large's +# payload is ftype 115 (`Q8_0_ROCMFPX_AGENT`): 311 tensors of +# GGML_TYPE_Q8_0_ROCMFPX (103), a type that exists ONLY in this fork, so +# stock llama.cpp, gufo and every other loader on the host reject the file. +# This is the same tree that served the Qwen3.8-Flash-Next ROCmFP4 STRIX +# endpoint 2026-08-30 → 09-11 (see the kingjones runbook), rebuilt here so a +# fresh host can be brought up from the repo alone. +# +# Why this commit and not a newer one: 36e9acd (2026-08-28) is what was +# qualified against this payload on 2026-10-02 — it loads ftype 115, survives +# warmup, and drafts with the MTP head at acceptance 0.65. The five legacy +# builds of 2026-08-25 crashed this same file in ggml_compute_forward_set_rows +# (see kingjones-q8agent-rocmfpx.md), so the pin is load-bearing: do not bump +# it without re-running that crash test. +# +# Host notes: +# * -DLLAMA_BUILD_WEBUI=OFF is REQUIRED on this box. The tree provisions its +# embedded UI assets via npm or the HF llama-ui bucket; npm is absent here +# and the bucket download half-failed, aborting the link at 98%. The API +# server is unaffected. +# * The build links against the host ROCm (/opt/rocm); no bundled toolchain +# like the gfx1201 R9700 build needed. +set -euo pipefail + +ROCMFPX_REPO="${ROCMFPX_REPO:-https://github.com/kingjones30/ROCmFPX.git}" +# The commit frank-large was qualified with (measured 2026-10-02). +ROCMFPX_COMMIT="${ROCMFPX_COMMIT:-36e9acd40e10a87cd3c3ef8ec734668757dc8520}" +SRC_DIR="${ROCMFPX_SRC_DIR:-/home/dank/GitPackages/kingjones-ROCmFPX}" +BUILD_DIR="${ROCMFPX_BUILD_DIR:-$SRC_DIR/build}" +GPU_TARGET="${ROCMFPX_GPU_TARGET:-gfx1151}" +JOBS="${JOBS:-$(nproc)}" + +# check_pin — refuse a source tree that is not the qualified commit. A moved +# fork means different kernels under the same path, and on this model line a +# wrong build is a segfault at first request, not a clean error. +check_pin() { + local commit + commit="$(git -C "$SRC_DIR" rev-parse HEAD 2>/dev/null || true)" + if [[ -n "$commit" && "$commit" != "$ROCMFPX_COMMIT" ]]; then + echo "ROCMFPX_COMMIT mismatch: $SRC_DIR is at $commit, pinned $ROCMFPX_COMMIT" >&2 + exit 1 + fi +} + +if [[ "${1:-}" == "--verify" ]]; then + check_pin + [[ -x "$BUILD_DIR/bin/llama-server" ]] || { + echo "no ROCmFPX llama-server at $BUILD_DIR/bin/llama-server" >&2 + exit 1 + } + # The format the endpoint exists for: the fork must carry the ROCmFPx + # tensor type, or the payload cannot load at all. + "$BUILD_DIR/bin/llama-server" --help 2>&1 | grep -q -- '--spec-type' || { + echo "$BUILD_DIR/bin/llama-server has no --spec-type: MTP drafting is impossible" >&2 + exit 1 + } + "$BUILD_DIR/bin/llama-server" --version | head -1 + exit 0 +fi + +echo "== source: $ROCMFPX_REPO @ $ROCMFPX_COMMIT ==" +if [[ ! -d "$SRC_DIR/.git" ]]; then + mkdir -p "$(dirname "$SRC_DIR")" + git clone "$ROCMFPX_REPO" "$SRC_DIR" +fi +git -C "$SRC_DIR" fetch --quiet origin "$ROCMFPX_COMMIT" || true +git -C "$SRC_DIR" checkout -q --detach "$ROCMFPX_COMMIT" +check_pin +echo " $(git -C "$SRC_DIR" rev-parse HEAD)" + +echo "== configure + build (release, HIP $GPU_TARGET) ==" +cmake -S "$SRC_DIR" -B "$BUILD_DIR" \ + -DGGML_HIP=ON \ + -DGPU_TARGETS="$GPU_TARGET" \ + -DGGML_NATIVE=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_BUILD_WEBUI=OFF +cmake --build "$BUILD_DIR" --target llama-server llama-quantize -j "$JOBS" + +echo "== verify ==" +"$BUILD_DIR/bin/llama-server" --version | head -1 +echo "frank-large runtime ready: $BUILD_DIR/bin/llama-server ($ROCMFPX_COMMIT)" diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index e0b9bf9..996e82b 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1077,27 +1077,29 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built -# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving -# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter -# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a -# ROOT unit: the runtime is a native binary linked against the host ROCm, -# so there is no container image to install and no generated unit to dodge. -# relink_unit points /etc/systemd/system/frank-large.service at the repo -# file and boot persistence is that unit's [Install] section. +# frank-large is the ROCmFPX fork of llama.cpp (kingjones30/ROCmFPX, pinned by +# scripts/build-rocmfpx.sh), serving +# kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF:Q8_0_ROCMFPX_AGENT with the +# repo's own Q4_0 MTP draft head and its BF16 vision projector, on the Strix +# Halo iGPU. It is a ROOT unit: the runtime is a native binary linked against +# the host ROCm, so there is no container image to install and no generated +# unit to dodge. relink_unit points /etc/systemd/system/frank-large.service at +# the repo file and boot persistence is that unit's [Install] section. # -# The runtime gate matters as much as the payload gate: gufo is built from -# source on this host (there is no prebuilt release), so a machine whose +# The runtime gate matters as much as the payload gate: the fork is built from +# source on this host, and the payload is ftype 115 (tensor type 103, +# GGML_TYPE_Q8_0_ROCMFPX) which only this fork can read — so a machine whose # /home/dank/GitPackages tree was never built has a unit that can never # become healthy, and a green deploy would leave the endpoint dark. # -# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had -# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the -# same as every cutover since 2026-09-16: nothing starts until 7770 is -# released. -LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}" -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" +# It replaces gufo v0.1.1 + unsloth UD-Q4_K_XL (the rollback target, still +# built by scripts/build-gufo.sh with its payload on disk), which had replaced +# the stock llama.cpp prebuilt. The port gate is the same as every cutover +# since 2026-09-16: nothing starts until 7770 is released. +LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" +LLM_LARGE_MMPROJ="${LLM_LARGE_MMPROJ:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1132,17 +1134,19 @@ wait_port_free() { deploy_llm_large() { # Refuse rather than restart-loop. A unit whose runtime or payload is # missing never becomes healthy, and a green deploy would leave the - # endpoint dark with nothing in the deploy log to say why. + # endpoint dark with nothing in the deploy log to say why. The gates + # mirror serve-frank-large.sh's own predicates exactly: runtime + # executable, target GGUF, MTP head, mmproj. if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then - echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2 - echo "frank-large: build it: bash scripts/build-gufo.sh" >&2 + echo "ERROR: ROCmFPX runtime absent at $LLM_LARGE_RUNTIME" >&2 + echo "frank-large: build it: bash scripts/build-rocmfpx.sh" >&2 return 1 fi local missing=0 f - for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do + for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_MMPROJ"; do if [[ ! -f "$f" ]]; then echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-kingjones.sh" >&2 missing=1 fi done @@ -1152,18 +1156,14 @@ deploy_llm_large() { # On a re-deploy the predecessor is THIS unit: stop it before waiting on # the port, or the wait deadlocks on the server it just stopped serving. # Then sweep a stray manual run -- that is how a bench harness leaves the - # port busy. Match the DRAFTER path, which every frank-large command - # line carries via --dflash-model and which is UNCHANGED by the 2026-09-28 - # payload swap: the target moved to a different repository while the - # drafter stayed put, so keying on the target filename would have quietly - # stopped matching. It is also the right key in principle — it names this - # endpoint's speculative drafter, not a binary. Match that, not a binary - # name: frank-small on this host serves a different GGUF over llama-server - # with no DFlash2 drafter and must not be killed. The pattern never - # appears in the deploying shell's own command line, so -f is safe here - # (it is not in `bash deploy-remote.sh`). + # port busy. Match the TARGET GGUF name, which every frank-large command + # line carries via --model and which no other process on this host uses: + # frank-small also runs llama-server here (with the MiMo GGUF) and must + # not be killed, so keying on the binary name would take the 3090 endpoint + # down with it. The pattern never appears in the deploying shell's own + # command line, so -f is safe here (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true + pkill -f 'Qwen3.8-27B-Q8_0_ROCMFPX_AGENT' 2>/dev/null || true # Nothing may start until the predecessor has released the port. On # 2026-09-16 the cutover failed exactly here: the old server logged # "Deactivated successfully", the next engine tried to bind 7770 a second diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 9422e08..3eb9b3c 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,63 +1,85 @@ -# frank-large — gufo (gufo-org/gufo) serving -# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) on the Strix Halo iGPU. +# frank-large — ROCmFPX llama.cpp serving +# kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF (Q8_0_ROCMFPX_AGENT, ftype +# 115) + its Q4_0 MTP draft head, on the Strix Halo iGPU (gfx1151), :7770. # -# These are the tunables read by scripts/serve-frank-large.sh. The flag -# list is owned by the serve script; this file only sets fleet values. +# 2026-10-02 (Frank): "the highest quality of that repo, one full context, +# whatever runtime it requires". The highest-quality rung is Q8_0_ROCMFPX_AGENT +# (26.28 GiB, 8.39 bpw: 187 tensors at plain Q8_0 + 311 at +# GGML_TYPE_Q8_0_ROCMFPX); the AGENT routing is also the variant the card +# tells you to take with MTP, because it keeps the output-head tensors at full +# Q8_0 (187 vs 2), which is what the draft scores against. The runtime the +# format REQUIRES is the ROCmFPX fork — type 103 exists nowhere else. # -# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP -# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one -# exception because the serve script itself consumes it for --host. - -# The target payload (scripts/download-frank-large-q4.sh): the serve script -# refuses to start without the main GGUF and the DFlash2 drafter. +# The shape: one session at the model's full native window (262,144), which is +# the whole point — the Q8_0 head costs quality the fleet wanted back, and +# the draft head pays for the bandwidth the bigger file spends. # -# 2026-10-01 swap (Frank): unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL, 16.35 GiB, -# replacing ajgazin's abliterated Swift-1.5 UD-Q8_K_XL (29.3 GiB). Two -# consequences worth knowing at a glance: -# * this is the VANILLA (non-abliterated) Qwen3.8-27B — the endpoint is no -# longer uncensored, which was one of the reasons the Swift payload was -# chosen on 2026-09-28; -# * ~13 GiB of weights became available for KV, which is what makes four -# full-sized f16 sessions fit. -# gufo needs no template for either: its Qwen renderer is compiled in. -MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL -MODEL_FILE=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf +# Budget (from the GGUF header, not from a boot log): attention KV lives only +# in the 16 full-attention layers (4 KV heads x 256 head_dim), 64 KiB/token +# with f16 K and V => 16.0 GiB at 262,144. The 48 Gated DeltaNet layers hold +# ~144 MiB of fixed state per sequence. Weights 26.28 + draft head 1.57 + +# mmproj 0.87 + KV 16.0 + state 0.14 + compute buffers ≈ 45-47 GiB against the +# iGPU's ~114 GiB GTT aperture, so f16 KV (the unquantized choice) is affordable +# here in a way it is not on frank-small's 24 GiB card. -# The DFlash2 drafter is UNCHANGED by the model swap and still lives beside -# the previous target, so it carries its own directory. It has a sliding -# window of 2048, which is what keeps its KV small (~40 MiB). -DRAFTER_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL -DRAFTER_FILE=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf +# The qualified ROCmFPX fork build (scripts/build-rocmfpx.sh pins the commit). +LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server +# One directory per HF repo, created by scripts/download-frank-large-kingjones.sh. +MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf +# The draft head: Q4_0, NOT the Q8_0 head. The head's per-step cost is its +# projection over a 248K vocabulary, so the cheaper head is faster end to end +# at equal-or-better acceptance (card: +6.0% at n-max 1, +10.2% at n-max 2). +DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf +# Vision tower, separate from the LLM file. Empty string = text-only. +MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf -# The gufo runtime, built from source by scripts/build-gufo.sh. -RUNTIME_DIR=/home/dank/GitPackages/gufo-v0.1.1 - -# Fleet endpoint: the same port and alias every frank-large has served, so -# no client, probe or models.yml entry changes address. +# Fleet endpoint: same port and alias as every frank-large before it, so the +# harness models.yml id and the homepage probe do not move. ALIAS=frank-large -HOST=0.0.0.0 -PORT=7770 +# HIP reads a bare HOST, so this is BIND_ADDR — the same reason the file +# avoids a bare TEMP. +BIND_ADDR=0.0.0.0 +LISTEN_PORT=7770 -# ONE session at the full native window (Frank, 2026-10-01 night: "roll back -# to qwen3.8 27b, one single max context"). The KV is UNQUANTIZED (f16) — -# gufo has no KV quantization lever at all (execution_policy.hpp pins -# kv_cache_storage to kFp16), so "unquantized" is inherent rather than -# configured. -# -# The arithmetic: f16 KV costs 64 KiB/token on this model (16 full-attention -# layers x 4 KV heads x 256 head_dim x 2 (K,V) x 2 bytes; the other 48 layers -# are linear-attention with a per-SEQUENCE fp32 state of ~144 MiB, not a -# per-token cache). 262,144 tokens = 16.0 GiB of KV. Budget: weights 16.35 + -# KV 16.0 + drafter 1.07 + linear state 0.14 + runtime ~4 = ~37.6 GiB against -# the 114 GiB GTT — enormous headroom; the session count or context policy -# can grow from here without pressure. -CONTEXT_SIZE=262144 -SESSIONS=1 +# One full native window. llama.cpp divides --ctx-size across --parallel +# slots, so 262144/1 IS the model's trained window; the card validated 65,536 +# and the fleet has been running this family at 262,144 since the gufo era. +CTX_SIZE=262144 +PARALLEL=1 -# DFlash2 spec decode (a block-diffusion drafter). gufo sizes the draft -# block itself; DRAFT_TOKENS caps the proposals per step (gufo's default is -# 7, chosen over llama.cpp's fixed depth 4 because gufo's adaptive policy -# shortens the block when acceptance drops) and DRAFT_POLICY is the block -# length controller. -DRAFT_TOKENS=7 -DRAFT_POLICY=adaptive +# Deck knobs. --ubatch-size has a measured optimum on this stack (512 peak on +# the sibling qwen35 model; larger ubatch costs GiBs of compute buffer for no +# speed), and --fit must stay off: --fit's auto placement does not know that +# 311 tensors of type 103 belong on the iGPU. +BATCH_SIZE=2048 +UBATCH_SIZE=512 +NGL=999 +FIT=off +FLASH_ATTN=on +# Unquantized KV — the quality-first choice, affordable at one slot. +CACHE_TYPE_K=f16 +CACHE_TYPE_V=f16 + +# Speculative decode: the model ships a standalone MTP head (mtp-Q4_0.gguf, +# arch qwen35, nextn_predict_layers=1, a 65-block draft). n-max 4 with n-min 0 +# and p-min 0.0 is the card's measured optimum (+31% end to end, acceptance +# 0.926; its own n-max 8 measurement only reached 0.757 acceptance and LOST). +# The draft runs entirely on the iGPU, so cross-device traffic stays off the +# bandwidth-limited link. +SPEC_TYPE=draft-mtp +SPEC_DRAFT_NGL=99 +SPEC_DRAFT_DEVICE=ROCm0 +SPEC_DRAFT_N_MAX=4 +SPEC_DRAFT_N_MIN=0 +SPEC_DRAFT_P_MIN=0.0 + +# The card's thinking sampling profile, which is also what the harness sends +# in extraBody, so the endpoint behaves the same to a client that omits it. +TEMPERATURE=1.0 +TOP_P=0.95 +TOP_K=20 +MIN_P=0.0 +PRESENCE_PENALTY=0.0 + +NUM_THREADS=16 +THREAD_BATCH=16 diff --git a/scripts/frank-large.service b/scripts/frank-large.service index 7c9e9a4..baebc6c 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,5 +1,5 @@ [Unit] -Description=gufo (gufo-org/gufo, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU) +Description=ROCmFPX llama.cpp (gfx1151) kingjones777 Qwen3.8-27B Q8_0_ROCMFPX_AGENT, MTP spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target @@ -13,7 +13,9 @@ Group=dank LimitMEMLOCK=infinity Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh -Restart=on-failure +# always, not on-failure: an engine that drains and exits 0 after a worker +# fault would otherwise leave the endpoint down until someone notices. +Restart=always RestartSec=10 [Install] diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index caf524c..da86080 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,121 +1,179 @@ #!/usr/bin/env bash -# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on -# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the -# inference engine built specifically for this hardware. +# Serve kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF as frank-large on +# Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of llama.cpp. # -# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source -# against the host ROCm by scripts/build-gufo.sh. It replaced the stock -# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client -# contract did not move. +# 2026-10-02 (Frank): swapped from gufo + Q4_K_XL to the highest-quality file +# in the repo, one session at the native 262,144 window. # -# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's -# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced -# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally -# interchangeable for this engine — same `qwen35` architecture, the same 866 -# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the -# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune -# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which -# is not an OSI licence. +# Why this runtime and not the previous one: the payload is ftype 115 — 311 +# tensors of GGML_TYPE_Q8_0_ROCMFPX (103), a tensor type that exists only in +# the ROCmFPX fork. Stock llama.cpp, SGLang and gufo all refuse the file, and +# gufo additionally gates every "Qwen3.8" artifact on a template-hash +# allowlist. The qualified build is scripts/build-rocmfpx.sh's pin: it loads +# ftype 115, survives warmup (the 2026-08-25 builds of the older fork line +# segfaulted in ggml_compute_forward_set_rows on this exact file), and drafts +# with the model's own MTP head. # -# Shape: one session, full 262144 context (the model's native window). The -# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV + -# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this -# context measured 66.5 GiB of the 116.7 GiB device. +# Why this file: Q8_0_ROCMFPX_AGENT is the top of the repo's quality ladder +# (26.28 GiB, 8.39 bpw) and the AGENT routing keeps 187 tensors — the +# output-head set the draft scores against — at plain Q8_0 instead of the 2 +# the non-AGENT build keeps, which is the variant the card names for MTP +# serving. Drafting uses the repo's Q4_0 MTP head: the head's cost is its +# projection over the 248K vocabulary, so the 4-bit head is faster end to end +# at equal-or-better acceptance than its Q8_0 sibling. # -# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block -# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the -# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048 -# (small draft KV). It is UNCHANGED by the model swap and still lives beside -# the previous target, which is why DRAFTER_DIR is its own tunable. -# -# Sampling: greedy (--temperature 0) server-side, as every frank-large has -# served. The harness overrides it per request via extraBody (1.0 / 0.95 / -# top-k 20). -# -# Template: gufo compiles the official Qwen Jinja into the binary and takes -# NO --chat-template-file — and it is not a Jinja engine at all -# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the -# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law -# was given up when this endpoint moved to gufo on 2026-09-28, by decision: -# the terse system-prompt injection, the in-band <|think_*|> control, the -# json|xml tool_call_format switch and the tool-argument truncation are all -# absent. gufo's own renderer keeps the effort instruction strings and -# preserve_thinking, which are the parts the harness depends on. Accepted -# 2026-09-28; recovering the terse prompt from the client side is the -# cheapest partial restore. -# -# Text-only: gufo auto-discovers a mmproj beside the model and this payload -# has none (the base repo ships one separately, unused), so the endpoint -# stays text-only. -# -# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides -# it; anything not set there falls through to the defaults below. +# The client contract lives here, not in frank-large.env: +# * --jinja — without it llama.cpp silently IGNORES the harness's +# chat_template_kwargs (enable_thinking / preserve_thinking), so the +# endpoint would look like it had no thinking control at all. The GGUF +# carries the official Qwen template, so no --chat-template-file. +# * --reasoning-format deepseek — thinking is emitted as reasoning_content, +# the field the harness reads for this endpoint. +# * the template accepts reasoning_effort xhigh|medium|low only, which is +# why models.yml folds the harness's levels onto that set. +# * --alias frank-large — that is the model id the harness resolves. +# * --ctx-checkpoints / --cache-ram — prompt-cache geometry for the 16 +# sliding-window-style layers; harmless where the hybrid state forces a +# full re-process, and required for the checkpoint feature to work at all. set -euo pipefail -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" +# Fleet values live in frank-large.env. Anything already in the environment +# wins over the file, so an operator can override one value on a single run: +# the caller's values are captured, the file is sourced, then they are put +# back. (A plain `source` would let the file clobber the environment.) +TUNABLES=(LLAMA_BIN MODEL_FILE DRAFT_FILE MMPROJ_FILE ALIAS BIND_ADDR + LISTEN_PORT CTX_SIZE PARALLEL BATCH_SIZE UBATCH_SIZE NGL FIT + FLASH_ATTN CACHE_TYPE_K CACHE_TYPE_V SPEC_TYPE SPEC_DRAFT_NGL + SPEC_DRAFT_DEVICE SPEC_DRAFT_N_MAX SPEC_DRAFT_N_MIN SPEC_DRAFT_P_MIN + TEMPERATURE TOP_P TOP_K MIN_P PRESENCE_PENALTY NUM_THREADS + THREAD_BATCH ROCM_PATH HIP_VISIBLE HSA_GFX_VERSION) +_OVERRIDES=() +for _var in "${TUNABLES[@]}"; do + [[ -n "${!_var+set}" ]] && _OVERRIDES+=("$_var=${!_var}") +done +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$(dirname "$0")/frank-large.env}" if [[ -f "$ENV_FILE" ]]; then set -a + # shellcheck disable=SC1090 source "$ENV_FILE" set +a fi - -# --- defaults (used when the env file does not set them) ------------------ -: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}" -: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}" -: "${ALIAS:=frank-large}" -: "${HOST:=0.0.0.0}" -: "${PORT:=7770}" -: "${CONTEXT_SIZE:=262144}" -: "${SESSIONS:=1}" -: "${DRAFT_TOKENS:=7}" -: "${DRAFT_POLICY:=adaptive}" - -GUFO_BIN="$RUNTIME_DIR/bin/gufo" - -# The target GGUF and the drafter are named explicitly because they do not -# share a directory any more: the target is a different repository's artifact -# and keeps its own long filename, while the DFlash2 drafter is unchanged and -# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to -# the deployed payload, so the env file only has to state them when it moves. -: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}" -: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}" -: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}" - -if [[ ! -x "$GUFO_BIN" ]]; then - echo "runtime not found at $GUFO_BIN" >&2 - echo "frank-large: build it: bash scripts/build-gufo.sh" >&2 - exit 1 -fi -for f in "$MODEL_FILE" "$DRAFTER_FILE"; do - if [[ ! -f "$f" ]]; then - echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2 - exit 1 - fi +for _kv in "${_OVERRIDES[@]+${_OVERRIDES[@]}}"; do + export "${_kv?}" done +unset _var _kv -# --- env ------------------------------------------------------------------- -# gufo links against the host ROCm through its RPATH (/opt/rocm -> -# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is -# already gfx1151 — `gufo diagnose` reports it natively — so no -# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt -# llama.cpp runtime and can only misreport the arch here. -export HIP_VISIBLE_DEVICES=0 +: "${LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" +: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}" +: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" +: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" +: "${ALIAS=frank-large}" +: "${BIND_ADDR=0.0.0.0}" +: "${LISTEN_PORT=7770}" +: "${CTX_SIZE=262144}" +: "${PARALLEL=1}" +: "${BATCH_SIZE=2048}" +: "${UBATCH_SIZE=512}" +: "${NGL=999}" +: "${FIT=off}" +: "${FLASH_ATTN=on}" +: "${CACHE_TYPE_K=f16}" +: "${CACHE_TYPE_V=f16}" +: "${SPEC_TYPE=draft-mtp}" +: "${SPEC_DRAFT_NGL=99}" +: "${SPEC_DRAFT_DEVICE=ROCm0}" +: "${SPEC_DRAFT_N_MAX=4}" +: "${SPEC_DRAFT_N_MIN=0}" +: "${SPEC_DRAFT_P_MIN=0.0}" +: "${TEMPERATURE=1.0}" +: "${TOP_P=0.95}" +: "${TOP_K=20}" +: "${MIN_P=0.0}" +: "${PRESENCE_PENALTY=0.0}" +: "${NUM_THREADS=16}" +: "${THREAD_BATCH=16}" +: "${ROCM_PATH=/opt/rocm}" +: "${HIP_VISIBLE=0}" +: "${HSA_GFX_VERSION=11.5.1}" -# --- the serve command --------------------------------------------------- -# Server options precede the subcommand, model options follow it. -exec "$GUFO_BIN" serve \ - --host "$HOST" \ - --port "$PORT" \ - --sessions "$SESSIONS" \ - llm \ +[[ -n "$LLAMA_BIN" ]] || { echo "frank-large: LLAMA_BIN is empty" >&2; exit 1; } +[[ -x "$LLAMA_BIN" ]] || { + echo "frank-large: llama-server '$LLAMA_BIN' is not executable;" >&2 + echo " build it with scripts/build-rocmfpx.sh" >&2 + exit 1 +} +# Refuse anything but the real payload. A path that is not a readable file +# makes llama-server hang rather than fail clean. +[[ -r "$MODEL_FILE" ]] || { + echo "frank-large: model file '$MODEL_FILE' is not readable;" >&2 + echo " fetch it with scripts/download-frank-large-kingjones.sh" >&2 + exit 1 +} +# The MTP head is the performance feature: without it the Q8_0 weights decode +# at memory-bandwidth speed and the endpoint loses the ~1.4x the head buys. +[[ -r "$DRAFT_FILE" ]] || { + echo "frank-large: MTP draft head '$DRAFT_FILE' is not readable;" >&2 + echo " fetch it with scripts/download-frank-large-kingjones.sh" >&2 + exit 1 +} +# The GGUF carries no vision tower; the projector file is the whole tower, so +# a missing file means a vision request would 500 rather than degrade. +# MMPROJ_FILE="" is the documented opt-out for a text-only endpoint. +MM_ARGS=() +if [[ -n "${MMPROJ_FILE:-}" ]]; then + [[ -r "$MMPROJ_FILE" ]] || { + echo "frank-large: mmproj '$MMPROJ_FILE' is not readable; set MMPROJ_FILE='' for text-only" >&2 + exit 1 + } + MM_ARGS=(--mmproj "$MMPROJ_FILE") +fi + +# HIP/ROCm host environment. HIP reads bare env vars (TEMP, HOST, +# GPU_MAX_HEAP_SIZE), which is why the tunables above are named TEMPERATURE +# and BIND_ADDR. HSA_OVERRIDE_GFX_VERSION is what lets a runtime built for a +# neighbouring gfx target run on this iGPU; the qualified pin targets gfx1151 +# natively and the override is a no-op for it. +export ROCM_PATH +export HIP_VISIBLE_DEVICES="$HIP_VISIBLE" +export HSA_OVERRIDE_GFX_VERSION="$HSA_GFX_VERSION" +export LD_LIBRARY_PATH="$(dirname "$LLAMA_BIN"):${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +exec /usr/bin/env "$LLAMA_BIN" \ --model "$MODEL_FILE" \ - --served-model-name "$ALIAS" \ - --context "$CONTEXT_SIZE" \ - --speculative dflash2 \ - --dflash-model "$DRAFTER_FILE" \ - --draft-tokens "$DRAFT_TOKENS" \ - --draft-policy "$DRAFT_POLICY" \ - --temperature 0 \ - --think auto \ - --preserve-thinking on + "${MM_ARGS[@]}" \ + --alias "$ALIAS" \ + --host "$BIND_ADDR" \ + --port "$LISTEN_PORT" \ + --jinja \ + --device ROCm0 \ + --n-gpu-layers "$NGL" \ + --parallel "$PARALLEL" \ + --ctx-size "$CTX_SIZE" \ + --batch-size "$BATCH_SIZE" \ + --ubatch-size "$UBATCH_SIZE" \ + --threads "$NUM_THREADS" \ + --threads-batch "$THREAD_BATCH" \ + --fit "$FIT" \ + --flash-attn "$FLASH_ATTN" \ + --direct-io \ + --cache-type-k "$CACHE_TYPE_K" \ + --cache-type-v "$CACHE_TYPE_V" \ + --ctx-checkpoints 16 \ + --cache-ram 8192 \ + --cache-idle-slots \ + --temp "$TEMPERATURE" \ + --top-p "$TOP_P" \ + --top-k "$TOP_K" \ + --min-p "$MIN_P" \ + --presence-penalty "$PRESENCE_PENALTY" \ + --reasoning on \ + --reasoning-format deepseek \ + --reasoning-budget -1 \ + --spec-type "$SPEC_TYPE" \ + --model-draft "$DRAFT_FILE" \ + --spec-draft-ngl "$SPEC_DRAFT_NGL" \ + --spec-draft-device "$SPEC_DRAFT_DEVICE" \ + --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ + --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ + --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ + --metrics diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 6e812eb..6d3ca31 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,338 +1,280 @@ #!/usr/bin/env bats -# Seam: the frank-large root unit on dankclaw. +# Seam: frank-large serves kingjones777's Qwen3.8-27B Q8_0_ROCMFPX_AGENT (a +# ftype-115 ROCmFPX file) with the repo's MTP draft head on the Strix Halo +# iGPU, via the ROCmFPX fork of llama.cpp — one session at the native +# 262,144 window. # -# Since 2026-09-28 this endpoint is served by gufo (gufo-org/gufo), the -# Strix Halo inference engine built from the pinned v0.1.1 tag, serving -# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) UD-Q8_K_XL with the z-lab -# DFlash2 Q4_K_M drafter. Two changes landed on this endpoint that day: the -# engine (from the stock llama.cpp prebuilt) and the payload (from -# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL). Port, alias and client contract did -# not move. +# This seam exists because the format is the constraint: GGML_TYPE_Q8_0_ROCMFPX +# (tensor type 103) is understood by the ROCmFPX fork and by nothing else on +# this host, so the runtime, the payload and the deployer's gate have to agree +# on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and +# its deployer twin disagreed during a cutover and the endpoint crash-looped; +# both sides are pinned here. # -# Most of this seam runs the serve script against a stub engine and asserts -# on the argv it actually execs, so what is pinned is the command the -# operator gets rather than the text of the script: a tunable renamed, -# dropped, or silently defaulted away fails here the way it would fail a -# deployment. -load helpers - -ENV_FILE="$REPO_ROOT/scripts/frank-large.env" -UNIT="$REPO_ROOT/scripts/frank-large.service" -SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" -FETCH="$REPO_ROOT/scripts/download-frank-large-q4.sh" -DRAFTER_FETCH="$REPO_ROOT/scripts/download-frank-large.sh" -BUILD="$REPO_ROOT/scripts/build-gufo.sh" -REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh" -# The deployed target's filename, kept here so a rename in the env file and a -# rename in this stub cannot drift apart unnoticed. -TARGET_STUB="Qwen3.8-27B-UD-Q4_K_XL.gguf" +# Everything is tested from what an operator or a supervisor can observe — the +# command line, the exit code, the deployer's error text — never from anything +# inside llama.cpp. setup() { - STAGE="$(mktemp -d)" - RUNTIME_DIR="$STAGE/gufo" - MODEL_DIR="$STAGE/models" - DRAFTER_DIR="$STAGE/draft" - mkdir -p "$RUNTIME_DIR/bin" "$MODEL_DIR" "$DRAFTER_DIR" - # A stub engine: the serve script execs it, so $@ is exactly what gufo - # would be invoked with. - cat > "$RUNTIME_DIR/bin/gufo" <<'STUB' -#!/usr/bin/env bash -printf '%s\n' "$@" -STUB - chmod +x "$RUNTIME_DIR/bin/gufo" - # The target and the drafter live in SEPARATE directories since the - # 2026-09-28 payload swap: the target came from a different repository, - # the DFlash2 drafter did not move. A stub that colocated them would - # never notice DRAFTER_DIR going stale. - : > "$MODEL_DIR/$TARGET_STUB" - : > "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" - STUB_ENV="$STAGE/stub.env" - cat > "$STUB_ENV" < "$TEST_TMP/bin" <<'STUB' +#!/bin/sh +printf '%s\n' "$*" +STUB + chmod +x "$TEST_TMP/bin" } -# argv_has — exact token match, so --port never satisfies a test -# looking for a different flag. -argv_has() { - grep -qxF -- "$1" <<<"$output" +@test "the env file defines the deployment values" { + grep -qE '^LLAMA_BIN=/' "$ENV_FILE" + grep -qE '^MODEL_FILE=/' "$ENV_FILE" + grep -qE '^DRAFT_FILE=/' "$ENV_FILE" + grep -qE '^ALIAS=[a-z0-9-]+$' "$ENV_FILE" + grep -qE '^LISTEN_PORT=[0-9]+$' "$ENV_FILE" + grep -qE '^CTX_SIZE=[0-9]+$' "$ENV_FILE" } -# argv_value — the token following , empty if the flag is absent. -argv_value() { - local i - i="$(grep -nx -- "$1" <<<"$output" | cut -d: -f1)" - [ -n "$i" ] || return 0 - sed -n "$((i + 1))p" <<<"$output" -} - -@test "the frank-large unit, env, serve and build files exist where the deployer installs them from" { - [ -f "$ENV_FILE" ] - [ -f "$UNIT" ] - [ -x "$SERVE" ] - [ -x "$BUILD" ] -} - -@test "frank-large is boot-persistent, self-healing and unprivileged" { - grep -q '^WantedBy=multi-user.target$' "$UNIT" - grep -q '^Restart=on-failure$' "$UNIT" - grep -q '^RestartSec=' "$UNIT" - grep -q '^User=dank$' "$UNIT" - # The engine pins buffers; the memlock budget must be unbounded or the - # load fails on the RLIMIT. - grep -q '^LimitMEMLOCK=infinity$' "$UNIT" - # The unit runs the repo's own serve script with the repo's env. - grep -q 'ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh' "$UNIT" - grep -q 'FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env' "$UNIT" -} - -@test "the fleet endpoint and alias survive the cutover" { - serve_argv - [ "$status" -eq 0 ] - # frank-large at :7770, whatever engine answers it. - [ "$(argv_value --served-model-name)" = "frank-large" ] - [ "$(argv_value --port)" = "7770" ] - [ "$(argv_value --host)" = "0.0.0.0" ] -} - -@test "the built-in defaults are one session of the full 262144 window with DFlash2" { - serve_argv - [ "$status" -eq 0 ] - # 262144 is the model's native window (the operator's call, 2026-09-21). - [ "$(argv_value --context)" -eq 262144 ] - # One session is the single stream every prior frank-large served. - [ "$(argv_value --sessions)" -eq 1 ] - [ "$(argv_value --speculative)" = "dflash2" ] - # A draft ceiling of 0 would silently serve the autoregressive rate. - [ "$(argv_value --draft-tokens)" -gt 0 ] -} - -@test "the shipped env file, not just the defaults, serves the deployed shape" { - # The defaults are a floor; systemd sources the env file. Only the - # on-host paths are redirected at the stub, so every fleet value in the - # shipped file is the one under test. MODEL_FILE and DRAFTER_FILE are - # absolute in the shipped file, so they need redirecting too — a sed that - # only rewrote MODEL_DIR would leave them pointing at /home/dank and the - # test would pass for the wrong reason (or fail on a box that has them). - sed -e "s#^RUNTIME_DIR=.*#RUNTIME_DIR=$RUNTIME_DIR#" \ - -e "s#^MODEL_DIR=.*#MODEL_DIR=$MODEL_DIR#" \ - -e "s#^MODEL_FILE=.*#MODEL_FILE=$MODEL_DIR/$TARGET_STUB#" \ - -e "s#^DRAFTER_DIR=.*#DRAFTER_DIR=$DRAFTER_DIR#" \ - -e "s#^DRAFTER_FILE=.*#DRAFTER_FILE=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf#" \ - "$ENV_FILE" > "$STAGE/shipped.env" - serve_argv "$STAGE/shipped.env" - [ "$status" -eq 0 ] - [ "$(argv_value --served-model-name)" = "frank-large" ] - [ "$(argv_value --host)" = "0.0.0.0" ] - [ "$(argv_value --port)" = "7770" ] - # 2026-10-01 night: Frank rolled frank-large back to ONE session at the - # full native window (gufo Q4_K_XL, f16 KV — gufo has no KV quant lever). - [ "$(argv_value --context)" -eq 262144 ] - [ "$(argv_value --sessions)" -eq 1 ] - [ "$(argv_value --speculative)" = "dflash2" ] - [ "$(argv_value --draft-tokens)" -gt 0 ] - [ "$(argv_value --draft-policy)" = "adaptive" ] - # The shipped file must name the target the fleet actually serves, not a - # path that merely happens to exist. - [ "$(argv_value --model)" = "$MODEL_DIR/$TARGET_STUB" ] - [ "$(argv_value --dflash-model)" = "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" ] -} - -@test "the engine is pointed at the payload that is actually on disk" { - serve_argv - [ "$status" -eq 0 ] - [ -f "$(argv_value --model)" ] - # A drafter path that does not exist would load the target and decode at - # the autoregressive rate, which reads as "the new runtime is slow". - [ -f "$(argv_value --dflash-model)" ] - [ "$(argv_value --model)" != "$(argv_value --dflash-model)" ] -} - -@test "frank-large stays text-only" { - # The payload has no mmproj, so an --mmproj would point at nothing. The - # fleet's image path is frank-small: its SGLang MiMo payload carries the - # vision tower inside the checkpoint (no projector file at all), and - # models.yml declares frank-large text-only. - serve_argv - ! argv_has --mmproj - grep -q 'MiMo-Q6K' "$REPO_ROOT/scripts/frank-small.env" -} - -@test "the engine is asked for greedy decoding and for preserved prior reasoning" { - # Greedy is what this endpoint has always served; a silent switch to - # sampled output would be a model-behaviour change disguised as a - # runtime swap. Prior thinking blocks must survive across turns or a - # multi-turn agent loses its own reasoning between tool calls. - serve_argv - [ "$(argv_value --temperature)" = "0" ] - [ "$(argv_value --preserve-thinking)" = "on" ] -} - -@test "the serve script refuses a missing runtime with the build pointer" { - rm -f "$RUNTIME_DIR/bin/gufo" - serve_argv +@test "the serve script fails when the runtime is missing" { + # Only the model, the head and the projector exist; the fork was never + # built. llama.cpp is not reached, so this fails fast and says what to + # run instead of failing at bind time. + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/nope/llama-server" \ + "MODEL_FILE=$TEST_TMP/model.gguf" "DRAFT_FILE=$TEST_TMP/draft.gguf" \ + "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"runtime not found"* ]] - [[ "$output" == *"build-gufo.sh"* ]] - # A restart loop would be the alternative; nothing may be exec'd. - ! argv_has serve + [[ "$output" == *"build it with scripts/build-rocmfpx.sh"* ]] } -@test "the serve script refuses a missing drafter with a fetch pointer" { - # The drafter lives in its own directory now, so the target can be present - # and this still has to be caught. - rm -f "$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf" - serve_argv +@test "the serve script refuses to start without the model file" { + make_stub + touch "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/absent.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"missing asset"* ]] - [[ "$output" == *"Qwen3.8-27B-DFlash2-Q4_K_M.gguf"* ]] - [[ "$output" == *"download-frank-large"* ]] - ! argv_has serve + [[ "$output" == *"is not readable"* ]] + [[ "$output" == *"download-frank-large-kingjones.sh"* ]] } -@test "the serve script refuses a missing target and names the file" { - rm -f "$MODEL_DIR/$TARGET_STUB" - serve_argv +@test "the serve script refuses to start without the MTP draft head" { + make_stub + touch "$TEST_TMP/model.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/absent.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"missing asset"* ]] - [[ "$output" == *"$TARGET_STUB"* ]] + [[ "$output" == *"MTP draft head"* ]] } -@test "both payload files are fetched resumably and verified against declared sizes + sha256" { - # The target and the drafter come from DIFFERENT repositories, fetched by - # two scripts. Anchor the repo checks to the URL ASSIGNMENT, not to the - # file: the same repo string also appears in each script's header comment, - # so a plain grep stayed green after the URL was repointed at another - # repository -- the script would then download something the serve script - # never asked for. - grep -qE '^URL="https://huggingface\.co/unsloth/Qwen3\.8-27B-GGUF/resolve/main/' "$FETCH" - grep -qE 'https://huggingface\.co/z-lab/Qwen3\.8-27B-DFlash2-GGUF/resolve/main/' "$DRAFTER_FETCH" - for f in "$FETCH" "$DRAFTER_FETCH"; do - # -f so an HTTP error is a failure rather than a small "artifact"; - # -C - so an interrupted 29 GiB run resumes. - grep -q -- '-fL -C -' "$f" - # The size and the sha256 are compared after the transfer, not trusted. - grep -q 'SHA256 MISMATCH' "$f" - grep -q 'SIZE MISMATCH' "$f" +@test "the serve script refuses a missing mmproj unless vision is opted out" { + make_stub + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/absent.gguf" "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"set MMPROJ_FILE='' for text-only"* ]] + # The documented opt-out must actually start, without a --mmproj argument. + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=" "$SERVE" + [ "$status" -eq 0 ] + [[ "$output" != *"--mmproj"* ]] +} + +@test "the serve script fails when LLAMA_BIN is empty" { + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env LLAMA_BIN= "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"LLAMA_BIN is empty"* ]] +} + +@test "the shipped env points at the ROCmFPX runtime and the AGENT payload" { + grep -qE "^LLAMA_BIN=$RUNTIME_STUB$" "$ENV_FILE" + grep -qE "^MODEL_FILE=$MODEL_STUB$" "$ENV_FILE" + grep -qE "^DRAFT_FILE=$DRAFT_STUB$" "$ENV_FILE" + grep -qE "^MMPROJ_FILE=$MMPROJ_STUB$" "$ENV_FILE" + # HIP reads bare HOST/TEMP/GPU_MAX_HEAP_SIZE out of the environment that + # `set -a` creates; a tunable named HOST leaks into the runtime. + ! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" +} + +@test "the argv contract keeps the fleet's port and alias" { + make_stub + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" \ + ALIAS=frank-large BIND_ADDR=127.0.0.1 LISTEN_PORT=7770 "$SERVE" + [ "$status" -eq 0 ] + # The caller's values must survive the env file, or a single-run operator + # override silently does nothing. + [[ "$output" == *"--host 127.0.0.1"* ]] + [[ "$output" == *"--port 7770"* ]] + [[ "$output" == *"--alias frank-large"* ]] + # Vision is wired: the projector is the whole vision tower. + [[ "$output" == *"--mmproj $TEST_TMP/mmproj.gguf"* ]] +} + +@test "the shipped argv is one full native window on one session" { + grep -qE '^CTX_SIZE=262144$' "$ENV_FILE" + grep -qE '^PARALLEL=1$' "$ENV_FILE" + # 262144 IS the model's trained window (general.context_length in the + # GGUF), so 1 slot gets the whole thing — no truncation, no yarn scaling. + # + # Budget, from the GGUF header: 26.28 GiB of weights + 1.57 GiB draft head + # + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x + # 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB + # of DeltaNet state + a couple GiB of compute buffers, against the iGPU's + # ~114 GiB GTT aperture. Change CTX_SIZE and this arithmetic, not a boot + # log, is what tells you it no longer fits. + ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" + kv_gib=$((ctx * 65536 / 1073741824)) + total_gib=$((27 + 2 + 1 + kv_gib + 3)) + [ "$total_gib" -le 114 ] +} + +@test "the MTP draft contract matches the model card's measured optimum" { + make_stub + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" + [ "$status" -eq 0 ] + # The head is the model's own nextn layer, loaded as a separate draft + # model because this checkpoint ships it standalone (no blk..nextn.* + # tensors in the target file). + [[ "$output" == *"--spec-type draft-mtp"* ]] + [[ "$output" == *"--model-draft $TEST_TMP/draft.gguf"* ]] + # n-max 4 is the measured optimum; n-max 8 measured WORSE (acceptance + # fell to 0.757 and the extra verification cost outran the savings), so + # the flag is a measurement, not a wish. + [[ "$output" == *"--spec-draft-n-max 4"* ]] + [[ "$output" == *"--spec-draft-n-min 0"* ]] + [[ "$output" == *"--spec-draft-p-min 0.0"* ]] + # The draft runs on the iGPU beside the target: no device-to-device hop + # on a link that measures single-digit GB/s. + [[ "$output" == *"--spec-draft-device ROCm0"* ]] + [[ "$output" == *"--spec-draft-ngl 99"* ]] +} + +@test "KV and placement stay unquantized with flash attention on" { + make_stub + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" + [ "$status" -eq 0 ] + [[ "$output" == *"--cache-type-k f16"* ]] + [[ "$output" == *"--cache-type-v f16"* ]] + # ROCmFPx quantized KV has no kernel on this branch, and the endpoint has + # the aperture to keep KV unquantized anyway. + [[ "$output" == *"--flash-attn on"* ]] + # --fit's placement heuristics do not know that 311 tensors of type 103 + # belong on the iGPU, so the whole stack is placed explicitly instead. + [[ "$output" == *"--fit off"* ]] + [[ "$output" == *"--n-gpu-layers 999"* ]] +} + +@test "the client contract wires thinking and template kwargs" { + make_stub + touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" + run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ + "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" + [ "$status" -eq 0 ] + # Without --jinja, llama.cpp silently ignores the harness's + # chat_template_kwargs, so thinking control disappears from the API. + [[ "$output" == *"--jinja"* ]] + # The harness reads `reasoning_content` for this endpoint, which is what + # the deepseek emission format produces (vLLM's `reasoning` is not it). + [[ "$output" == *"--reasoning-format deepseek"* ]] + # The template needs an unbounded thinking budget. + [[ "$output" == *"--reasoning-budget -1"* ]] +} + +@test "the deployer gates exactly the paths the serve script opens" { + # A gate that checks a different file than the engine opens is the + # 2026-10-01 crash-loop: deploy passed, the unit restart-looped. Read the + # engine's own paths out of the env file and require the deployer's + # defaults to name the same ones. + for pair in "LLAMA_BIN:LLM_LARGE_RUNTIME" \ + "MODEL_FILE:LLM_LARGE_MODEL" \ + "DRAFT_FILE:LLM_LARGE_DRAFT" \ + "MMPROJ_FILE:LLM_LARGE_MMPROJ"; do + env_key="${pair%%:*}" + dep_key="${pair##*:}" + want="$(sed -nE "s|^${env_key}=(.*)\$|\1|p" "$ENV_FILE")" + [ -n "$want" ] + got="$(grep -oE "${dep_key}:-[^}]+" "$DEPLOYER" | head -1 | sed "s|^${dep_key}:-||")" + [ "$got" = "$want" ] done } -@test "the serve script and the fetch scripts agree on the payload filenames" { - # The serve script's MODEL_FILE and the fetch script's REL are two - # independent spellings of one filename, and a repo is not required to - # keep them in step: a re-upload under a new name, or an env edited - # without the fetch, leaves the endpoint unable to start with a - # "missing asset" that no test would have predicted. Bind them here. - local served fetched drafter_served drafter_fetched - served="$(basename "$(grep '^MODEL_FILE=' "$ENV_FILE" | cut -d= -f2-)")" - fetched="$(sed -n 's/^REL="\(.*\)"$/\1/p' "$FETCH")" - [ -n "$served" ] - [ "$served" = "$fetched" ] - drafter_served="$(basename "$(grep '^DRAFTER_FILE=' "$ENV_FILE" | cut -d= -f2-)")" - drafter_fetched="$(sed -n 's/^ *"\(Qwen3\.8-27B-DFlash2[^ ]*\.gguf\) .*/\1/p' "$DRAFTER_FETCH")" - [ -n "$drafter_served" ] - [ "$drafter_served" = "$drafter_fetched" ] +@test "the deployer points a missing runtime at the fork build script" { + grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER" + grep -qE 'fetch it: bash scripts/download-frank-large-kingjones\.sh' "$DEPLOYER" } -@test "the drafter is still fetched by the script that carries its Hub repo" { - # Guards a future "simplification" that folds the drafter into the - # payload-fetch script and silently orphans the artifact the serve script - # still points at via DRAFTER_DIR. The literal drafter name must not - # appear in that script AT ALL, comments included. - grep -q 'Qwen3.8-27B-UD-Q8_K_XL' "$DRAFTER_FETCH" - ! grep -q 'DFlash2' "$FETCH" +@test "the stray-process sweep cannot take frank-small down with frank-large" { + # frank-small serves a different GGUF over llama-server on the SAME host. + # A sweep keyed on the binary name (`pkill -f llama-server`) would kill + # the 3090 endpoint too, so it is keyed on frank-large's own payload file. + grep -q "pkill -f 'Qwen3.8-27B-Q8_0_ROCMFPX_AGENT'" "$DEPLOYER" + ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER" } @test "the build script refuses a source tree that is not the pinned commit" { - # gufo ships no prebuilt Linux binaries, so the runtime is built here. - # A tag that moves under us would silently install different kernels - # under the same RUNTIME_DIR, so the pin is enforced, not documented. - local src="$STAGE/not-the-pinned-commit" - mkdir -p "$src" - git -C "$src" init -q - git -C "$src" -c user.email=t@t -c user.name=t commit -q --allow-empty -m drift - run env GUFO_SRC_DIR="$src" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify + # The pin is load-bearing: the older builds of this fork line segfault in + # ggml_compute_forward_set_rows on this exact payload. A runtime that + # differs from what was qualified must be heard about at deploy time. + wrong="$(git -C "$SRC" rev-parse HEAD)" + run env ROCMFPX_SRC_DIR="$SRC" ROCMFPX_COMMIT="ffff$wrong" \ + "$BUILD" --verify [ "$status" -ne 0 ] - [[ "$output" == *"GUFO_COMMIT mismatch"* ]] + [[ "$output" == *"ROCMFPX_COMMIT"*mismatch* ]] } -@test "the build script reports a runtime that was never built" { - # --verify on a host with no install must say so rather than exit 0 on - # an empty check. - run env GUFO_SRC_DIR="$STAGE/no-source-tree" GUFO_PREFIX="$STAGE/absent-prefix" bash "$BUILD" --verify - [ "$status" -ne 0 ] - [[ "$output" == *"no gufo at"* ]] +@test "the build script pins a real commit and the gfx1151 target" { + grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD" + grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD" + grep -qE 'DGPU_TARGETS=' "$BUILD" + # The webui asset fetch half-failed on this box and aborted the link at + # 98%; the build must not depend on it. + grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD" } -@test "the deploy handler gates on the runtime, the payload and the port" { - grep -q 'relink_unit frank-large' "$REMOTE_DEPLOY" - grep -q 'systemctl enable frank-large.service' "$REMOTE_DEPLOY" - grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY" - grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_RUNTIME' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_MODEL' "$REMOTE_DEPLOY" - grep -q 'LLM_LARGE_DRAFT' "$REMOTE_DEPLOY" +@test "the deployed unit keeps the fleet's restart and boot-persistence shape" { + grep -qE '^Restart=(always|on-failure)$' "$UNIT" + grep -qE '^WantedBy=' "$UNIT" + grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT" + grep -qE '^User=dank$' "$UNIT" + # The payload is 26 GiB of mmap'd weights plus pinned staging buffers; an + # RLIMIT smaller than the pinned set fails the load in a way that looks + # like an engine crash. + grep -qE '^LimitMEMLOCK=infinity$' "$UNIT" } -@test "the deploy handler will not stop a serving endpoint it cannot replace" { - # The runtime gate must be live (a real conditional on an executable - # path, not a branch that never fires) and it must return BEFORE the - # stop, or a host whose gufo was never built takes frank-large down on - # its way to reporting the failure. - local large gate bail stop - large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" - [[ "$large" == *'build-gufo.sh'* ]] - gate="$(grep -n 'if \[\[ ! -x "\$LLM_LARGE_RUNTIME" \]\]' <<<"$large" | head -1 | cut -d: -f1)" - stop="$(grep -n 'systemctl stop frank-large' <<<"$large" | head -1 | cut -d: -f1)" - [ -n "$gate" ] - [ -n "$stop" ] - bail="$(awk -v g="$gate" 'NR > g && /return 1/ { print NR; exit }' <<<"$large")" - [ -n "$bail" ] - [ "$bail" -lt "$stop" ] -} - -@test "the stray-run sweep keys on the drafter and cannot take frank-small down" { - # The sweep used to key on the TARGET filename, which the 2026-09-28 - # payload swap moved to a different repository. It would have stopped - # matching the live process, and a stray run would then hold 7770 past the - # port gate and fail the deploy. The drafter is the part that did NOT - # move, and every frank-large command line carries it via --dflash-model. - grep -q "pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M'" "$REMOTE_DEPLOY" - # frank-small serves a different GGUF on the 3090 with no DFlash2 drafter, - # so a binary-name sweep is the only thing that would endanger it. - local large - large="$(sed -n '/^deploy_llm_large() {/,/^}/p' "$REMOTE_DEPLOY")" - ! grep -q 'pkill -x llama-server' <<<"$large" - ! grep -q 'pkill -x gufo' <<<"$large" -} - -@test "the deploy handler retires the halogen quadlet without deleting it" { - grep -q 'retire_halogen_large_quadlet' "$REMOTE_DEPLOY" - grep -q 'HOME/.AGENTTRASH' "$REMOTE_DEPLOY" - ! grep -qE '(^|[^[:alnum:]_])rm (-[a-zA-Z]+ )?.*frank-large' "$REMOTE_DEPLOY" - # Nothing live ships for this endpoint as a container any more. The - # archived copy is host state in an ignored .AGENTTRASH — deliberately - # untracked, and present only in a primary checkout — so search the - # tracked tree, not the filesystem. - [ ! -e "$REPO_ROOT/dankclaw/frank-large/quadlet/frank-large.container" ] - ! git -C "$REPO_ROOT" grep -q 'Image=.*halogen' -- dankclaw +@test "the unit points at no retired frank-large runtime path" { + # The host keeps every retired payload and build on disk, so a stale path + # in a unit still resolves — to a runtime nobody has measured in weeks. + ! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT" + ! grep -q 'halogen' "$UNIT" }