From 7d3dd33de0dee410125c2cdc24245ae9cbe7a663 Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 3 Oct 2026 02:04:22 -0700 Subject: [PATCH] frank-large: swap to the uncensored Q6_0_ROCMFPX_AGENT payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frank's order: replace frank-large with kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF's Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf, following that card. The file is ftype 114 — 339 GGML_TYPE_Q6_0_ROCMFPX (102) + 159 GGML_TYPE_Q8_0_ROCMFPX (103) + 353 F32 over 851 tensors, qwen35/64 layers/full_attention_interval 4/4 KV heads/262,144 window — read out of its own header. The pinned ROCmFPX build already knows both types (ggml.h 102/103 plus convert/cpy/getrows/mmq/fattn device paths for 102), so no rebuild: the fork stays at 36e9acd and the payload directory is the only new host state. Card compliance, all already our flags: Q4_0 draft head (not Q8_0), --spec-draft-n-max 4 (the knee on both cards), --spec-draft-ngl 99, --jinja, --fit off, -fa on, -dio, one slot at full window, thinking with the card's own sampling triple (which the file also embeds as its general.sampling defaults). Kept from the deployed shape: the BF16 projector (this repo ships only a Q8_0 one; BF16 is the file verified 4/4 on spatial ground truth here) and the byte-identical MTP head, which the new script hardlinks instead of re-fetching 1.6 GiB. Franksdank-Runbook: serverAppConfigs-docs/LLM/frank-large-rocmfpx-q6agent-uncensored-mtp.md --- scripts/deploy-remote.sh | 39 +++--- ...wnload-frank-large-kingjones-uncensored.sh | 116 ++++++++++++++++++ scripts/frank-large.env | 78 +++++++----- scripts/serve-frank-large.sh | 59 +++++---- tests/frank_large_unit.bats | 35 +++--- 5 files changed, 236 insertions(+), 91 deletions(-) create mode 100755 scripts/download-frank-large-kingjones-uncensored.sh diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 6acae9d..72c2fa8 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1088,27 +1088,30 @@ deploy_anubis() { # frank-large is the ROCmFPX fork of llama.cpp (kingjones30/ROCmFPX, pinned by # scripts/build-rocmfpx.sh), serving -# kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF:Q8_0_ROCMFPX_AGENT with the -# repo's own Q4_0 MTP draft head and its BF16 vision projector, on the Strix -# Halo iGPU. It is a ROOT unit: the runtime is a native binary linked against -# the host ROCm, so there is no container image to install and no generated -# unit to dodge. relink_unit points /etc/systemd/system/frank-large.service at -# the repo file and boot persistence is that unit's [Install] section. +# kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF:Q6_0_ROCMFPX_AGENT +# with the repo's own Q4_0 MTP draft head and the BF16 vision projector, on the +# Strix Halo iGPU. It is a ROOT unit: the runtime is a native binary linked +# against the host ROCm, so there is no container image to install and no +# generated unit to dodge. relink_unit points +# /etc/systemd/system/frank-large.service at the repo file and boot persistence +# is that unit's [Install] section. # # The runtime gate matters as much as the payload gate: the fork is built from -# source on this host, and the payload is ftype 115 (tensor type 103, -# GGML_TYPE_Q8_0_ROCMFPX) which only this fork can read — so a machine whose -# /home/dank/GitPackages tree was never built has a unit that can never -# become healthy, and a green deploy would leave the endpoint dark. +# source on this host, and the payload is ftype 114 (tensor types 102 +# GGML_TYPE_Q6_0_ROCMFPX and 103 GGML_TYPE_Q8_0_ROCMFPX) which only this fork +# can read — so a machine whose /home/dank/GitPackages tree was never built has +# a unit that can never become healthy, and a green deploy would leave the +# endpoint dark. # -# It replaces gufo v0.1.1 + unsloth UD-Q4_K_XL (the rollback target, still -# built by scripts/build-gufo.sh with its payload on disk), which had replaced -# the stock llama.cpp prebuilt. The port gate is the same as every cutover +# It replaces the ALIGNED kingjones Q8_0_ROCMFPX_AGENT file (rollback target: +# its payload and its draft head are still on disk in +# ~/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP, where this payload's +# draft-head hardlink also lives). The port gate is the same as every cutover # since 2026-09-16: nothing starts until 7770 is released. LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" -LLM_LARGE_MMPROJ="${LLM_LARGE_MMPROJ:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" +LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}" +LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" +LLM_LARGE_MMPROJ="${LLM_LARGE_MMPROJ:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1155,7 +1158,7 @@ deploy_llm_large() { for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_MMPROJ"; do if [[ ! -f "$f" ]]; then echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-kingjones.sh" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-kingjones-uncensored.sh" >&2 missing=1 fi done @@ -1172,7 +1175,7 @@ deploy_llm_large() { # down with it. The pattern never appears in the deploying shell's own # command line, so -f is safe here (it is not in `bash deploy-remote.sh`). run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-27B-Q8_0_ROCMFPX_AGENT' 2>/dev/null || true + pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT' 2>/dev/null || true # Nothing may start until the predecessor has released the port. On # 2026-09-16 the cutover failed exactly here: the old server logged # "Deactivated successfully", the next engine tried to bind 7770 a second diff --git a/scripts/download-frank-large-kingjones-uncensored.sh b/scripts/download-frank-large-kingjones-uncensored.sh new file mode 100755 index 0000000..060a161 --- /dev/null +++ b/scripts/download-frank-large-kingjones-uncensored.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Download kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF for +# frank-large (Dankclaw iGPU). Public repo — no auth header, and never a token +# on the command line (it is world-readable in /proc//cmdline). +# +# The payload is the repo's Q6 AGENT rung: +# * Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf — ftype 114, 23.15 GiB. +# Q6_0_ROCMFPX tensor type (102) + the AGENT routing, which keeps the +# output-head and attention tensors at plain Q8_0 — the variant the card +# names for MTP serving ("Take Q6 AGENT (114) if you want protected +# heads"). Like every ROCmFPX file it loads on nothing but the fork. +# +# The other two files frank-large needs are NOT re-downloaded: +# * mtp-Qwen3.8-27B-Q4_0.gguf — the uncensored repo redistributes the +# ALIGNED repo's draft head byte for byte (identical Hub LFS oid +# 051a1764…), so it is hardlinked out of the aligned payload dir when that +# dir holds the verified bytes, and fetched from this repo otherwise. +# * mmproj-Qwen3.8-27B-BF16.gguf — this repo ships a Q8_0 projector; the +# fleet keeps the BF16 one, which is the projector that was verified 4/4 +# on spatial ground truth on this host. Hardlinked the same way, fetched +# from the ALIGNED repo if it is not on disk. +# +# Fresh downloads, no resume (`curl -C -` once produced an oversized file on a +# 31 GiB payload); -f/--fail-with-body so an HTTP 401 is not a success, then +# byte count and the Hub LFS sha256. Idempotent: complete files are skipped. +# +# Usage: +# scripts/download-frank-large-kingjones-uncensored.sh [DEST_DIR] [--verify-only] +set -euo pipefail + +REPO="kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF" +ALIGNED_REPO="kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF" +BASE="https://huggingface.co/${REPO}/resolve/main" +ALIGNED_BASE="https://huggingface.co/${ALIGNED_REPO}/resolve/main" +SOURCE_DIR="${SOURCE_DIR:-/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP}" + +DEST="" +VERIFY_ONLY=0 +for arg in "$@"; do + case "$arg" in + --verify-only) VERIFY_ONLY=1 ;; + *) DEST="$arg" ;; + esac +done +DEST="${DEST:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP}" + +# name bytes sha256 base_url may_link — may_link=1 means the same bytes may +# already exist in SOURCE_DIR (hardlink them instead of re-fetching). +FILES=( + "Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf 24856607008 e966deb3d1a5a1b9ca7765126292f2d822f3dccdedb3b4872e673dec1cddc199 $BASE 0" + # The draft head is redistributed byte-identical by both repos; the BF16 + # projector only exists in the aligned repo. + "mtp-Qwen3.8-27B-Q4_0.gguf 1680271648 051a1764cff8c4f3ee6ae8b00593a0364c7539c67fa50ffc58f3f96509fca38e $BASE 1" + "mmproj-Qwen3.8-27B-BF16.gguf 931145888 de2a49866988ea272c43dc2a43ee2662c25725a717ca5b9059ea420a97f53fe8 $ALIGNED_BASE 1" +) + +matches() { # matches FILE SIZE SHA — byte count AND LFS sha256 + [[ -f "$1" ]] && [[ "$(stat -c%s "$1")" == "$2" ]] \ + && sha256sum -c --quiet <(printf '%s %s\n' "$3" "$1") +} + +mkdir -p "$DEST" +status=0 + +fetch() { # fetch NAME SIZE SHA BASE_URL + local name="$1" size="$2" sha="$3" url="$4" f="$DEST/$1" + echo "FETCHING $name" + curl -fsSL --fail-with-body -o "$f.part" "$url/$name" + mv "$f.part" "$f" + if [[ "$(stat -c%s "$f")" != "$size" ]]; then + echo "SIZE FAIL $name ($(stat -c%s "$f") != $size)" + return 1 + fi + if ! sha256sum -c --quiet <(printf '%s %s\n' "$sha" "$f"); then + echo "SHA FAIL $name" + return 1 + fi + echo "VERIFIED $name" +} + +for entry in "${FILES[@]}"; do + read -r name size sha url may_link <<<"$entry" + f="$DEST/$name" + if matches "$f" "$size" "$sha"; then + echo "OK $name" + continue + fi + if (( VERIFY_ONLY )); then + echo "MISSING $name" + status=1 + continue + fi + # Shared bytes: prefer the local copy (same filesystem, so a hardlink costs + # nothing) over re-downloading gigabytes that hash identically. + if [[ "$may_link" == 1 && -n "${SOURCE_DIR:-}" ]] && matches "$SOURCE_DIR/$name" "$size" "$sha"; then + echo "LINKING $name from $SOURCE_DIR" + ln -f "$SOURCE_DIR/$name" "$f" 2>/dev/null || cp -f "$SOURCE_DIR/$name" "$f" + if matches "$f" "$size" "$sha"; then + echo "VERIFIED $name" + continue + fi + echo "SHA FAIL $name (linked copy)" + status=1 + continue + fi + if ! fetch "$name" "$size" "$sha" "$url"; then + status=1 + fi +done + +if (( status == 0 )); then + echo "DOWNLOAD_DONE $DEST" +else + echo "DOWNLOAD_FAILED $DEST" >&2 +fi +exit "$status" diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 3eb9b3c..7cad06b 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,37 +1,47 @@ # frank-large — ROCmFPX llama.cpp serving -# kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF (Q8_0_ROCMFPX_AGENT, ftype -# 115) + its Q4_0 MTP draft head, on the Strix Halo iGPU (gfx1151), :7770. +# kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF +# (Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT, ftype 114) + its Q4_0 MTP draft +# head, on the Strix Halo iGPU (gfx1151), :7770. # -# 2026-10-02 (Frank): "the highest quality of that repo, one full context, -# whatever runtime it requires". The highest-quality rung is Q8_0_ROCMFPX_AGENT -# (26.28 GiB, 8.39 bpw: 187 tensors at plain Q8_0 + 311 at -# GGML_TYPE_Q8_0_ROCMFPX); the AGENT routing is also the variant the card -# tells you to take with MTP, because it keeps the output-head tensors at full -# Q8_0 (187 vs 2), which is what the draft scores against. The runtime the -# format REQUIRES is the ROCmFPX fork — type 103 exists nowhere else. +# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT (ftype 115) +# to this repo's Q6 AGENT rung — same card, same recipe, abliterated weights. +# The card is blunt about what that buys: refusal counts go from 23/24 refuse +# to 23/24 comply with the quality score byte-identical (6/8 both arms), and +# "abliteration removes content-refusal. It does not add capability." It costs +# one quant rung: 8.39 bpw → ~6.6 bpw, 26.28 GiB → 23.15 GiB. # -# The shape: one session at the model's full native window (262,144), which is -# the whole point — the Q8_0 head costs quality the fleet wanted back, and -# the draft head pays for the bandwidth the bigger file spends. +# What the file actually contains (read out of its own GGUF header, not from +# the card): arch qwen35, 64 layers, full_attention_interval 4, 24 heads / +# 4 KV heads / 256 head_dim, context_length 262144, and 851 tensors — 353 F32 +# + 339 GGML_TYPE_Q6_0_ROCMFPX (102) + 159 GGML_TYPE_Q8_0_ROCMFPX (103). The +# AGENT routing is what puts the embeddings, the output head and every fourth +# attention projection at the Q8 rung. Two custom tensor types, both fork-only +# → the runtime is chosen by the file, exactly as it was for ftype 115. +# +# The shape is unchanged: one session at the model's full native window, MTP +# drafting at n-max 4, unquantized KV. # # Budget (from the GGUF header, not from a boot log): attention KV lives only # in the 16 full-attention layers (4 KV heads x 256 head_dim), 64 KiB/token # with f16 K and V => 16.0 GiB at 262,144. The 48 Gated DeltaNet layers hold -# ~144 MiB of fixed state per sequence. Weights 26.28 + draft head 1.57 + -# mmproj 0.87 + KV 16.0 + state 0.14 + compute buffers ≈ 45-47 GiB against the +# ~144 MiB of fixed state per sequence. Weights 23.15 + draft head 1.57 + +# mmproj 0.87 + KV 16.0 + state 0.14 + compute buffers ≈ 42-44 GiB against the # iGPU's ~114 GiB GTT aperture, so f16 KV (the unquantized choice) is affordable # here in a way it is not on frank-small's 24 GiB card. - -# The qualified ROCmFPX fork build (scripts/build-rocmfpx.sh pins the commit). LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server -# One directory per HF repo, created by scripts/download-frank-large-kingjones.sh. -MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf +# One directory per HF repo, created by +# scripts/download-frank-large-kingjones-uncensored.sh. +MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf # The draft head: Q4_0, NOT the Q8_0 head. The head's per-step cost is its # projection over a 248K vocabulary, so the cheaper head is faster end to end # at equal-or-better acceptance (card: +6.0% at n-max 1, +10.2% at n-max 2). -DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf -# Vision tower, separate from the LLM file. Empty string = text-only. -MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf +# The uncensored repo redistributes this file byte for byte (same Hub LFS oid); +# the download script hardlinks the verified copy instead of re-fetching 1.6 GiB. +DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf +# Vision tower, separate from the LLM file. This repo ships a Q8_0 projector; +# the fleet keeps the BF16 one, which is the file verified 4/4 on spatial +# ground truth on this host (hardlinked here, same bytes). Empty = text-only. +MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf # Fleet endpoint: same port and alias as every frank-large before it, so the # harness models.yml id and the homepage probe do not move. @@ -42,15 +52,16 @@ BIND_ADDR=0.0.0.0 LISTEN_PORT=7770 # One full native window. llama.cpp divides --ctx-size across --parallel -# slots, so 262144/1 IS the model's trained window; the card validated 65,536 -# and the fleet has been running this family at 262,144 since the gufo era. +# slots, so 262144/1 IS the window the file itself declares +# (qwen35.context_length = 262144); the card only validated 65,536 and the +# fleet has been running this family at 262,144 since the gufo era. CTX_SIZE=262144 PARALLEL=1 # Deck knobs. --ubatch-size has a measured optimum on this stack (512 peak on # the sibling qwen35 model; larger ubatch costs GiBs of compute buffer for no # speed), and --fit must stay off: --fit's auto placement does not know that -# 311 tensors of type 103 belong on the iGPU. +# 498 tensors of types 102 and 103 belong on the iGPU. BATCH_SIZE=2048 UBATCH_SIZE=512 NGL=999 @@ -61,10 +72,11 @@ CACHE_TYPE_K=f16 CACHE_TYPE_V=f16 # Speculative decode: the model ships a standalone MTP head (mtp-Q4_0.gguf, -# arch qwen35, nextn_predict_layers=1, a 65-block draft). n-max 4 with n-min 0 -# and p-min 0.0 is the card's measured optimum (+31% end to end, acceptance -# 0.926; its own n-max 8 measurement only reached 0.757 acceptance and LOST). -# The draft runs entirely on the iGPU, so cross-device traffic stays off the +# arch qwen35, nextn_predict_layers=1, a 65-block draft). n-max 4 is the knee +# on BOTH cards: aligned 30.13 / 30.30 / 27.52 at n=3/4/5 (acceptance 0.926 at +# the knee, and n-max 8 measured WORSE at 0.757), uncensored Q6 24.96 / 27.66 +# / 25.27. n-min 0 and p-min 0.0 are the card's own recipe. The draft runs +# entirely on the iGPU, so cross-device traffic stays off the # bandwidth-limited link. SPEC_TYPE=draft-mtp SPEC_DRAFT_NGL=99 @@ -73,8 +85,14 @@ SPEC_DRAFT_N_MAX=4 SPEC_DRAFT_N_MIN=0 SPEC_DRAFT_P_MIN=0.0 -# The card's thinking sampling profile, which is also what the harness sends -# in extraBody, so the endpoint behaves the same to a client that omits it. +# The card's thinking sampling profile — the same triple the file embeds as +# its own defaults (general.sampling.temp/top_p/top_k = 1.0 / 0.95 / 20) and +# what the harness sends in extraBody, so the endpoint behaves the same to a +# client that omits it. Card traps that do NOT bite here, but do not "fix" +# them by changing this: reasoning_effort "none" THROWS (use +# chat_template_kwargs enable_thinking:false), and a small max_tokens with +# thinking on returns empty content — the serve script runs +# --reasoning-budget -1, so the budget is never the limiter. TEMPERATURE=1.0 TOP_P=0.95 TOP_K=20 diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index da86080..360b7ba 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,26 +1,32 @@ #!/usr/bin/env bash -# Serve kingjones777/Qwen3.8-27B-ROCmFP4-STRIX-MTP-GGUF as frank-large on -# Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of llama.cpp. +# Serve kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF as +# frank-large on Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of +# llama.cpp. # -# 2026-10-02 (Frank): swapped from gufo + Q4_K_XL to the highest-quality file -# in the repo, one session at the native 262,144 window. +# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT to this +# repo's Q6_0_ROCMFPX_AGENT — abliterated weights, one quant rung cheaper, +# everything else (window, KV, MTP depth, sampler) unchanged. # -# Why this runtime and not the previous one: the payload is ftype 115 — 311 -# tensors of GGML_TYPE_Q8_0_ROCMFPX (103), a tensor type that exists only in -# the ROCmFPX fork. Stock llama.cpp, SGLang and gufo all refuse the file, and -# gufo additionally gates every "Qwen3.8" artifact on a template-hash -# allowlist. The qualified build is scripts/build-rocmfpx.sh's pin: it loads -# ftype 115, survives warmup (the 2026-08-25 builds of the older fork line -# segfaulted in ggml_compute_forward_set_rows on this exact file), and drafts -# with the model's own MTP head. +# Why this runtime and not another: the payload is ftype 114 — 339 tensors of +# GGML_TYPE_Q6_0_ROCMFPX (102) and 159 of GGML_TYPE_Q8_0_ROCMFPX (103), tensor +# types that exist only in the ROCmFPX fork. Stock llama.cpp, SGLang and gufo +# all refuse the file, and gufo additionally gates every "Qwen3.8" artifact on +# a template-hash allowlist. The qualified build is scripts/build-rocmfpx.sh's +# pin: it defines both types (ggml.h 102/103), it carries the device paths for +# type 102 (convert/cpy/getrows/mmq/fattn in ggml-cuda), it survives warmup — +# the 2026-08-25 builds of the older fork line segfaulted in +# ggml_compute_forward_set_rows on this family — and it drafts with the +# model's own MTP head. # -# Why this file: Q8_0_ROCMFPX_AGENT is the top of the repo's quality ladder -# (26.28 GiB, 8.39 bpw) and the AGENT routing keeps 187 tensors — the -# output-head set the draft scores against — at plain Q8_0 instead of the 2 -# the non-AGENT build keeps, which is the variant the card names for MTP -# serving. Drafting uses the repo's Q4_0 MTP head: the head's cost is its -# projection over the 248K vocabulary, so the 4-bit head is faster end to end -# at equal-or-better acceptance than its Q8_0 sibling. +# Why this file: Q6_0_ROCMFPX_AGENT (23.15 GiB, ftype 114) is the card's +# protected-head rung — "Take Q6 AGENT (114) if you want protected heads": the +# AGENT routing keeps the embeddings, the output head and every fourth +# attention projection at the Q8 rung (159 tensors), which is the variant the +# card names for MTP serving, and the refusal study was scored on this exact +# file (23/24 comply where the aligned Q8 AGENT refuses 23/24, quality score +# identical at 6/8). Drafting uses the repo's Q4_0 MTP head: the head's cost is +# its projection over the 248K vocabulary, so the 4-bit head is faster end to +# end at equal-or-better acceptance than its Q8_0 sibling. # # The client contract lives here, not in frank-large.env: # * --jinja — without it llama.cpp silently IGNORES the harness's @@ -64,9 +70,9 @@ done unset _var _kv : "${LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" -: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf}" -: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" -: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" +: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}" +: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" +: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" : "${ALIAS=frank-large}" : "${BIND_ADDR=0.0.0.0}" : "${LISTEN_PORT=7770}" @@ -106,14 +112,15 @@ unset _var _kv # makes llama-server hang rather than fail clean. [[ -r "$MODEL_FILE" ]] || { echo "frank-large: model file '$MODEL_FILE' is not readable;" >&2 - echo " fetch it with scripts/download-frank-large-kingjones.sh" >&2 + echo " fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2 exit 1 } -# The MTP head is the performance feature: without it the Q8_0 weights decode -# at memory-bandwidth speed and the endpoint loses the ~1.4x the head buys. +# The MTP head is the performance feature: without it these weights decode at +# memory-bandwidth speed (measured on the aligned sibling: 17.0 t/s with the +# head against 7.6 without) and the endpoint loses the ~2.2x the head buys. [[ -r "$DRAFT_FILE" ]] || { echo "frank-large: MTP draft head '$DRAFT_FILE' is not readable;" >&2 - echo " fetch it with scripts/download-frank-large-kingjones.sh" >&2 + echo " fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2 exit 1 } # The GGUF carries no vision tower; the projector file is the whole tower, so diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 6d3ca31..9ce87e3 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,12 +1,13 @@ #!/usr/bin/env bats -# Seam: frank-large serves kingjones777's Qwen3.8-27B Q8_0_ROCMFPX_AGENT (a -# ftype-115 ROCmFPX file) with the repo's MTP draft head on the Strix Halo -# iGPU, via the ROCmFPX fork of llama.cpp — one session at the native -# 262,144 window. +# Seam: frank-large serves kingjones777's UNCENSORED Qwen3.8-27B +# Q6_0_ROCMFPX_AGENT (a ftype-114 ROCmFPX file) with the repo's MTP draft head +# on the Strix Halo iGPU, via the ROCmFPX fork of llama.cpp — one session at +# the native 262,144 window. # -# This seam exists because the format is the constraint: GGML_TYPE_Q8_0_ROCMFPX -# (tensor type 103) is understood by the ROCmFPX fork and by nothing else on -# this host, so the runtime, the payload and the deployer's gate have to agree +# This seam exists because the format is the constraint: GGML_TYPE_Q6_0_ROCMFPX +# (tensor type 102) and GGML_TYPE_Q8_0_ROCMFPX (103) are understood by the +# ROCmFPX fork and by nothing else on this host, so the runtime, the payload +# and the deployer's gate have to agree # on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and # its deployer twin disagreed during a cutover and the endpoint crash-looped; # both sides are pinned here. @@ -23,9 +24,9 @@ setup() { DEPLOYER="$REPO_ROOT/scripts/deploy-remote.sh" BUILD="$REPO_ROOT/scripts/build-rocmfpx.sh" RUNTIME_STUB="/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server" - MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Q8_0_ROCMFPX_AGENT.gguf" - DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf" - MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf" + MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf" + DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf" + MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf" TEST_TMP="$(mktemp -d)" SRC="$TEST_TMP/src" @@ -76,7 +77,7 @@ STUB "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" [ "$status" -ne 0 ] [[ "$output" == *"is not readable"* ]] - [[ "$output" == *"download-frank-large-kingjones.sh"* ]] + [[ "$output" == *"download-frank-large-kingjones-uncensored.sh"* ]] } @test "the serve script refuses to start without the MTP draft head" { @@ -139,10 +140,10 @@ STUB @test "the shipped argv is one full native window on one session" { grep -qE '^CTX_SIZE=262144$' "$ENV_FILE" grep -qE '^PARALLEL=1$' "$ENV_FILE" - # 262144 IS the model's trained window (general.context_length in the - # GGUF), so 1 slot gets the whole thing — no truncation, no yarn scaling. + # 262144 IS the model's trained window (qwen35.context_length in the GGUF), + # so 1 slot gets the whole thing — no truncation, no yarn scaling. # - # Budget, from the GGUF header: 26.28 GiB of weights + 1.57 GiB draft head + # Budget, from the GGUF header: 23.15 GiB of weights + 1.57 GiB draft head # + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x # 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB # of DeltaNet state + a couple GiB of compute buffers, against the iGPU's @@ -150,7 +151,7 @@ STUB # log, is what tells you it no longer fits. ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" kv_gib=$((ctx * 65536 / 1073741824)) - total_gib=$((27 + 2 + 1 + kv_gib + 3)) + total_gib=$((24 + 2 + 1 + kv_gib + 3)) [ "$total_gib" -le 114 ] } @@ -230,14 +231,14 @@ STUB @test "the deployer points a missing runtime at the fork build script" { grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER" - grep -qE 'fetch it: bash scripts/download-frank-large-kingjones\.sh' "$DEPLOYER" + grep -qE 'fetch it: bash scripts/download-frank-large-kingjones-uncensored\.sh' "$DEPLOYER" } @test "the stray-process sweep cannot take frank-small down with frank-large" { # frank-small serves a different GGUF over llama-server on the SAME host. # A sweep keyed on the binary name (`pkill -f llama-server`) would kill # the 3090 endpoint too, so it is keyed on frank-large's own payload file. - grep -q "pkill -f 'Qwen3.8-27B-Q8_0_ROCMFPX_AGENT'" "$DEPLOYER" + grep -q "pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT'" "$DEPLOYER" ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER" }