From fece3ffdd0925bc186d8ca7d6ad515f8288f4422 Mon Sep 17 00:00:00 2001 From: Frank Date: Mon, 5 Oct 2026 16:29:11 -0700 Subject: [PATCH] frank-large: cut over to halogen container (qwen3.8-27b .hgn + DFlash2) Runtime swap ROCmFPX llama.cpp -> peonist-ai/halogen (image digest dc0a39a0, tag 0.1.4), serving qwen3.8-27b-p1w4d-d2.hgn (33.4 GiB, the .hgn format loads in no other engine) with HALOGEN_DRAFTER=2 (DFlash2) on ONE slot of 200700 ctx: slots>1 disables the drafter (engine law). Accepted: TEXT-ONLY (vision encoder ignored; frank-small owns images) and the censored base (rollback = revert this commit; payload, fork build and build-rocmfpx.sh pin stay on disk). - serve script: podman run under the root unit, exec podman wait so Restart=always is real; gates image digest, checkpoint byte-count, flat regular-file tokenizer, GPU nodes; engine stdout to log. - deploy handler: gates the artifacts, sweeps the legacy bare llama-server + old container name by NAME (never pkill llama-server: frank-small), retires the 2026-09 quadlet, port cutover 7770. - fetch script: -C - resume, -f, byte count, sha256, --verify-only. - seam rewritten (17 tests): halogen is now the deployed runtime. --- scripts/deploy-remote.sh | 114 +++--- scripts/download-frank-large-halogen.sh | 49 +++ scripts/frank-large.env | 192 +++++----- scripts/frank-large.service | 15 +- scripts/serve-frank-large.sh | 293 +++++++--------- tests/frank_large_unit.bats | 449 ++++++++++++------------ 6 files changed, 552 insertions(+), 560 deletions(-) create mode 100755 scripts/download-frank-large-halogen.sh diff --git a/scripts/deploy-remote.sh b/scripts/deploy-remote.sh index 91edada..dec27a3 100755 --- a/scripts/deploy-remote.sh +++ b/scripts/deploy-remote.sh @@ -1086,32 +1086,31 @@ deploy_anubis() { wait_container_healthy anubis 60 || return 1 } -# frank-large is the ROCmFPX fork of llama.cpp (kingjones30/ROCmFPX, pinned by -# scripts/build-rocmfpx.sh), serving -# kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF:Q6_0_ROCMFPX_AGENT -# with the repo's own Q4_0 MTP draft head and the BF16 vision projector, on the -# Strix Halo iGPU. It is a ROOT unit: the runtime is a native binary linked -# against the host ROCm, so there is no container image to install and no -# generated unit to dodge. relink_unit points -# /etc/systemd/system/frank-large.service at the repo file and boot persistence -# is that unit's [Install] section. +# frank-large is the halogen CONTAINER (ghcr.io/peonist-ai/halogen, +# digest-pinned through scripts/frank-large.env) serving +# peonist-ai/halogen-qwen3.8-27b — qwen3.8-27b-p1w4d-d2.hgn, 33.4 GiB, +# DFlash2 spec decode, 1 slot x 200700, text-only — on the Strix Halo +# iGPU. It stays a ROOT unit because the serve script drives `podman run` +# directly (the frank-primary pattern): relink_unit points +# /etc/systemd/system/frank-large.service at the repo file, boot +# persistence is that unit's [Install], and LimitMEMLOCK=infinity is +# fleet parity. 2026-10-05 cutover from the ROCmFPX llama.cpp shape — +# rollback = revert the swap commit's trio; that payload, its draft head +# and the fork build all stay on disk +# (~/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP, +# ~/GitPackages/kingjones-ROCmFPX). # -# The runtime gate matters as much as the payload gate: the fork is built from -# source on this host, and the payload is ftype 114 (tensor types 102 -# GGML_TYPE_Q6_0_ROCMFPX and 103 GGML_TYPE_Q8_0_ROCMFPX) which only this fork -# can read — so a machine whose /home/dank/GitPackages tree was never built has -# a unit that can never become healthy, and a green deploy would leave the -# endpoint dark. -# -# It replaces the ALIGNED kingjones Q8_0_ROCMFPX_AGENT file (rollback target: -# its payload and its draft head are still on disk in -# ~/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP, where this payload's -# draft-head hardlink also lives). The port gate is the same as every cutover -# since 2026-09-16: nothing starts until 7770 is released. -LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" -LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}" -LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" -LLM_LARGE_MMPROJ="${LLM_LARGE_MMPROJ:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" +# The gates mirror serve-frank-large.sh's predicates EXACTLY (the +# 2026-10-01 crash-loop lesson): image digest in dank's podman storage, +# the .hgn checkpoint at the exact byte count, the tokenizer a regular +# (not symlinked) file — a green deploy against a missing artifact would +# leave a restart-looping unit and a dark endpoint. The port gate is the +# same as every cutover since 2026-09-16: nothing starts until 7770 is +# released. +LLM_LARGE_IMAGE="${LLM_LARGE_IMAGE:-ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea}" +LLM_LARGE_CKPT="${LLM_LARGE_CKPT:-/home/dank/models/halogen-qwen3.8-27b/qwen3.8-27b-p1w4d-d2.hgn}" +LLM_LARGE_CKPT_BYTES="${LLM_LARGE_CKPT_BYTES:-35880329102}" +LLM_LARGE_TOKENIZER="${LLM_LARGE_TOKENIZER:-/home/dank/models/halogen-qwen3.8-27b/frank-large.tokenizer.gguf}" retire_halogen_large_quadlet() { local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet" @@ -1144,43 +1143,44 @@ wait_port_free() { } deploy_llm_large() { - # Refuse rather than restart-loop. A unit whose runtime or payload is - # missing never becomes healthy, and a green deploy would leave the - # endpoint dark with nothing in the deploy log to say why. The gates - # mirror serve-frank-large.sh's own predicates exactly: runtime - # executable, target GGUF, MTP head, mmproj. - if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then - echo "ERROR: ROCmFPX runtime absent at $LLM_LARGE_RUNTIME" >&2 - echo "frank-large: build it: bash scripts/build-rocmfpx.sh" >&2 + # Refuse rather than restart-loop; every gate below has a twin in + # serve-frank-large.sh and they must agree (2026-10-01 lesson). + # Capture podman output before grepping: `podman images | grep -q` + # races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30). + if ! grep -qF "${LLM_LARGE_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then + echo "ERROR: halogen image absent: $LLM_LARGE_IMAGE" >&2 + echo "frank-large: pull it: podman pull ${LLM_LARGE_IMAGE%@*}" >&2 + return 1 + fi + if [[ ! -f "$LLM_LARGE_CKPT" ]]; then + echo "ERROR: halogen checkpoint absent at $LLM_LARGE_CKPT" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + return 1 + fi + if [[ "$(stat -c %s "$LLM_LARGE_CKPT")" != "$LLM_LARGE_CKPT_BYTES" ]]; then + echo "ERROR: halogen checkpoint is not $LLM_LARGE_CKPT_BYTES bytes (partial download?)" >&2 + echo "frank-large: re-check: bash scripts/download-frank-large-halogen.sh --verify-only" >&2 + return 1 + fi + if [[ ! -f "$LLM_LARGE_TOKENIZER" || -L "$LLM_LARGE_TOKENIZER" ]]; then + echo "ERROR: tokenizer missing or a symlink at $LLM_LARGE_TOKENIZER" >&2 + echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2 return 1 fi - local missing=0 f - for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_MMPROJ"; do - if [[ ! -f "$f" ]]; then - echo "ERROR: payload absent at $f" >&2 - echo "frank-large: fetch it: bash scripts/download-frank-large-kingjones-uncensored.sh" >&2 - missing=1 - fi - done - [[ "$missing" -eq 0 ]] || return 1 retire_halogen_large_quadlet - # On a re-deploy the predecessor is THIS unit: stop it before waiting on - # the port, or the wait deadlocks on the server it just stopped serving. - # Then sweep a stray manual run -- that is how a bench harness leaves the - # port busy. Match the TARGET GGUF name, which every frank-large command - # line carries via --model and which no other process on this host uses: - # frank-small also runs llama-server here (with the MiMo GGUF) and must - # not be killed, so keying on the binary name would take the 3090 endpoint - # down with it. The pattern never appears in the deploying shell's own - # command line, so -f is safe here (it is not in `bash deploy-remote.sh`). + # The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops + # the container) before waiting on the port. The stray sweep is keyed + # on the FULL container name, which appears in the argv of a manual + # `podman run --name frank-large-halogen` / `podman wait` but NOT in + # this deployer's own command line (`bash deploy-remote.sh + # frank-large`) — so it cannot shoot itself in the foot (the fleet + # pkill law), and it can never match frank-small's llama-server. run_privileged systemctl stop frank-large.service 2>/dev/null || true - pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT' 2>/dev/null || true - # Nothing may start until the predecessor has released the port. On - # 2026-09-16 the cutover failed exactly here: the old server logged - # "Deactivated successfully", the next engine tried to bind 7770 a second - # later and reported "Address already in use". Gate on the port, not on - # hope. + pkill -f 'frank-large-halogen' 2>/dev/null || true + podman rm -f frank-large frank-large-halogen >/dev/null 2>&1 || true + # Nothing may start until the predecessor has released the port + # (2026-09-16: "Deactivated successfully" was not "port free"). wait_port_free 7770 || return 1 relink_unit frank-large run_privileged systemctl daemon-reload diff --git a/scripts/download-frank-large-halogen.sh b/scripts/download-frank-large-halogen.sh new file mode 100755 index 0000000..6bc6f7c --- /dev/null +++ b/scripts/download-frank-large-halogen.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# download-frank-large-halogen.sh — fetch frank-large's halogen payload: +# the qwen3.8-27b .hgn checkpoint and the export tokenizer, from the +# PUBLIC peonist-ai/halogen-qwen3.8-27b repo (no token — and per the fleet +# secret law, none would ever touch argv). +# +# The fetch law, same as every sibling script: curl -f (an access error +# must not become a "successful" 153-byte artifact), exact byte count from +# the model card, LFS sha256 verification, --verify-only to re-check an +# existing payload without downloading. +set -euo pipefail + +DEST="${FRANK_LARGE_MODEL_DIR:-$HOME/models/halogen-qwen3.8-27b}" +BASE="https://huggingface.co/peonist-ai/halogen-qwen3.8-27b/resolve/main" + +FILES=(qwen3.8-27b-p1w4d-d2.hgn frank-large.tokenizer.gguf) +# Byte sizes and sha256 (LFS oid) as published by the Hub on 2026-10-05 — +# re-check with: curl -s "https://huggingface.co/api/models/peonist-ai/halogen-qwen3.8-27b/tree/main?recursive=true" +SIZES=(35880329102 1293343) +SHAS=(40874e7a5c27efcf1c83c44374db85726fa4086355b3c8aae1b363644d7e6234 + 4c1ab3a19932f7d16eefc2c7fa60715c91133fe99c6dce683a3a7f11d463e7e9) + +VERIFY_ONLY=0 +[ "${1:-}" = "--verify-only" ] && VERIFY_ONLY=1 + +mkdir -p "$DEST" + +for i in "${!FILES[@]}"; do + f="${FILES[$i]}"; want="${SIZES[$i]}"; sha="${SHAS[$i]}" + path="$DEST/$f" + if [ "$VERIFY_ONLY" -eq 0 ]; then + echo "FETCH $f -> $path" + # -C - resumes a partial file; -f keeps HTTP errors fatal. + curl -fL --retry 5 -C - -o "$path" "$BASE/$f" + fi + [ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; } + got="$(stat -c %s "$path")" + [ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; } + echo "OK $f ($got bytes)" +done + +echo "byte counts match; verifying sha256 (one pass over 33.4 GiB)..." +for i in "${!FILES[@]}"; do + f="${FILES[$i]}"; sha="${SHAS[$i]}" + got="$(sha256sum "$DEST/$f" | cut -d' ' -f1)" + [ "$got" = "$sha" ] || { echo "SHA256 $f: $got != $sha" >&2; exit 1; } + echo "OK $f sha256" +done +echo "DOWNLOAD_DONE $DEST" diff --git a/scripts/frank-large.env b/scripts/frank-large.env index 7cad06b..13ce754 100644 --- a/scripts/frank-large.env +++ b/scripts/frank-large.env @@ -1,103 +1,103 @@ -# frank-large — ROCmFPX llama.cpp serving -# kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF -# (Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT, ftype 114) + its Q4_0 MTP draft -# head, on the Strix Halo iGPU (gfx1151), :7770. +# frank-large.env — tunables for the frank-large endpoint on Dankclaw's +# Strix Halo iGPU (gfx1151), host port 7770. # -# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT (ftype 115) -# to this repo's Q6 AGENT rung — same card, same recipe, abliterated weights. -# The card is blunt about what that buys: refusal counts go from 23/24 refuse -# to 23/24 comply with the quality score byte-identical (6/8 both arms), and -# "abliteration removes content-refusal. It does not add capability." It costs -# one quant rung: 8.39 bpw → ~6.6 bpw, 26.28 GiB → 23.15 GiB. +# RUNTIME (2026-10-05, Frank): halogen — the closed-source C++/HIP engine +# (github.com/peonist-ai/halogen-server, image ghcr.io/peonist-ai/halogen, +# digest-pinned below) serving peonist-ai/halogen-qwen3.8-27b, a ~6.32 bpw +# NVFP4-derivative quant of the BASE Qwen/Qwen3.8-27B, as a rootless +# container driven by scripts/serve-frank-large.sh. This CUTS OVER the +# 2026-10-02..10-03 ROCmFPX llama.cpp shape (uncensored +# Q6_0_ROCMFPX_AGENT + MTP, 27.6-28.1 t/s): rollback = revert the swap +# commit's trio; the ROCmFPX payload, draft head and build all stay on +# disk at /home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/. # -# What the file actually contains (read out of its own GGUF header, not from -# the card): arch qwen35, 64 layers, full_attention_interval 4, 24 heads / -# 4 KV heads / 256 head_dim, context_length 262144, and 851 tensors — 353 F32 -# + 339 GGML_TYPE_Q6_0_ROCMFPX (102) + 159 GGML_TYPE_Q8_0_ROCMFPX (103). The -# AGENT routing is what puts the embeddings, the output head and every fourth -# attention projection at the Q8 rung. Two custom tensor types, both fork-only -# → the runtime is chosen by the file, exactly as it was for ftype 115. +# WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own +# card it "will not load in transformers, vLLM, or llama.cpp". Accepted +# consequences (Frank, 2026-10-05): TEXT-ONLY (halogen ignores the model's +# vision encoder; the fleet's vision path is frank-small) and the CENSORED +# base (this is a finetune of Qwen/Qwen3.8-27B, not the abliterated file — +# refusals may return). What it buys: vendor-measured decode DFlash2 31.7 +# t/s mean (vs this endpoint's MTP 27.6-28.1 and the old halogen-era MTP +# 26.9), prefill ~566 t/s at 32k (was ~400 on the sibling engine), 1.4-2.3x +# wall-clock on prefill-heavy agent traffic, at HIGHER effective precision +# (~6.32 bpw vs ~6.6 with the old W4A4 promotion — see FLAGS.md). # -# The shape is unchanged: one session at the model's full native window, MTP -# drafting at n-max 4, unquantized KV. +# SHAPE — the two knobs that are NOT free: +# HALOGEN_KV_SLOTS=1 KEEPS SPECULATION ON. halogen makes batching and +# speculation mutually exclusive: with >1 slot the drafter is disabled +# and per-stream decode falls to serial speed (the sibling engine +# measured 41 t/s -> 14 t/s when slots were raised, and halogen's own +# FLAGS.md prints the same law). frank-large was already a one-session +# endpoint under ROCmFPX (PARALLEL=1), so slots=1 is the same contract, +# not a regression. +# HALOGEN_SLOT_CTX=200700 (Frank's window) is the per-conversation hard +# ceiling — a prompt over it is a 400 naming the limit, never a silent +# truncation. KV pool = slots x slot_ctx x 64 KiB = 12.2 GiB. Budget +# against the 121 GiB of unified memory: weights 33.4 GiB (mmap'd — page +# cache, not anon), KV 12.2, engine working set + auto prompt cache +# ~20-30, host baseline ~10 -> ~75-85 GiB in use. The 2026-09-16 halogen +# era ran the same engine class on this box with LESS headroom. +# HALOGEN_DRAFTER=2 (DFlash2) is the shipped default and the fastest +# exact drafter on this checkpoint; the .hgn BAKES BOTH drafters (MTP + +# DFlash2), so unlike the Flash-Next era there is no sidecar to mount +# and no --dflash flag — the drafter is an int knob, byte-identical +# output to greedy verification either way. # -# Budget (from the GGUF header, not from a boot log): attention KV lives only -# in the 16 full-attention layers (4 KV heads x 256 head_dim), 64 KiB/token -# with f16 K and V => 16.0 GiB at 262,144. The 48 Gated DeltaNet layers hold -# ~144 MiB of fixed state per sequence. Weights 23.15 + draft head 1.57 + -# mmproj 0.87 + KV 16.0 + state 0.14 + compute buffers ≈ 42-44 GiB against the -# iGPU's ~114 GiB GTT aperture, so f16 KV (the unquantized choice) is affordable -# here in a way it is not on frank-small's 24 GiB card. -LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server -# One directory per HF repo, created by -# scripts/download-frank-large-kingjones-uncensored.sh. -MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf -# The draft head: Q4_0, NOT the Q8_0 head. The head's per-step cost is its -# projection over a 248K vocabulary, so the cheaper head is faster end to end -# at equal-or-better acceptance (card: +6.0% at n-max 1, +10.2% at n-max 2). -# The uncensored repo redistributes this file byte for byte (same Hub LFS oid); -# the download script hardlinks the verified copy instead of re-fetching 1.6 GiB. -DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf -# Vision tower, separate from the LLM file. This repo ships a Q8_0 projector; -# the fleet keeps the BF16 one, which is the file verified 4/4 on spatial -# ground truth on this host (hardlinked here, same bytes). Empty = text-only. -MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf +# CLIENT CONTRACT deltas vs the ROCmFPX shape (the harness side lives in +# ~/.omp/agent/models.yml): +# * NO server-side sampler defaults: a request without temperature +# decodes GREEDY. The harness sends temp 1.0 / top_p 0.95 via +# extraBody, so that wire recipe is load-bearing now, not cosmetic. +# * max_tokens defaults to 8192 PER REQUEST and INCLUDES thinking; a +# reasoning reply over a small budget returns empty content with +# finish_reason "length". The harness maxTokens is 57344: compaction +# fires at 70% of 200700 = 140,490 and 140,490 + 57,344 = 197,834 <= +# 200,700 keeps prompt + budget inside the pool (the 262144-era 65536 +# would overflow). Requests over HALOGEN_MAX_TOKENS_CAP are a 400 +# naming the cap; cap and queue timeout are COUPLED (see FLAGS.md +# table) — do not move one without the other. +# * response_format json_object/json_schema is a 400 (no constrained +# decoding in this build). +# * Thinking lands in reasoning_content — same field as the llama.cpp +# era, so the harness compat block does NOT change. reasoning_effort +# accepts none|low|xhigh on this build (template default xhigh); the +# models.yml fold (high->xhigh) still applies. +# * The engine's token protocol has no auth and binds loopback INSIDE +# the container; only the frontend port is published. Keep it that way. -# Fleet endpoint: same port and alias as every frank-large before it, so the -# harness models.yml id and the homepage probe do not move. -ALIAS=frank-large -# HIP reads a bare HOST, so this is BIND_ADDR — the same reason the file -# avoids a bare TEMP. -BIND_ADDR=0.0.0.0 -LISTEN_PORT=7770 +# Image: digest-pinned (fetch law: tag pulls silently re-point; the digest +# is what was qualified). 0.1.4 = the version the model card names. +FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea +# Container identity: the halogen container name. +FRANK_LARGE_CONTAINER=frank-large-halogen +# Old quadlet-era container identity; the serve script clears stale names. +FRANK_LARGE_LEGACY_CONTAINERS=frank-large +# host:container — the frontend listens on 8731 inside (HALOGEN_API_PORT +# default); the engine port (8730) is never published. +FRANK_LARGE_PUBLISH=7770:8731 -# One full native window. llama.cpp divides --ctx-size across --parallel -# slots, so 262144/1 IS the window the file itself declares -# (qwen35.context_length = 262144); the card only validated 65,536 and the -# fleet has been running this family at 262,144 since the gufo era. -CTX_SIZE=262144 -PARALLEL=1 +FRANK_LARGE_MODEL_DIR=/home/dank/models/halogen-qwen3.8-27b +FRANK_LARGE_CHECKPOINT=qwen3.8-27b-p1w4d-d2.hgn +FRANK_LARGE_CHECKPOINT_BYTES=35880329102 +FRANK_LARGE_CHECKPOINT_SHA256=40874e7a5c27efcf1c83c44374db85726fa4086355b3c8aae1b363644d7e6234 +FRANK_LARGE_TOKENIZER=frank-large.tokenizer.gguf +FRANK_LARGE_TOKENIZER_BYTES=1293343 +FRANK_LARGE_TOKENIZER_SHA256=4c1ab3a19932f7d16eefc2c7fa60715c91133fe99c6dce683a3a7f11d463e7e9 -# Deck knobs. --ubatch-size has a measured optimum on this stack (512 peak on -# the sibling qwen35 model; larger ubatch costs GiBs of compute buffer for no -# speed), and --fit must stay off: --fit's auto placement does not know that -# 498 tensors of types 102 and 103 belong on the iGPU. -BATCH_SIZE=2048 -UBATCH_SIZE=512 -NGL=999 -FIT=off -FLASH_ATTN=on -# Unquantized KV — the quality-first choice, affordable at one slot. -CACHE_TYPE_K=f16 -CACHE_TYPE_V=f16 - -# Speculative decode: the model ships a standalone MTP head (mtp-Q4_0.gguf, -# arch qwen35, nextn_predict_layers=1, a 65-block draft). n-max 4 is the knee -# on BOTH cards: aligned 30.13 / 30.30 / 27.52 at n=3/4/5 (acceptance 0.926 at -# the knee, and n-max 8 measured WORSE at 0.757), uncensored Q6 24.96 / 27.66 -# / 25.27. n-min 0 and p-min 0.0 are the card's own recipe. The draft runs -# entirely on the iGPU, so cross-device traffic stays off the -# bandwidth-limited link. -SPEC_TYPE=draft-mtp -SPEC_DRAFT_NGL=99 -SPEC_DRAFT_DEVICE=ROCm0 -SPEC_DRAFT_N_MAX=4 -SPEC_DRAFT_N_MIN=0 -SPEC_DRAFT_P_MIN=0.0 - -# The card's thinking sampling profile — the same triple the file embeds as -# its own defaults (general.sampling.temp/top_p/top_k = 1.0 / 0.95 / 20) and -# what the harness sends in extraBody, so the endpoint behaves the same to a -# client that omits it. Card traps that do NOT bite here, but do not "fix" -# them by changing this: reasoning_effort "none" THROWS (use -# chat_template_kwargs enable_thinking:false), and a small max_tokens with -# thinking on returns empty content — the serve script runs -# --reasoning-budget -1, so the budget is never the limiter. -TEMPERATURE=1.0 -TOP_P=0.95 -TOP_K=20 -MIN_P=0.0 -PRESENCE_PENALTY=0.0 - -NUM_THREADS=16 -THREAD_BATCH=16 +# --- engine flags (docs/FLAGS.md of the engine repo; anything unset stays +# at the image default, which is what was qualified) ------------------------ +HALOGEN_MODEL_ID=frank-large +HALOGEN_SLOT_CTX=200700 +HALOGEN_KV_SLOTS=1 +HALOGEN_DRAFTER=2 +# Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds +# the GPU, the timeout bounds how long the NEXT client waits. 65536 at +# ~10 t/s serial worst case needs > 6550 s queue timeout; the defaults +# honor that. Do not lower the timeout alone. +HALOGEN_MAX_TOKENS_CAP=65536 +HALOGEN_QUEUE_TIMEOUT=7200 +# Engine prompt cache: EMPTY = auto (sized from available memory at +# startup). Pin an explicit MB only to argue with the auto figure — a +# full-context entry is ~18 GB at 262K, so a too-small pin yields a cache +# that reports enabled and never hits (the engine warns at startup). +#HALOGEN_CACHE_MB= diff --git a/scripts/frank-large.service b/scripts/frank-large.service index baebc6c..8db85c1 100644 --- a/scripts/frank-large.service +++ b/scripts/frank-large.service @@ -1,5 +1,5 @@ [Unit] -Description=ROCmFPX llama.cpp (gfx1151) kingjones777 Qwen3.8-27B Q8_0_ROCMFPX_AGENT, MTP spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU) +Description=halogen container (peonist-ai/halogen-qwen3.8-27b, DFlash2 spec decode, 1 slot x 200700, text-only), root systemd unit (dankclaw, frank-large, Strix Halo iGPU) After=network-online.target Wants=network-online.target @@ -13,10 +13,19 @@ Group=dank LimitMEMLOCK=infinity Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env" ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh -# always, not on-failure: an engine that drains and exits 0 after a worker -# fault would otherwise leave the endpoint down until someone notices. +# systemd stop alone cannot stop the container child; stop it explicitly +# (frank-primary pattern). The engine drains on SIGTERM. +ExecStop=/usr/bin/podman stop -t 60 frank-large-halogen +TimeoutStartSec=1200 +TimeoutStopSec=120 +# always, not on-failure: an engine that exits 0 after an internal fault +# would otherwise leave the endpoint down until someone notices. Restart=always RestartSec=10 +# The engine mmaps 33.4 GiB of weights and keeps a multi-GiB working set +# on a box that also hosts frank-small, ComfyUI and the Caddy/observability +# stack; make it a late OOM candidate, not the first. +OOMScoreAdjust=-500 [Install] WantedBy=multi-user.target diff --git a/scripts/serve-frank-large.sh b/scripts/serve-frank-large.sh index 360b7ba..5e4b5fb 100755 --- a/scripts/serve-frank-large.sh +++ b/scripts/serve-frank-large.sh @@ -1,186 +1,133 @@ #!/usr/bin/env bash -# Serve kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF as -# frank-large on Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of -# llama.cpp. +# serve-frank-large.sh — frank-large endpoint: halogen (closed-source +# C++/HIP engine, image ghcr.io/peonist-ai/halogen, digest-pinned in +# frank-large.env) serving peonist-ai/halogen-qwen3.8-27b on Dankclaw's +# Strix Halo iGPU, host :7770. 2026-10-05 cutover from the ROCmFPX +# llama.cpp shape — revert the swap commit's trio to roll back (payload +# and fork build stay on disk). # -# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT to this -# repo's Q6_0_ROCMFPX_AGENT — abliterated weights, one quant rung cheaper, -# everything else (window, KV, MTP depth, sampler) unchanged. +# The engine is driven by `podman run` DIRECTLY (the fleet's frank-primary +# pattern): the image entrypoint `halogen-docker` in default "all" mode +# starts engine + OpenAI frontend in one container. The frontend listens on +# 8731 inside (HALOGEN_API_PORT default) and is published as host 7770; the +# engine's own token port (8730, no auth) stays unpublished on the +# container's loopback. The image ships NO default checkpoint: +# HALOGEN_CHECKPOINT selects the file inside the /models mount. # -# Why this runtime and not another: the payload is ftype 114 — 339 tensors of -# GGML_TYPE_Q6_0_ROCMFPX (102) and 159 of GGML_TYPE_Q8_0_ROCMFPX (103), tensor -# types that exist only in the ROCmFPX fork. Stock llama.cpp, SGLang and gufo -# all refuse the file, and gufo additionally gates every "Qwen3.8" artifact on -# a template-hash allowlist. The qualified build is scripts/build-rocmfpx.sh's -# pin: it defines both types (ggml.h 102/103), it carries the device paths for -# type 102 (convert/cpy/getrows/mmq/fattn in ggml-cuda), it survives warmup — -# the 2026-08-25 builds of the older fork line segfaulted in -# ggml_compute_forward_set_rows on this family — and it drafts with the -# model's own MTP head. +# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson — +# serve gate and deploy gate disagreeing = green deploy, red unit): +# image digest present; checkpoint present at the exact byte count; +# tokenizer present, readable, and a FLAT REGULAR FILE — the engine repo +# warns that a HF cache snapshot's symlinks into blobs/ dangle inside a +# container; GPU device nodes present. # -# Why this file: Q6_0_ROCMFPX_AGENT (23.15 GiB, ftype 114) is the card's -# protected-head rung — "Take Q6 AGENT (114) if you want protected heads": the -# AGENT routing keeps the embeddings, the output head and every fourth -# attention projection at the Q8 rung (159 tensors), which is the variant the -# card names for MTP serving, and the refusal study was scored on this exact -# file (23/24 comply where the aligned Q8 AGENT refuses 23/24, quality score -# identical at 6/8). Drafting uses the repo's Q4_0 MTP head: the head's cost is -# its projection over the 248K vocabulary, so the 4-bit head is faster end to -# end at equal-or-better acceptance than its Q8_0 sibling. -# -# The client contract lives here, not in frank-large.env: -# * --jinja — without it llama.cpp silently IGNORES the harness's -# chat_template_kwargs (enable_thinking / preserve_thinking), so the -# endpoint would look like it had no thinking control at all. The GGUF -# carries the official Qwen template, so no --chat-template-file. -# * --reasoning-format deepseek — thinking is emitted as reasoning_content, -# the field the harness reads for this endpoint. -# * the template accepts reasoning_effort xhigh|medium|low only, which is -# why models.yml folds the harness's levels onto that set. -# * --alias frank-large — that is the model id the harness resolves. -# * --ctx-checkpoints / --cache-ram — prompt-cache geometry for the 16 -# sliding-window-style layers; harmless where the hybrid state forces a -# full re-process, and required for the checkpoint feature to work at all. +# Why containers instead of the bare binary (the old frank-large shape): +# the payload is .hgn and only halogen reads it; halogen ships as a +# container; the root unit keeps LimitMEMLOCK=infinity for parity and to +# outlive the login session, exactly like frank-primary's. set -euo pipefail -# Fleet values live in frank-large.env. Anything already in the environment -# wins over the file, so an operator can override one value on a single run: -# the caller's values are captured, the file is sourced, then they are put -# back. (A plain `source` would let the file clobber the environment.) -TUNABLES=(LLAMA_BIN MODEL_FILE DRAFT_FILE MMPROJ_FILE ALIAS BIND_ADDR - LISTEN_PORT CTX_SIZE PARALLEL BATCH_SIZE UBATCH_SIZE NGL FIT - FLASH_ATTN CACHE_TYPE_K CACHE_TYPE_V SPEC_TYPE SPEC_DRAFT_NGL - SPEC_DRAFT_DEVICE SPEC_DRAFT_N_MAX SPEC_DRAFT_N_MIN SPEC_DRAFT_P_MIN - TEMPERATURE TOP_P TOP_K MIN_P PRESENCE_PENALTY NUM_THREADS - THREAD_BATCH ROCM_PATH HIP_VISIBLE HSA_GFX_VERSION) -_OVERRIDES=() -for _var in "${TUNABLES[@]}"; do - [[ -n "${!_var+set}" ]] && _OVERRIDES+=("$_var=${!_var}") -done -ENV_FILE="${FRANK_LARGE_ENV_FILE:-$(dirname "$0")/frank-large.env}" -if [[ -f "$ENV_FILE" ]]; then - set -a - # shellcheck disable=SC1090 - source "$ENV_FILE" - set +a +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}" +if [ ! -f "$ENV_FILE" ]; then + echo "frank-large: env file not found: $ENV_FILE" >&2 + exit 1 fi -for _kv in "${_OVERRIDES[@]+${_OVERRIDES[@]}}"; do - export "${_kv?}" -done -unset _var _kv +set -a +# shellcheck disable=SC1090 +source "$ENV_FILE" +set +a -: "${LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}" -: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}" -: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}" -: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}" -: "${ALIAS=frank-large}" -: "${BIND_ADDR=0.0.0.0}" -: "${LISTEN_PORT=7770}" -: "${CTX_SIZE=262144}" -: "${PARALLEL=1}" -: "${BATCH_SIZE=2048}" -: "${UBATCH_SIZE=512}" -: "${NGL=999}" -: "${FIT=off}" -: "${FLASH_ATTN=on}" -: "${CACHE_TYPE_K=f16}" -: "${CACHE_TYPE_V=f16}" -: "${SPEC_TYPE=draft-mtp}" -: "${SPEC_DRAFT_NGL=99}" -: "${SPEC_DRAFT_DEVICE=ROCm0}" -: "${SPEC_DRAFT_N_MAX=4}" -: "${SPEC_DRAFT_N_MIN=0}" -: "${SPEC_DRAFT_P_MIN=0.0}" -: "${TEMPERATURE=1.0}" -: "${TOP_P=0.95}" -: "${TOP_K=20}" -: "${MIN_P=0.0}" -: "${PRESENCE_PENALTY=0.0}" -: "${NUM_THREADS=16}" -: "${THREAD_BATCH=16}" -: "${ROCM_PATH=/opt/rocm}" -: "${HIP_VISIBLE=0}" -: "${HSA_GFX_VERSION=11.5.1}" +: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE must name the digest-pinned image}" +: "${FRANK_LARGE_CONTAINER:?FRANK_LARGE_CONTAINER must name the container}" +: "${FRANK_LARGE_PUBLISH:?FRANK_LARGE_PUBLISH must be host:container}" +: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR must name the payload dir}" +: "${FRANK_LARGE_CHECKPOINT:?FRANK_LARGE_CHECKPOINT must name the .hgn inside it}" +: "${FRANK_LARGE_CHECKPOINT_BYTES:?FRANK_LARGE_CHECKPOINT_BYTES must pin the size}" +: "${FRANK_LARGE_TOKENIZER:?FRANK_LARGE_TOKENIZER must name the tokenizer file}" +: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-large)}" +: "${HALOGEN_SLOT_CTX:?HALOGEN_SLOT_CTX must be set}" +: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set (1 keeps the drafter on)}" +: "${HALOGEN_DRAFTER:?HALOGEN_DRAFTER must be set (2 = DFlash2)}" -[[ -n "$LLAMA_BIN" ]] || { echo "frank-large: LLAMA_BIN is empty" >&2; exit 1; } -[[ -x "$LLAMA_BIN" ]] || { - echo "frank-large: llama-server '$LLAMA_BIN' is not executable;" >&2 - echo " build it with scripts/build-rocmfpx.sh" >&2 +# Capture podman's output BEFORE grepping: under `set -o pipefail` the +# `podman images | grep -q` pattern races (grep exits on first match, +# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the +# 2026-09-30 deploy refusal). +DIGESTS="$(podman images --format '{{.RepoDigests}}')" +if ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$DIGESTS"; then + echo "frank-large: image $FRANK_LARGE_IMAGE not in podman storage;" >&2 + echo " pull it: podman pull ${FRANK_LARGE_IMAGE%@*}" >&2 exit 1 -} -# Refuse anything but the real payload. A path that is not a readable file -# makes llama-server hang rather than fail clean. -[[ -r "$MODEL_FILE" ]] || { - echo "frank-large: model file '$MODEL_FILE' is not readable;" >&2 - echo " fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2 - exit 1 -} -# The MTP head is the performance feature: without it these weights decode at -# memory-bandwidth speed (measured on the aligned sibling: 17.0 t/s with the -# head against 7.6 without) and the endpoint loses the ~2.2x the head buys. -[[ -r "$DRAFT_FILE" ]] || { - echo "frank-large: MTP draft head '$DRAFT_FILE' is not readable;" >&2 - echo " fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2 - exit 1 -} -# The GGUF carries no vision tower; the projector file is the whole tower, so -# a missing file means a vision request would 500 rather than degrade. -# MMPROJ_FILE="" is the documented opt-out for a text-only endpoint. -MM_ARGS=() -if [[ -n "${MMPROJ_FILE:-}" ]]; then - [[ -r "$MMPROJ_FILE" ]] || { - echo "frank-large: mmproj '$MMPROJ_FILE' is not readable; set MMPROJ_FILE='' for text-only" >&2 - exit 1 - } - MM_ARGS=(--mmproj "$MMPROJ_FILE") fi -# HIP/ROCm host environment. HIP reads bare env vars (TEMP, HOST, -# GPU_MAX_HEAP_SIZE), which is why the tunables above are named TEMPERATURE -# and BIND_ADDR. HSA_OVERRIDE_GFX_VERSION is what lets a runtime built for a -# neighbouring gfx target run on this iGPU; the qualified pin targets gfx1151 -# natively and the override is a no-op for it. -export ROCM_PATH -export HIP_VISIBLE_DEVICES="$HIP_VISIBLE" -export HSA_OVERRIDE_GFX_VERSION="$HSA_GFX_VERSION" -export LD_LIBRARY_PATH="$(dirname "$LLAMA_BIN"):${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +# The checkpoint gate is size-exact: a truncated curl exits 0 on some +# failures and a partial .hgn loads far enough to look alive, so the byte +# count from the model card is the contract (the fleet fetch law). +CKPT="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT" +if [ ! -f "$CKPT" ]; then + echo "frank-large: checkpoint $CKPT missing;" >&2 + echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + exit 1 +fi +CKPT_SIZE="$(stat -c %s "$CKPT")" +if [ "$CKPT_SIZE" != "$FRANK_LARGE_CHECKPOINT_BYTES" ]; then + echo "frank-large: checkpoint is $CKPT_SIZE bytes, expected $FRANK_LARGE_CHECKPOINT_BYTES (partial download?)" >&2 + exit 1 +fi -exec /usr/bin/env "$LLAMA_BIN" \ - --model "$MODEL_FILE" \ - "${MM_ARGS[@]}" \ - --alias "$ALIAS" \ - --host "$BIND_ADDR" \ - --port "$LISTEN_PORT" \ - --jinja \ - --device ROCm0 \ - --n-gpu-layers "$NGL" \ - --parallel "$PARALLEL" \ - --ctx-size "$CTX_SIZE" \ - --batch-size "$BATCH_SIZE" \ - --ubatch-size "$UBATCH_SIZE" \ - --threads "$NUM_THREADS" \ - --threads-batch "$THREAD_BATCH" \ - --fit "$FIT" \ - --flash-attn "$FLASH_ATTN" \ - --direct-io \ - --cache-type-k "$CACHE_TYPE_K" \ - --cache-type-v "$CACHE_TYPE_V" \ - --ctx-checkpoints 16 \ - --cache-ram 8192 \ - --cache-idle-slots \ - --temp "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --top-k "$TOP_K" \ - --min-p "$MIN_P" \ - --presence-penalty "$PRESENCE_PENALTY" \ - --reasoning on \ - --reasoning-format deepseek \ - --reasoning-budget -1 \ - --spec-type "$SPEC_TYPE" \ - --model-draft "$DRAFT_FILE" \ - --spec-draft-ngl "$SPEC_DRAFT_NGL" \ - --spec-draft-device "$SPEC_DRAFT_DEVICE" \ - --spec-draft-n-max "$SPEC_DRAFT_N_MAX" \ - --spec-draft-n-min "$SPEC_DRAFT_N_MIN" \ - --spec-draft-p-min "$SPEC_DRAFT_P_MIN" \ - --metrics +# The tokenizer must be a regular file in a FLAT directory: pointing the +# mount at a HuggingFace cache snapshot hands the container symlinks into +# a sibling blobs/ that dangle inside the mount. +TOK="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_TOKENIZER" +if [ ! -f "$TOK" ] || [ -L "$TOK" ]; then + echo "frank-large: tokenizer $TOK missing or a symlink (HF cache snapshot?);" >&2 + echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2 + exit 1 +fi + +# GPU access is mandatory but the node paths are variables so the seam can +# drive the gate on a box without an AMD GPU. +DEV_KFD="${FRANK_LARGE_DEV_KFD:-/dev/kfd}" +DEV_DRI="${FRANK_LARGE_DEV_DRI:-/dev/dri}" +if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then + echo "frank-large: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2 + exit 1 +fi +for CTR in ${FRANK_LARGE_LEGACY_CONTAINERS:-} "$FRANK_LARGE_CONTAINER"; do + podman rm -f "$CTR" >/dev/null 2>&1 || true +done + +LOG="${FRANK_LARGE_LOG:-/home/dank/logs/frank_large.log}" +mkdir -p "$(dirname "$LOG")" +# for the udev-granted supplementary group, seccomp unconfined for the ROCm +# ioctl surface, and unbounded memlock/nofile (the engine mmaps 33.4 GiB of +# weights and wants a big fd budget). No comment lines inside the backslash +# chain — a mid-chain # terminates the continuation (the 2026-10-05 +# exit-127 crash loop on the sibling serve script). +podman run -d --name "$FRANK_LARGE_CONTAINER" \ + --publish "$FRANK_LARGE_PUBLISH" \ + --ipc=host \ + --ulimit memlock=-1:-1 --ulimit nofile=1048576:1048576 \ + --device "$DEV_KFD" --device "$DEV_DRI" \ + --group-add keep-groups \ + --security-opt seccomp=unconfined \ + -e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \ + -e HALOGEN_CHECKPOINT="/models/$FRANK_LARGE_CHECKPOINT" \ + -e HALOGEN_TOKENIZER="/tokenizer/$FRANK_LARGE_TOKENIZER" \ + -e HALOGEN_SLOT_CTX="$HALOGEN_SLOT_CTX" \ + -e HALOGEN_KV_SLOTS="${HALOGEN_KV_SLOTS:-1}" \ + -e HALOGEN_DRAFTER="$HALOGEN_DRAFTER" \ + -e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \ + -e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \ + ${HALOGEN_CACHE_MB:+-e HALOGEN_CACHE_MB="$HALOGEN_CACHE_MB"} \ + -v "$FRANK_LARGE_MODEL_DIR:/models:ro" \ + -v "$FRANK_LARGE_MODEL_DIR:/tokenizer:ro" \ + "$FRANK_LARGE_IMAGE" \ + > "$LOG" 2>&1 + +# This script's lifetime IS the container's: podman wait returns the +# container's exit code, so a crash is a nonzero ExecStart and the unit's +# Restart=always is real. systemd stop goes through the unit's ExecStop +# (podman wait alone cannot stop the child). +exec podman wait "$FRANK_LARGE_CONTAINER" diff --git a/tests/frank_large_unit.bats b/tests/frank_large_unit.bats index 9ce87e3..9dea3f7 100755 --- a/tests/frank_large_unit.bats +++ b/tests/frank_large_unit.bats @@ -1,265 +1,242 @@ #!/usr/bin/env bats -# Seam: frank-large serves kingjones777's UNCENSORED Qwen3.8-27B -# Q6_0_ROCMFPX_AGENT (a ftype-114 ROCmFPX file) with the repo's MTP draft head -# on the Strix Halo iGPU, via the ROCmFPX fork of llama.cpp — one session at -# the native 262,144 window. +# Seam: frank-large serves peonist-ai/halogen-qwen3.8-27b (a .hgn halogen +# checkpoint) on the Strix Halo iGPU through the halogen CONTAINER +# (ghcr.io/peonist-ai/halogen, digest-pinned), driven by the root unit's +# serve script — one conversation of 200,700 tokens with the DFlash2 +# drafter ON. 2026-10-05 cutover from the ROCmFPX llama.cpp shape. # -# This seam exists because the format is the constraint: GGML_TYPE_Q6_0_ROCMFPX -# (tensor type 102) and GGML_TYPE_Q8_0_ROCMFPX (103) are understood by the -# ROCmFPX fork and by nothing else on this host, so the runtime, the payload -# and the deployer's gate have to agree -# on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and -# its deployer twin disagreed during a cutover and the endpoint crash-looped; -# both sides are pinned here. +# This seam exists because the artifacts are a triangle that must agree on +# exact strings the engine reads: the image DIGEST, the checkpoint file +# name AND byte count inside the /models mount, and the tokenizer's +# flat-file shape (a HF cache snapshot's symlinks dangle inside the +# container). A serve gate and a deployer gate that disagree is the +# 2026-10-01 crash-loop; both sides are pinned here, plus the one engine +# law the fleet must never "optimize" away: HALOGEN_KV_SLOTS=1 is what +# keeps speculative decoding on. # -# Everything is tested from what an operator or a supervisor can observe — the -# command line, the exit code, the deployer's error text — never from anything -# inside llama.cpp. +# Everything is tested from what an operator or supervisor can observe — +# the command line podman would run, the exit code, the deployer's error +# text — never from anything inside the engine. + setup() { - load helpers - - SERVE="$REPO_ROOT/scripts/serve-frank-large.sh" - UNIT="$REPO_ROOT/scripts/frank-large.service" - ENV_FILE="$REPO_ROOT/scripts/frank-large.env" - DEPLOYER="$REPO_ROOT/scripts/deploy-remote.sh" - BUILD="$REPO_ROOT/scripts/build-rocmfpx.sh" - RUNTIME_STUB="/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server" - MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf" - DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf" - MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf" - + ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" + SERVE="$ROOT/scripts/serve-frank-large.sh" + ENV_FILE="$ROOT/scripts/frank-large.env" + UNIT="$ROOT/scripts/frank-large.service" + DEPLOYER="$ROOT/scripts/deploy-remote.sh" + BUILD="$ROOT/scripts/build-rocmfpx.sh" TEST_TMP="$(mktemp -d)" - SRC="$TEST_TMP/src" - mkdir -p "$SRC" - git -C "$SRC" init -q - git -C "$SRC" -c user.email=t -c user.name=t commit -q --allow-empty -m x + touch "$TEST_TMP/empty.env" } teardown() { rm -rf "$TEST_TMP" } -# make_stub — an executable that prints its whole argv on one line, so -# "what command line would the supervisor run" is observable output. -make_stub() { - cat > "$TEST_TMP/bin" <<'STUB' -#!/bin/sh -printf '%s\n' "$*" +# podman_stub — a fake podman on PATH that prints the argv of `run` and +# `wait` and answers `images` with whatever digests the test names, so +# "what would the supervisor run" is observable output and the digest +# gate is testable without a 3.5 GB image. +podman_stub() { + mkdir -p "$TEST_TMP/bin" + cat > "$TEST_TMP/bin/podman" <<'STUB' +#!/usr/bin/env bash +case "$1" in + images) printf '%s\n' "${STUB_DIGESTS:-}" ;; + rm) exit 0 ;; + run|wait) echo "PODMAN $*"; exit "${STUB_WAIT_RC:-0}" ;; + *) exit 0 ;; +esac STUB - chmod +x "$TEST_TMP/bin" + chmod +x "$TEST_TMP/bin/podman" } -@test "the env file defines the deployment values" { - grep -qE '^LLAMA_BIN=/' "$ENV_FILE" - grep -qE '^MODEL_FILE=/' "$ENV_FILE" - grep -qE '^DRAFT_FILE=/' "$ENV_FILE" - grep -qE '^ALIAS=[a-z0-9-]+$' "$ENV_FILE" - grep -qE '^LISTEN_PORT=[0-9]+$' "$ENV_FILE" - grep -qE '^CTX_SIZE=[0-9]+$' "$ENV_FILE" +# good_env — the minimum env the serve script demands, with the digest +# stub agrees on and zero-byte artifacts passing the size gate. +good_env() { + : > "$TEST_TMP/empty.env" + touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/tok.gguf" + GOOD_ENV=( + "FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env" + "FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:cafe" + "FRANK_LARGE_CONTAINER=frank-large-halogen" + "FRANK_LARGE_LEGACY_CONTAINERS=frank-large" + "FRANK_LARGE_DEV_KFD=$TEST_TMP/qwen.hgn" + "FRANK_LARGE_DEV_DRI=$TEST_TMP" + "FRANK_LARGE_LOG=$TEST_TMP/serve.log" + "FRANK_LARGE_PUBLISH=7770:8731" + "FRANK_LARGE_MODEL_DIR=$TEST_TMP" + "FRANK_LARGE_CHECKPOINT=qwen.hgn" + "FRANK_LARGE_CHECKPOINT_BYTES=0" + "FRANK_LARGE_TOKENIZER=tok.gguf" + "HALOGEN_MODEL_ID=frank-large" + "HALOGEN_SLOT_CTX=200700" + "HALOGEN_KV_SLOTS=1" + "HALOGEN_DRAFTER=2" + "STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen@sha256:cafe" + ) } -@test "the serve script fails when the runtime is missing" { - # Only the model, the head and the projector exist; the fork was never - # built. llama.cpp is not reached, so this fails fast and says what to - # run instead of failing at bind time. - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/nope/llama-server" \ - "MODEL_FILE=$TEST_TMP/model.gguf" "DRAFT_FILE=$TEST_TMP/draft.gguf" \ - "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" +@test "the env file defines the halogen deployment values" { + grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen@sha256:[0-9a-f]{64}$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_CONTAINER=frank-large-halogen$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_PUBLISH=7770:8731$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_CHECKPOINT=.*\.hgn$' "$ENV_FILE" + grep -qE '^FRANK_LARGE_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE" + grep -qE '^HALOGEN_MODEL_ID=frank-large$' "$ENV_FILE" + grep -qE '^HALOGEN_SLOT_CTX=[0-9]+$' "$ENV_FILE" +} + +@test "the drafter stays on: slots must stay 1" { + # halogen makes batching and speculation mutually exclusive — with + # more than one slot the drafter is DISABLED and per-stream decode + # falls to serial speed (the sibling engine measured 41 -> 14 t/s + # when slots went up). frank-large was a one-session endpoint before + # this swap too (PARALLEL=1), so slots=1 preserves the contract; + # raising it is a product decision, never a default change. + grep -qE '^HALOGEN_KV_SLOTS=1$' "$ENV_FILE" + grep -qE '^HALOGEN_DRAFTER=2$' "$ENV_FILE" + # The cap/timeout pair is coupled (FLAGS.md table): a request at the + # cap must not outlast the queue timeout, or everyone queued behind + # it eats a 503. + grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE" + grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE" +} + +@test "the serve script refuses when there is no GPU to serve on" { + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_DEV_KFD="$TEST_TMP/absent-node" "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"build it with scripts/build-rocmfpx.sh"* ]] + [[ "$output" == *"no gfx1151 to serve on"* ]] } -@test "the serve script refuses to start without the model file" { - make_stub - touch "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/absent.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" - [ "$status" -ne 0 ] - [[ "$output" == *"is not readable"* ]] - [[ "$output" == *"download-frank-large-kingjones-uncensored.sh"* ]] +@test "the KV budget fits unified memory with margin" { + # Pool arithmetic is the engine's documented product: slots x + # slot_ctx x 64 KiB. Against it: 33.4 GiB of mapped weights, the + # engine working set + auto prompt cache (~25 GiB of headroom + # allowed), host baseline and frank-small co-tenancy. The box has + # 121 GiB unified; the 2026-09-16 halogen era ran the same engine + # class here on a LARGER pool with less co-tenancy slack. + ctx="$(sed -nE 's/^HALOGEN_SLOT_CTX=([0-9]+)$/\1/p' "$ENV_FILE")" + slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")" + pool_gib=$(( slots * ctx * 65536 / 1073741824 )) + total_gib=$(( 34 + pool_gib + 25 + 15 )) + [ "$total_gib" -le 121 ] } -@test "the serve script refuses to start without the MTP draft head" { - make_stub - touch "$TEST_TMP/model.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/absent.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" +@test "the serve script refuses when the image is not in storage" { + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" STUB_DIGESTS="" "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"MTP draft head"* ]] + [[ "$output" == *"not in podman storage"* ]] + [[ "$output" == *"podman pull"* ]] } -@test "the serve script refuses a missing mmproj unless vision is opted out" { - make_stub - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/absent.gguf" "$SERVE" +@test "the serve script refuses a missing checkpoint and names the fetch script" { + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_CHECKPOINT=absent.hgn "$SERVE" [ "$status" -ne 0 ] - [[ "$output" == *"set MMPROJ_FILE='' for text-only"* ]] - # The documented opt-out must actually start, without a --mmproj argument. - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=" "$SERVE" + [[ "$output" == *"checkpoint"*"missing"* ]] + [[ "$output" == *"download-frank-large-halogen.sh"* ]] +} + +@test "the serve script refuses a checkpoint of the wrong byte count" { + # A truncated curl exit-0 artifact must not boot; the model card's + # byte count is the contract (fleet fetch law). + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_CHECKPOINT_BYTES=42 "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"expected 42"* ]] +} + +@test "the serve script refuses a symlinked tokenizer" { + # A HuggingFace cache snapshot's entries are symlinks into blobs/ and + # dangle inside the container mount. + podman_stub + good_env + ln -sf /nowhere/x "$TEST_TMP/link.gguf" + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \ + FRANK_LARGE_TOKENIZER=link.gguf "$SERVE" + [ "$status" -ne 0 ] + [[ "$output" == *"missing or a symlink"* ]] +} + +@test "the render names the container, the port map and the engine shape" { + podman_stub + good_env + run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE" [ "$status" -eq 0 ] - [[ "$output" != *"--mmproj"* ]] + # `podman run` output is redirected to the log by design (the unit + # journal must not absorb engine chatter); the stub's printed argv + # lands there, so the render assertions read the same log. + output="$output +$(cat "$TEST_TMP/serve.log")" + [[ "$output" == *"--name frank-large-halogen"* ]] + [[ "$output" == *"--publish 7770:8731"* ]] + [[ "$output" == *"-e HALOGEN_MODEL_ID=frank-large"* ]] + [[ "$output" == *"-e HALOGEN_SLOT_CTX=200700"* ]] + [[ "$output" == *"-e HALOGEN_KV_SLOTS=1"* ]] + [[ "$output" == *"-e HALOGEN_DRAFTER=2"* ]] + [[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]] + [[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]] + [[ "$output" == *"ghcr.io/peonist-ai/halogen@sha256:cafe"* ]] + # The unit's restart policy is real because the script becomes + # `podman wait`: the container's exit code is the service's exit. + [[ "$output" == *"PODMAN wait frank-large-halogen"* ]] } -@test "the serve script fails when LLAMA_BIN is empty" { - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env LLAMA_BIN= "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" - [ "$status" -ne 0 ] - [[ "$output" == *"LLAMA_BIN is empty"* ]] +@test "the shipped env pins the fleet's window and shape" { + # 200700 is the per-conversation hard ceiling (a longer prompt is a + # 400 naming the limit, never a silent truncation), and the harness + # compaction math must fit prompt + budget under it: OMP compacts at + # 70% = 140490, and 140490 + 57344 = 197834 <= 200700. + grep -qE '^HALOGEN_SLOT_CTX=200700$' "$ENV_FILE" + ctx=200700 + [ $(( ctx * 70 / 100 + 57344 )) -le "$ctx" ] } -@test "the shipped env points at the ROCmFPX runtime and the AGENT payload" { - grep -qE "^LLAMA_BIN=$RUNTIME_STUB$" "$ENV_FILE" - grep -qE "^MODEL_FILE=$MODEL_STUB$" "$ENV_FILE" - grep -qE "^DRAFT_FILE=$DRAFT_STUB$" "$ENV_FILE" - grep -qE "^MMPROJ_FILE=$MMPROJ_STUB$" "$ENV_FILE" - # HIP reads bare HOST/TEMP/GPU_MAX_HEAP_SIZE out of the environment that - # `set -a` creates; a tunable named HOST leaks into the runtime. +@test "no HIP-sensitive bare names leak through the env file" { + # HIP reads bare env vars out of a `set -a` source; a tunable named + # HOST or TEMP used to send the JIT cache to path 1.0 and segfault + # every load (kept from the llama.cpp era — the container gets only + # the explicit -e list, but the host env still reaches podman itself). ! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE" } -@test "the argv contract keeps the fleet's port and alias" { - make_stub - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" \ - ALIAS=frank-large BIND_ADDR=127.0.0.1 LISTEN_PORT=7770 "$SERVE" - [ "$status" -eq 0 ] - # The caller's values must survive the env file, or a single-run operator - # override silently does nothing. - [[ "$output" == *"--host 127.0.0.1"* ]] - [[ "$output" == *"--port 7770"* ]] - [[ "$output" == *"--alias frank-large"* ]] - # Vision is wired: the projector is the whole vision tower. - [[ "$output" == *"--mmproj $TEST_TMP/mmproj.gguf"* ]] +@test "the deployer gates exactly the artifacts the serve script opens" { + # Serve and deploy must name the SAME image digest, checkpoint and + # tokenizer: the 2026-10-01 crash-loop was exactly these twins + # disagreeing mid-cutover. + img="$(sed -nE 's/^FRANK_LARGE_IMAGE=(.*)$/\1/p' "$ENV_FILE")" + grep -qF "LLM_LARGE_IMAGE=\"\${LLM_LARGE_IMAGE:-$img}\"" "$DEPLOYER" + dir="$(sed -nE 's/^FRANK_LARGE_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")" + ckpt="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")" + tok="$(sed -nE 's/^FRANK_LARGE_TOKENIZER=(.*)$/\1/p' "$ENV_FILE")" + grep -qF "LLM_LARGE_CKPT=\"\${LLM_LARGE_CKPT:-$dir/$ckpt}\"" "$DEPLOYER" + grep -qF "LLM_LARGE_TOKENIZER=\"\${LLM_LARGE_TOKENIZER:-$dir/$tok}\"" "$DEPLOYER" + bytes="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")" + grep -qF "LLM_LARGE_CKPT_BYTES=\"\${LLM_LARGE_CKPT_BYTES:-$bytes}\"" "$DEPLOYER" } -@test "the shipped argv is one full native window on one session" { - grep -qE '^CTX_SIZE=262144$' "$ENV_FILE" - grep -qE '^PARALLEL=1$' "$ENV_FILE" - # 262144 IS the model's trained window (qwen35.context_length in the GGUF), - # so 1 slot gets the whole thing — no truncation, no yarn scaling. - # - # Budget, from the GGUF header: 23.15 GiB of weights + 1.57 GiB draft head - # + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x - # 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB - # of DeltaNet state + a couple GiB of compute buffers, against the iGPU's - # ~114 GiB GTT aperture. Change CTX_SIZE and this arithmetic, not a boot - # log, is what tells you it no longer fits. - ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")" - kv_gib=$((ctx * 65536 / 1073741824)) - total_gib=$((24 + 2 + 1 + kv_gib + 3)) - [ "$total_gib" -le 114 ] +@test "the deployer points a missing artifact at the halogen fetch script" { + grep -qE 'download-frank-large-halogen\.sh' "$DEPLOYER" + grep -qE 'podman pull' "$DEPLOYER" } -@test "the MTP draft contract matches the model card's measured optimum" { - make_stub - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" - [ "$status" -eq 0 ] - # The head is the model's own nextn layer, loaded as a separate draft - # model because this checkpoint ships it standalone (no blk..nextn.* - # tensors in the target file). - [[ "$output" == *"--spec-type draft-mtp"* ]] - [[ "$output" == *"--model-draft $TEST_TMP/draft.gguf"* ]] - # n-max 4 is the measured optimum; n-max 8 measured WORSE (acceptance - # fell to 0.757 and the extra verification cost outran the savings), so - # the flag is a measurement, not a wish. - [[ "$output" == *"--spec-draft-n-max 4"* ]] - [[ "$output" == *"--spec-draft-n-min 0"* ]] - [[ "$output" == *"--spec-draft-p-min 0.0"* ]] - # The draft runs on the iGPU beside the target: no device-to-device hop - # on a link that measures single-digit GB/s. - [[ "$output" == *"--spec-draft-device ROCm0"* ]] - [[ "$output" == *"--spec-draft-ngl 99"* ]] -} - -@test "KV and placement stay unquantized with flash attention on" { - make_stub - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" - [ "$status" -eq 0 ] - [[ "$output" == *"--cache-type-k f16"* ]] - [[ "$output" == *"--cache-type-v f16"* ]] - # ROCmFPx quantized KV has no kernel on this branch, and the endpoint has - # the aperture to keep KV unquantized anyway. - [[ "$output" == *"--flash-attn on"* ]] - # --fit's placement heuristics do not know that 311 tensors of type 103 - # belong on the iGPU, so the whole stack is placed explicitly instead. - [[ "$output" == *"--fit off"* ]] - [[ "$output" == *"--n-gpu-layers 999"* ]] -} - -@test "the client contract wires thinking and template kwargs" { - make_stub - touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf" - run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \ - "DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE" - [ "$status" -eq 0 ] - # Without --jinja, llama.cpp silently ignores the harness's - # chat_template_kwargs, so thinking control disappears from the API. - [[ "$output" == *"--jinja"* ]] - # The harness reads `reasoning_content` for this endpoint, which is what - # the deepseek emission format produces (vLLM's `reasoning` is not it). - [[ "$output" == *"--reasoning-format deepseek"* ]] - # The template needs an unbounded thinking budget. - [[ "$output" == *"--reasoning-budget -1"* ]] -} - -@test "the deployer gates exactly the paths the serve script opens" { - # A gate that checks a different file than the engine opens is the - # 2026-10-01 crash-loop: deploy passed, the unit restart-looped. Read the - # engine's own paths out of the env file and require the deployer's - # defaults to name the same ones. - for pair in "LLAMA_BIN:LLM_LARGE_RUNTIME" \ - "MODEL_FILE:LLM_LARGE_MODEL" \ - "DRAFT_FILE:LLM_LARGE_DRAFT" \ - "MMPROJ_FILE:LLM_LARGE_MMPROJ"; do - env_key="${pair%%:*}" - dep_key="${pair##*:}" - want="$(sed -nE "s|^${env_key}=(.*)\$|\1|p" "$ENV_FILE")" - [ -n "$want" ] - got="$(grep -oE "${dep_key}:-[^}]+" "$DEPLOYER" | head -1 | sed "s|^${dep_key}:-||")" - [ "$got" = "$want" ] - done -} - -@test "the deployer points a missing runtime at the fork build script" { - grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER" - grep -qE 'fetch it: bash scripts/download-frank-large-kingjones-uncensored\.sh' "$DEPLOYER" -} - -@test "the stray-process sweep cannot take frank-small down with frank-large" { - # frank-small serves a different GGUF over llama-server on the SAME host. - # A sweep keyed on the binary name (`pkill -f llama-server`) would kill - # the 3090 endpoint too, so it is keyed on frank-large's own payload file. - grep -q "pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT'" "$DEPLOYER" +@test "the stray sweep cannot take frank-small or the deployer down" { + # frank-small also listens on this host (7771, llama-server) and the + # deployer's own command line contains "frank-large"; only the FULL + # container name matches a stray manual halogen run and nothing else. + grep -q "pkill -f 'frank-large-halogen'" "$DEPLOYER" ! grep -qE "pkill -f 'llama-server'" "$DEPLOYER" -} - -@test "the build script refuses a source tree that is not the pinned commit" { - # The pin is load-bearing: the older builds of this fork line segfault in - # ggml_compute_forward_set_rows on this exact payload. A runtime that - # differs from what was qualified must be heard about at deploy time. - wrong="$(git -C "$SRC" rev-parse HEAD)" - run env ROCMFPX_SRC_DIR="$SRC" ROCMFPX_COMMIT="ffff$wrong" \ - "$BUILD" --verify - [ "$status" -ne 0 ] - [[ "$output" == *"ROCMFPX_COMMIT"*mismatch* ]] -} - -@test "the build script pins a real commit and the gfx1151 target" { - grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD" - grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD" - grep -qE 'DGPU_TARGETS=' "$BUILD" - # The webui asset fetch half-failed on this box and aborted the link at - # 98%; the build must not depend on it. - grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD" + ! grep -qE "pkill -f 'frank-large'" "$DEPLOYER" } @test "the deployed unit keeps the fleet's restart and boot-persistence shape" { @@ -267,15 +244,25 @@ STUB grep -qE '^WantedBy=' "$UNIT" grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT" grep -qE '^User=dank$' "$UNIT" - # The payload is 26 GiB of mmap'd weights plus pinned staging buffers; an - # RLIMIT smaller than the pinned set fails the load in a way that looks - # like an engine crash. grep -qE '^LimitMEMLOCK=infinity$' "$UNIT" + # The service's lifetime is the container's; stopping the service + # must stop the container by its real name. + grep -qE '^ExecStop=.*podman stop .* frank-large-halogen$' "$UNIT" } -@test "the unit points at no retired frank-large runtime path" { - # The host keeps every retired payload and build on disk, so a stale path - # in a unit still resolves — to a runtime nobody has measured in weeks. +@test "the unit runs the halogen runtime and no retired path" { + grep -q 'halogen' "$UNIT" + # Every retired payload/build stays on the host, so a stale path in + # a unit still resolves — to a runtime nobody has measured in weeks. ! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT" - ! grep -q 'halogen' "$UNIT" + ! grep -qE 'kingjones-ROCmFPX' "$UNIT" +} + +@test "the rollback door stays qualified: the ROCmFPX build pin survives" { + # Rolling back this swap means rebuilding the fork the old payload + # needs; the pin (real commit, gfx1151, no webui fetch) is what makes + # that door trustworthy months later. + grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD" + grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD" + grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD" }