frank-large: cut over to halogen container (qwen3.8-27b .hgn + DFlash2)
Runtime swap ROCmFPX llama.cpp -> peonist-ai/halogen (image digest dc0a39a0, tag 0.1.4), serving qwen3.8-27b-p1w4d-d2.hgn (33.4 GiB, the .hgn format loads in no other engine) with HALOGEN_DRAFTER=2 (DFlash2) on ONE slot of 200700 ctx: slots>1 disables the drafter (engine law). Accepted: TEXT-ONLY (vision encoder ignored; frank-small owns images) and the censored base (rollback = revert this commit; payload, fork build and build-rocmfpx.sh pin stay on disk). - serve script: podman run under the root unit, exec podman wait so Restart=always is real; gates image digest, checkpoint byte-count, flat regular-file tokenizer, GPU nodes; engine stdout to log. - deploy handler: gates the artifacts, sweeps the legacy bare llama-server + old container name by NAME (never pkill llama-server: frank-small), retires the 2026-09 quadlet, port cutover 7770. - fetch script: -C - resume, -f, byte count, sha256, --verify-only. - seam rewritten (17 tests): halogen is now the deployed runtime.
This commit is contained in:
Frank
+57
-57
@@ -1086,32 +1086,31 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is the ROCmFPX fork of llama.cpp (kingjones30/ROCmFPX, pinned by
|
||||
# scripts/build-rocmfpx.sh), serving
|
||||
# kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF:Q6_0_ROCMFPX_AGENT
|
||||
# with the repo's own Q4_0 MTP draft head and the BF16 vision projector, on the
|
||||
# Strix Halo iGPU. It is a ROOT unit: the runtime is a native binary linked
|
||||
# against the host ROCm, so there is no container image to install and no
|
||||
# generated unit to dodge. relink_unit points
|
||||
# /etc/systemd/system/frank-large.service at the repo file and boot persistence
|
||||
# is that unit's [Install] section.
|
||||
# frank-large is the halogen CONTAINER (ghcr.io/peonist-ai/halogen,
|
||||
# digest-pinned through scripts/frank-large.env) serving
|
||||
# peonist-ai/halogen-qwen3.8-27b — qwen3.8-27b-p1w4d-d2.hgn, 33.4 GiB,
|
||||
# DFlash2 spec decode, 1 slot x 200700, text-only — on the Strix Halo
|
||||
# iGPU. It stays a ROOT unit because the serve script drives `podman run`
|
||||
# directly (the frank-primary pattern): relink_unit points
|
||||
# /etc/systemd/system/frank-large.service at the repo file, boot
|
||||
# persistence is that unit's [Install], and LimitMEMLOCK=infinity is
|
||||
# fleet parity. 2026-10-05 cutover from the ROCmFPX llama.cpp shape —
|
||||
# rollback = revert the swap commit's trio; that payload, its draft head
|
||||
# and the fork build all stay on disk
|
||||
# (~/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP,
|
||||
# ~/GitPackages/kingjones-ROCmFPX).
|
||||
#
|
||||
# The runtime gate matters as much as the payload gate: the fork is built from
|
||||
# source on this host, and the payload is ftype 114 (tensor types 102
|
||||
# GGML_TYPE_Q6_0_ROCMFPX and 103 GGML_TYPE_Q8_0_ROCMFPX) which only this fork
|
||||
# can read — so a machine whose /home/dank/GitPackages tree was never built has
|
||||
# a unit that can never become healthy, and a green deploy would leave the
|
||||
# endpoint dark.
|
||||
#
|
||||
# It replaces the ALIGNED kingjones Q8_0_ROCMFPX_AGENT file (rollback target:
|
||||
# its payload and its draft head are still on disk in
|
||||
# ~/models/kingjones-Qwen3.8-27B-ROCmFP4-STRIX-MTP, where this payload's
|
||||
# draft-head hardlink also lives). The port gate is the same as every cutover
|
||||
# since 2026-09-16: nothing starts until 7770 is released.
|
||||
LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}"
|
||||
LLM_LARGE_MMPROJ="${LLM_LARGE_MMPROJ:-/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}"
|
||||
# The gates mirror serve-frank-large.sh's predicates EXACTLY (the
|
||||
# 2026-10-01 crash-loop lesson): image digest in dank's podman storage,
|
||||
# the .hgn checkpoint at the exact byte count, the tokenizer a regular
|
||||
# (not symlinked) file — a green deploy against a missing artifact would
|
||||
# leave a restart-looping unit and a dark endpoint. The port gate is the
|
||||
# same as every cutover since 2026-09-16: nothing starts until 7770 is
|
||||
# released.
|
||||
LLM_LARGE_IMAGE="${LLM_LARGE_IMAGE:-ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea}"
|
||||
LLM_LARGE_CKPT="${LLM_LARGE_CKPT:-/home/dank/models/halogen-qwen3.8-27b/qwen3.8-27b-p1w4d-d2.hgn}"
|
||||
LLM_LARGE_CKPT_BYTES="${LLM_LARGE_CKPT_BYTES:-35880329102}"
|
||||
LLM_LARGE_TOKENIZER="${LLM_LARGE_TOKENIZER:-/home/dank/models/halogen-qwen3.8-27b/frank-large.tokenizer.gguf}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1144,43 +1143,44 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the deploy log to say why. The gates
|
||||
# mirror serve-frank-large.sh's own predicates exactly: runtime
|
||||
# executable, target GGUF, MTP head, mmproj.
|
||||
if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then
|
||||
echo "ERROR: ROCmFPX runtime absent at $LLM_LARGE_RUNTIME" >&2
|
||||
echo "frank-large: build it: bash scripts/build-rocmfpx.sh" >&2
|
||||
# Refuse rather than restart-loop; every gate below has a twin in
|
||||
# serve-frank-large.sh and they must agree (2026-10-01 lesson).
|
||||
# Capture podman output before grepping: `podman images | grep -q`
|
||||
# races under pipefail (SIGPIPE 141 becomes "no match", 2026-09-30).
|
||||
if ! grep -qF "${LLM_LARGE_IMAGE#*@}" <<<"$(podman images --format '{{.RepoDigests}}' 2>/dev/null)"; then
|
||||
echo "ERROR: halogen image absent: $LLM_LARGE_IMAGE" >&2
|
||||
echo "frank-large: pull it: podman pull ${LLM_LARGE_IMAGE%@*}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_LARGE_CKPT" ]]; then
|
||||
echo "ERROR: halogen checkpoint absent at $LLM_LARGE_CKPT" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "$(stat -c %s "$LLM_LARGE_CKPT")" != "$LLM_LARGE_CKPT_BYTES" ]]; then
|
||||
echo "ERROR: halogen checkpoint is not $LLM_LARGE_CKPT_BYTES bytes (partial download?)" >&2
|
||||
echo "frank-large: re-check: bash scripts/download-frank-large-halogen.sh --verify-only" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_LARGE_TOKENIZER" || -L "$LLM_LARGE_TOKENIZER" ]]; then
|
||||
echo "ERROR: tokenizer missing or a symlink at $LLM_LARGE_TOKENIZER" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT" "$LLM_LARGE_MMPROJ"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large-kingjones-uncensored.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
||||
# Then sweep a stray manual run -- that is how a bench harness leaves the
|
||||
# port busy. Match the TARGET GGUF name, which every frank-large command
|
||||
# line carries via --model and which no other process on this host uses:
|
||||
# frank-small also runs llama-server here (with the MiMo GGUF) and must
|
||||
# not be killed, so keying on the binary name would take the 3090 endpoint
|
||||
# down with it. The pattern never appears in the deploying shell's own
|
||||
# command line, so -f is safe here (it is not in `bash deploy-remote.sh`).
|
||||
# The predecessor on a re-deploy is THIS unit: stop it (ExecStop stops
|
||||
# the container) before waiting on the port. The stray sweep is keyed
|
||||
# on the FULL container name, which appears in the argv of a manual
|
||||
# `podman run --name frank-large-halogen` / `podman wait` but NOT in
|
||||
# this deployer's own command line (`bash deploy-remote.sh
|
||||
# frank-large`) — so it cannot shoot itself in the foot (the fleet
|
||||
# pkill law), and it can never match frank-small's llama-server.
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
# later and reported "Address already in use". Gate on the port, not on
|
||||
# hope.
|
||||
pkill -f 'frank-large-halogen' 2>/dev/null || true
|
||||
podman rm -f frank-large frank-large-halogen >/dev/null 2>&1 || true
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16: "Deactivated successfully" was not "port free").
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
run_privileged systemctl daemon-reload
|
||||
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
# download-frank-large-halogen.sh — fetch frank-large's halogen payload:
|
||||
# the qwen3.8-27b .hgn checkpoint and the export tokenizer, from the
|
||||
# PUBLIC peonist-ai/halogen-qwen3.8-27b repo (no token — and per the fleet
|
||||
# secret law, none would ever touch argv).
|
||||
#
|
||||
# The fetch law, same as every sibling script: curl -f (an access error
|
||||
# must not become a "successful" 153-byte artifact), exact byte count from
|
||||
# the model card, LFS sha256 verification, --verify-only to re-check an
|
||||
# existing payload without downloading.
|
||||
set -euo pipefail
|
||||
|
||||
DEST="${FRANK_LARGE_MODEL_DIR:-$HOME/models/halogen-qwen3.8-27b}"
|
||||
BASE="https://huggingface.co/peonist-ai/halogen-qwen3.8-27b/resolve/main"
|
||||
|
||||
FILES=(qwen3.8-27b-p1w4d-d2.hgn frank-large.tokenizer.gguf)
|
||||
# Byte sizes and sha256 (LFS oid) as published by the Hub on 2026-10-05 —
|
||||
# re-check with: curl -s "https://huggingface.co/api/models/peonist-ai/halogen-qwen3.8-27b/tree/main?recursive=true"
|
||||
SIZES=(35880329102 1293343)
|
||||
SHAS=(40874e7a5c27efcf1c83c44374db85726fa4086355b3c8aae1b363644d7e6234
|
||||
4c1ab3a19932f7d16eefc2c7fa60715c91133fe99c6dce683a3a7f11d463e7e9)
|
||||
|
||||
VERIFY_ONLY=0
|
||||
[ "${1:-}" = "--verify-only" ] && VERIFY_ONLY=1
|
||||
|
||||
mkdir -p "$DEST"
|
||||
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; want="${SIZES[$i]}"; sha="${SHAS[$i]}"
|
||||
path="$DEST/$f"
|
||||
if [ "$VERIFY_ONLY" -eq 0 ]; then
|
||||
echo "FETCH $f -> $path"
|
||||
# -C - resumes a partial file; -f keeps HTTP errors fatal.
|
||||
curl -fL --retry 5 -C - -o "$path" "$BASE/$f"
|
||||
fi
|
||||
[ -f "$path" ] || { echo "MISSING $path (run without --verify-only)" >&2; exit 1; }
|
||||
got="$(stat -c %s "$path")"
|
||||
[ "$got" = "$want" ] || { echo "SIZE $path: $got != $want" >&2; exit 1; }
|
||||
echo "OK $f ($got bytes)"
|
||||
done
|
||||
|
||||
echo "byte counts match; verifying sha256 (one pass over 33.4 GiB)..."
|
||||
for i in "${!FILES[@]}"; do
|
||||
f="${FILES[$i]}"; sha="${SHAS[$i]}"
|
||||
got="$(sha256sum "$DEST/$f" | cut -d' ' -f1)"
|
||||
[ "$got" = "$sha" ] || { echo "SHA256 $f: $got != $sha" >&2; exit 1; }
|
||||
echo "OK $f sha256"
|
||||
done
|
||||
echo "DOWNLOAD_DONE $DEST"
|
||||
+96
-96
@@ -1,103 +1,103 @@
|
||||
# frank-large — ROCmFPX llama.cpp serving
|
||||
# kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF
|
||||
# (Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT, ftype 114) + its Q4_0 MTP draft
|
||||
# head, on the Strix Halo iGPU (gfx1151), :7770.
|
||||
# frank-large.env — tunables for the frank-large endpoint on Dankclaw's
|
||||
# Strix Halo iGPU (gfx1151), host port 7770.
|
||||
#
|
||||
# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT (ftype 115)
|
||||
# to this repo's Q6 AGENT rung — same card, same recipe, abliterated weights.
|
||||
# The card is blunt about what that buys: refusal counts go from 23/24 refuse
|
||||
# to 23/24 comply with the quality score byte-identical (6/8 both arms), and
|
||||
# "abliteration removes content-refusal. It does not add capability." It costs
|
||||
# one quant rung: 8.39 bpw → ~6.6 bpw, 26.28 GiB → 23.15 GiB.
|
||||
# RUNTIME (2026-10-05, Frank): halogen — the closed-source C++/HIP engine
|
||||
# (github.com/peonist-ai/halogen-server, image ghcr.io/peonist-ai/halogen,
|
||||
# digest-pinned below) serving peonist-ai/halogen-qwen3.8-27b, a ~6.32 bpw
|
||||
# NVFP4-derivative quant of the BASE Qwen/Qwen3.8-27B, as a rootless
|
||||
# container driven by scripts/serve-frank-large.sh. This CUTS OVER the
|
||||
# 2026-10-02..10-03 ROCmFPX llama.cpp shape (uncensored
|
||||
# Q6_0_ROCMFPX_AGENT + MTP, 27.6-28.1 t/s): rollback = revert the swap
|
||||
# commit's trio; the ROCmFPX payload, draft head and build all stay on
|
||||
# disk at /home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/.
|
||||
#
|
||||
# What the file actually contains (read out of its own GGUF header, not from
|
||||
# the card): arch qwen35, 64 layers, full_attention_interval 4, 24 heads /
|
||||
# 4 KV heads / 256 head_dim, context_length 262144, and 851 tensors — 353 F32
|
||||
# + 339 GGML_TYPE_Q6_0_ROCMFPX (102) + 159 GGML_TYPE_Q8_0_ROCMFPX (103). The
|
||||
# AGENT routing is what puts the embeddings, the output head and every fourth
|
||||
# attention projection at the Q8 rung. Two custom tensor types, both fork-only
|
||||
# → the runtime is chosen by the file, exactly as it was for ftype 115.
|
||||
# WHY THE RUNTIME MOVED WITH THE MODEL: the payload is .hgn — per its own
|
||||
# card it "will not load in transformers, vLLM, or llama.cpp". Accepted
|
||||
# consequences (Frank, 2026-10-05): TEXT-ONLY (halogen ignores the model's
|
||||
# vision encoder; the fleet's vision path is frank-small) and the CENSORED
|
||||
# base (this is a finetune of Qwen/Qwen3.8-27B, not the abliterated file —
|
||||
# refusals may return). What it buys: vendor-measured decode DFlash2 31.7
|
||||
# t/s mean (vs this endpoint's MTP 27.6-28.1 and the old halogen-era MTP
|
||||
# 26.9), prefill ~566 t/s at 32k (was ~400 on the sibling engine), 1.4-2.3x
|
||||
# wall-clock on prefill-heavy agent traffic, at HIGHER effective precision
|
||||
# (~6.32 bpw vs ~6.6 with the old W4A4 promotion — see FLAGS.md).
|
||||
#
|
||||
# The shape is unchanged: one session at the model's full native window, MTP
|
||||
# drafting at n-max 4, unquantized KV.
|
||||
# SHAPE — the two knobs that are NOT free:
|
||||
# HALOGEN_KV_SLOTS=1 KEEPS SPECULATION ON. halogen makes batching and
|
||||
# speculation mutually exclusive: with >1 slot the drafter is disabled
|
||||
# and per-stream decode falls to serial speed (the sibling engine
|
||||
# measured 41 t/s -> 14 t/s when slots were raised, and halogen's own
|
||||
# FLAGS.md prints the same law). frank-large was already a one-session
|
||||
# endpoint under ROCmFPX (PARALLEL=1), so slots=1 is the same contract,
|
||||
# not a regression.
|
||||
# HALOGEN_SLOT_CTX=200700 (Frank's window) is the per-conversation hard
|
||||
# ceiling — a prompt over it is a 400 naming the limit, never a silent
|
||||
# truncation. KV pool = slots x slot_ctx x 64 KiB = 12.2 GiB. Budget
|
||||
# against the 121 GiB of unified memory: weights 33.4 GiB (mmap'd — page
|
||||
# cache, not anon), KV 12.2, engine working set + auto prompt cache
|
||||
# ~20-30, host baseline ~10 -> ~75-85 GiB in use. The 2026-09-16 halogen
|
||||
# era ran the same engine class on this box with LESS headroom.
|
||||
# HALOGEN_DRAFTER=2 (DFlash2) is the shipped default and the fastest
|
||||
# exact drafter on this checkpoint; the .hgn BAKES BOTH drafters (MTP +
|
||||
# DFlash2), so unlike the Flash-Next era there is no sidecar to mount
|
||||
# and no --dflash flag — the drafter is an int knob, byte-identical
|
||||
# output to greedy verification either way.
|
||||
#
|
||||
# Budget (from the GGUF header, not from a boot log): attention KV lives only
|
||||
# in the 16 full-attention layers (4 KV heads x 256 head_dim), 64 KiB/token
|
||||
# with f16 K and V => 16.0 GiB at 262,144. The 48 Gated DeltaNet layers hold
|
||||
# ~144 MiB of fixed state per sequence. Weights 23.15 + draft head 1.57 +
|
||||
# mmproj 0.87 + KV 16.0 + state 0.14 + compute buffers ≈ 42-44 GiB against the
|
||||
# iGPU's ~114 GiB GTT aperture, so f16 KV (the unquantized choice) is affordable
|
||||
# here in a way it is not on frank-small's 24 GiB card.
|
||||
LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server
|
||||
# One directory per HF repo, created by
|
||||
# scripts/download-frank-large-kingjones-uncensored.sh.
|
||||
MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf
|
||||
# The draft head: Q4_0, NOT the Q8_0 head. The head's per-step cost is its
|
||||
# projection over a 248K vocabulary, so the cheaper head is faster end to end
|
||||
# at equal-or-better acceptance (card: +6.0% at n-max 1, +10.2% at n-max 2).
|
||||
# The uncensored repo redistributes this file byte for byte (same Hub LFS oid);
|
||||
# the download script hardlinks the verified copy instead of re-fetching 1.6 GiB.
|
||||
DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf
|
||||
# Vision tower, separate from the LLM file. This repo ships a Q8_0 projector;
|
||||
# the fleet keeps the BF16 one, which is the file verified 4/4 on spatial
|
||||
# ground truth on this host (hardlinked here, same bytes). Empty = text-only.
|
||||
MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf
|
||||
# CLIENT CONTRACT deltas vs the ROCmFPX shape (the harness side lives in
|
||||
# ~/.omp/agent/models.yml):
|
||||
# * NO server-side sampler defaults: a request without temperature
|
||||
# decodes GREEDY. The harness sends temp 1.0 / top_p 0.95 via
|
||||
# extraBody, so that wire recipe is load-bearing now, not cosmetic.
|
||||
# * max_tokens defaults to 8192 PER REQUEST and INCLUDES thinking; a
|
||||
# reasoning reply over a small budget returns empty content with
|
||||
# finish_reason "length". The harness maxTokens is 57344: compaction
|
||||
# fires at 70% of 200700 = 140,490 and 140,490 + 57,344 = 197,834 <=
|
||||
# 200,700 keeps prompt + budget inside the pool (the 262144-era 65536
|
||||
# would overflow). Requests over HALOGEN_MAX_TOKENS_CAP are a 400
|
||||
# naming the cap; cap and queue timeout are COUPLED (see FLAGS.md
|
||||
# table) — do not move one without the other.
|
||||
# * response_format json_object/json_schema is a 400 (no constrained
|
||||
# decoding in this build).
|
||||
# * Thinking lands in reasoning_content — same field as the llama.cpp
|
||||
# era, so the harness compat block does NOT change. reasoning_effort
|
||||
# accepts none|low|xhigh on this build (template default xhigh); the
|
||||
# models.yml fold (high->xhigh) still applies.
|
||||
# * The engine's token protocol has no auth and binds loopback INSIDE
|
||||
# the container; only the frontend port is published. Keep it that way.
|
||||
|
||||
# Fleet endpoint: same port and alias as every frank-large before it, so the
|
||||
# harness models.yml id and the homepage probe do not move.
|
||||
ALIAS=frank-large
|
||||
# HIP reads a bare HOST, so this is BIND_ADDR — the same reason the file
|
||||
# avoids a bare TEMP.
|
||||
BIND_ADDR=0.0.0.0
|
||||
LISTEN_PORT=7770
|
||||
# Image: digest-pinned (fetch law: tag pulls silently re-point; the digest
|
||||
# is what was qualified). 0.1.4 = the version the model card names.
|
||||
FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:dc0a39a0016d6cfc58a197978febaafdf8d28403f724ded6111d98b5fb7ac0ea
|
||||
# Container identity: the halogen container name.
|
||||
FRANK_LARGE_CONTAINER=frank-large-halogen
|
||||
# Old quadlet-era container identity; the serve script clears stale names.
|
||||
FRANK_LARGE_LEGACY_CONTAINERS=frank-large
|
||||
# host:container — the frontend listens on 8731 inside (HALOGEN_API_PORT
|
||||
# default); the engine port (8730) is never published.
|
||||
FRANK_LARGE_PUBLISH=7770:8731
|
||||
|
||||
# One full native window. llama.cpp divides --ctx-size across --parallel
|
||||
# slots, so 262144/1 IS the window the file itself declares
|
||||
# (qwen35.context_length = 262144); the card only validated 65,536 and the
|
||||
# fleet has been running this family at 262,144 since the gufo era.
|
||||
CTX_SIZE=262144
|
||||
PARALLEL=1
|
||||
FRANK_LARGE_MODEL_DIR=/home/dank/models/halogen-qwen3.8-27b
|
||||
FRANK_LARGE_CHECKPOINT=qwen3.8-27b-p1w4d-d2.hgn
|
||||
FRANK_LARGE_CHECKPOINT_BYTES=35880329102
|
||||
FRANK_LARGE_CHECKPOINT_SHA256=40874e7a5c27efcf1c83c44374db85726fa4086355b3c8aae1b363644d7e6234
|
||||
FRANK_LARGE_TOKENIZER=frank-large.tokenizer.gguf
|
||||
FRANK_LARGE_TOKENIZER_BYTES=1293343
|
||||
FRANK_LARGE_TOKENIZER_SHA256=4c1ab3a19932f7d16eefc2c7fa60715c91133fe99c6dce683a3a7f11d463e7e9
|
||||
|
||||
# Deck knobs. --ubatch-size has a measured optimum on this stack (512 peak on
|
||||
# the sibling qwen35 model; larger ubatch costs GiBs of compute buffer for no
|
||||
# speed), and --fit must stay off: --fit's auto placement does not know that
|
||||
# 498 tensors of types 102 and 103 belong on the iGPU.
|
||||
BATCH_SIZE=2048
|
||||
UBATCH_SIZE=512
|
||||
NGL=999
|
||||
FIT=off
|
||||
FLASH_ATTN=on
|
||||
# Unquantized KV — the quality-first choice, affordable at one slot.
|
||||
CACHE_TYPE_K=f16
|
||||
CACHE_TYPE_V=f16
|
||||
|
||||
# Speculative decode: the model ships a standalone MTP head (mtp-Q4_0.gguf,
|
||||
# arch qwen35, nextn_predict_layers=1, a 65-block draft). n-max 4 is the knee
|
||||
# on BOTH cards: aligned 30.13 / 30.30 / 27.52 at n=3/4/5 (acceptance 0.926 at
|
||||
# the knee, and n-max 8 measured WORSE at 0.757), uncensored Q6 24.96 / 27.66
|
||||
# / 25.27. n-min 0 and p-min 0.0 are the card's own recipe. The draft runs
|
||||
# entirely on the iGPU, so cross-device traffic stays off the
|
||||
# bandwidth-limited link.
|
||||
SPEC_TYPE=draft-mtp
|
||||
SPEC_DRAFT_NGL=99
|
||||
SPEC_DRAFT_DEVICE=ROCm0
|
||||
SPEC_DRAFT_N_MAX=4
|
||||
SPEC_DRAFT_N_MIN=0
|
||||
SPEC_DRAFT_P_MIN=0.0
|
||||
|
||||
# The card's thinking sampling profile — the same triple the file embeds as
|
||||
# its own defaults (general.sampling.temp/top_p/top_k = 1.0 / 0.95 / 20) and
|
||||
# what the harness sends in extraBody, so the endpoint behaves the same to a
|
||||
# client that omits it. Card traps that do NOT bite here, but do not "fix"
|
||||
# them by changing this: reasoning_effort "none" THROWS (use
|
||||
# chat_template_kwargs enable_thinking:false), and a small max_tokens with
|
||||
# thinking on returns empty content — the serve script runs
|
||||
# --reasoning-budget -1, so the budget is never the limiter.
|
||||
TEMPERATURE=1.0
|
||||
TOP_P=0.95
|
||||
TOP_K=20
|
||||
MIN_P=0.0
|
||||
PRESENCE_PENALTY=0.0
|
||||
|
||||
NUM_THREADS=16
|
||||
THREAD_BATCH=16
|
||||
# --- engine flags (docs/FLAGS.md of the engine repo; anything unset stays
|
||||
# at the image default, which is what was qualified) ------------------------
|
||||
HALOGEN_MODEL_ID=frank-large
|
||||
HALOGEN_SLOT_CTX=200700
|
||||
HALOGEN_KV_SLOTS=1
|
||||
HALOGEN_DRAFTER=2
|
||||
# Coupled pair (FLAGS.md table): the cap bounds how long ONE request holds
|
||||
# the GPU, the timeout bounds how long the NEXT client waits. 65536 at
|
||||
# ~10 t/s serial worst case needs > 6550 s queue timeout; the defaults
|
||||
# honor that. Do not lower the timeout alone.
|
||||
HALOGEN_MAX_TOKENS_CAP=65536
|
||||
HALOGEN_QUEUE_TIMEOUT=7200
|
||||
# Engine prompt cache: EMPTY = auto (sized from available memory at
|
||||
# startup). Pin an explicit MB only to argue with the auto figure — a
|
||||
# full-context entry is ~18 GB at 262K, so a too-small pin yields a cache
|
||||
# that reports enabled and never hits (the engine warns at startup).
|
||||
#HALOGEN_CACHE_MB=
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[Unit]
|
||||
Description=ROCmFPX llama.cpp (gfx1151) kingjones777 Qwen3.8-27B Q8_0_ROCMFPX_AGENT, MTP spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
|
||||
Description=halogen container (peonist-ai/halogen-qwen3.8-27b, DFlash2 spec decode, 1 slot x 200700, text-only), root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
@@ -13,10 +13,19 @@ Group=dank
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
# always, not on-failure: an engine that drains and exits 0 after a worker
|
||||
# fault would otherwise leave the endpoint down until someone notices.
|
||||
# systemd stop alone cannot stop the container child; stop it explicitly
|
||||
# (frank-primary pattern). The engine drains on SIGTERM.
|
||||
ExecStop=/usr/bin/podman stop -t 60 frank-large-halogen
|
||||
TimeoutStartSec=1200
|
||||
TimeoutStopSec=120
|
||||
# always, not on-failure: an engine that exits 0 after an internal fault
|
||||
# would otherwise leave the endpoint down until someone notices.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# The engine mmaps 33.4 GiB of weights and keeps a multi-GiB working set
|
||||
# on a box that also hosts frank-small, ComfyUI and the Caddy/observability
|
||||
# stack; make it a late OOM candidate, not the first.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
+120
-173
@@ -1,186 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve kingjones777/Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP-GGUF as
|
||||
# frank-large on Dankclaw's Strix Halo iGPU (gfx1151) with the ROCmFPX fork of
|
||||
# llama.cpp.
|
||||
# serve-frank-large.sh — frank-large endpoint: halogen (closed-source
|
||||
# C++/HIP engine, image ghcr.io/peonist-ai/halogen, digest-pinned in
|
||||
# frank-large.env) serving peonist-ai/halogen-qwen3.8-27b on Dankclaw's
|
||||
# Strix Halo iGPU, host :7770. 2026-10-05 cutover from the ROCmFPX
|
||||
# llama.cpp shape — revert the swap commit's trio to roll back (payload
|
||||
# and fork build stay on disk).
|
||||
#
|
||||
# 2026-10-03 (Frank): swapped from the ALIGNED Q8_0_ROCMFPX_AGENT to this
|
||||
# repo's Q6_0_ROCMFPX_AGENT — abliterated weights, one quant rung cheaper,
|
||||
# everything else (window, KV, MTP depth, sampler) unchanged.
|
||||
# The engine is driven by `podman run` DIRECTLY (the fleet's frank-primary
|
||||
# pattern): the image entrypoint `halogen-docker` in default "all" mode
|
||||
# starts engine + OpenAI frontend in one container. The frontend listens on
|
||||
# 8731 inside (HALOGEN_API_PORT default) and is published as host 7770; the
|
||||
# engine's own token port (8730, no auth) stays unpublished on the
|
||||
# container's loopback. The image ships NO default checkpoint:
|
||||
# HALOGEN_CHECKPOINT selects the file inside the /models mount.
|
||||
#
|
||||
# Why this runtime and not another: the payload is ftype 114 — 339 tensors of
|
||||
# GGML_TYPE_Q6_0_ROCMFPX (102) and 159 of GGML_TYPE_Q8_0_ROCMFPX (103), tensor
|
||||
# types that exist only in the ROCmFPX fork. Stock llama.cpp, SGLang and gufo
|
||||
# all refuse the file, and gufo additionally gates every "Qwen3.8" artifact on
|
||||
# a template-hash allowlist. The qualified build is scripts/build-rocmfpx.sh's
|
||||
# pin: it defines both types (ggml.h 102/103), it carries the device paths for
|
||||
# type 102 (convert/cpy/getrows/mmq/fattn in ggml-cuda), it survives warmup —
|
||||
# the 2026-08-25 builds of the older fork line segfaulted in
|
||||
# ggml_compute_forward_set_rows on this family — and it drafts with the
|
||||
# model's own MTP head.
|
||||
# Gates mirror the deployer EXACTLY (the 2026-10-01 crash-loop lesson —
|
||||
# serve gate and deploy gate disagreeing = green deploy, red unit):
|
||||
# image digest present; checkpoint present at the exact byte count;
|
||||
# tokenizer present, readable, and a FLAT REGULAR FILE — the engine repo
|
||||
# warns that a HF cache snapshot's symlinks into blobs/ dangle inside a
|
||||
# container; GPU device nodes present.
|
||||
#
|
||||
# Why this file: Q6_0_ROCMFPX_AGENT (23.15 GiB, ftype 114) is the card's
|
||||
# protected-head rung — "Take Q6 AGENT (114) if you want protected heads": the
|
||||
# AGENT routing keeps the embeddings, the output head and every fourth
|
||||
# attention projection at the Q8 rung (159 tensors), which is the variant the
|
||||
# card names for MTP serving, and the refusal study was scored on this exact
|
||||
# file (23/24 comply where the aligned Q8 AGENT refuses 23/24, quality score
|
||||
# identical at 6/8). Drafting uses the repo's Q4_0 MTP head: the head's cost is
|
||||
# its projection over the 248K vocabulary, so the 4-bit head is faster end to
|
||||
# end at equal-or-better acceptance than its Q8_0 sibling.
|
||||
#
|
||||
# The client contract lives here, not in frank-large.env:
|
||||
# * --jinja — without it llama.cpp silently IGNORES the harness's
|
||||
# chat_template_kwargs (enable_thinking / preserve_thinking), so the
|
||||
# endpoint would look like it had no thinking control at all. The GGUF
|
||||
# carries the official Qwen template, so no --chat-template-file.
|
||||
# * --reasoning-format deepseek — thinking is emitted as reasoning_content,
|
||||
# the field the harness reads for this endpoint.
|
||||
# * the template accepts reasoning_effort xhigh|medium|low only, which is
|
||||
# why models.yml folds the harness's levels onto that set.
|
||||
# * --alias frank-large — that is the model id the harness resolves.
|
||||
# * --ctx-checkpoints / --cache-ram — prompt-cache geometry for the 16
|
||||
# sliding-window-style layers; harmless where the hybrid state forces a
|
||||
# full re-process, and required for the checkpoint feature to work at all.
|
||||
# Why containers instead of the bare binary (the old frank-large shape):
|
||||
# the payload is .hgn and only halogen reads it; halogen ships as a
|
||||
# container; the root unit keeps LimitMEMLOCK=infinity for parity and to
|
||||
# outlive the login session, exactly like frank-primary's.
|
||||
set -euo pipefail
|
||||
|
||||
# Fleet values live in frank-large.env. Anything already in the environment
|
||||
# wins over the file, so an operator can override one value on a single run:
|
||||
# the caller's values are captured, the file is sourced, then they are put
|
||||
# back. (A plain `source` would let the file clobber the environment.)
|
||||
TUNABLES=(LLAMA_BIN MODEL_FILE DRAFT_FILE MMPROJ_FILE ALIAS BIND_ADDR
|
||||
LISTEN_PORT CTX_SIZE PARALLEL BATCH_SIZE UBATCH_SIZE NGL FIT
|
||||
FLASH_ATTN CACHE_TYPE_K CACHE_TYPE_V SPEC_TYPE SPEC_DRAFT_NGL
|
||||
SPEC_DRAFT_DEVICE SPEC_DRAFT_N_MAX SPEC_DRAFT_N_MIN SPEC_DRAFT_P_MIN
|
||||
TEMPERATURE TOP_P TOP_K MIN_P PRESENCE_PENALTY NUM_THREADS
|
||||
THREAD_BATCH ROCM_PATH HIP_VISIBLE HSA_GFX_VERSION)
|
||||
_OVERRIDES=()
|
||||
for _var in "${TUNABLES[@]}"; do
|
||||
[[ -n "${!_var+set}" ]] && _OVERRIDES+=("$_var=${!_var}")
|
||||
done
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$(dirname "$0")/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [ ! -f "$ENV_FILE" ]; then
|
||||
echo "frank-large: env file not found: $ENV_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
for _kv in "${_OVERRIDES[@]+${_OVERRIDES[@]}}"; do
|
||||
export "${_kv?}"
|
||||
done
|
||||
unset _var _kv
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
|
||||
: "${LLAMA_BIN=/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server}"
|
||||
: "${MODEL_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf}"
|
||||
: "${DRAFT_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf}"
|
||||
: "${MMPROJ_FILE=/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf}"
|
||||
: "${ALIAS=frank-large}"
|
||||
: "${BIND_ADDR=0.0.0.0}"
|
||||
: "${LISTEN_PORT=7770}"
|
||||
: "${CTX_SIZE=262144}"
|
||||
: "${PARALLEL=1}"
|
||||
: "${BATCH_SIZE=2048}"
|
||||
: "${UBATCH_SIZE=512}"
|
||||
: "${NGL=999}"
|
||||
: "${FIT=off}"
|
||||
: "${FLASH_ATTN=on}"
|
||||
: "${CACHE_TYPE_K=f16}"
|
||||
: "${CACHE_TYPE_V=f16}"
|
||||
: "${SPEC_TYPE=draft-mtp}"
|
||||
: "${SPEC_DRAFT_NGL=99}"
|
||||
: "${SPEC_DRAFT_DEVICE=ROCm0}"
|
||||
: "${SPEC_DRAFT_N_MAX=4}"
|
||||
: "${SPEC_DRAFT_N_MIN=0}"
|
||||
: "${SPEC_DRAFT_P_MIN=0.0}"
|
||||
: "${TEMPERATURE=1.0}"
|
||||
: "${TOP_P=0.95}"
|
||||
: "${TOP_K=20}"
|
||||
: "${MIN_P=0.0}"
|
||||
: "${PRESENCE_PENALTY=0.0}"
|
||||
: "${NUM_THREADS=16}"
|
||||
: "${THREAD_BATCH=16}"
|
||||
: "${ROCM_PATH=/opt/rocm}"
|
||||
: "${HIP_VISIBLE=0}"
|
||||
: "${HSA_GFX_VERSION=11.5.1}"
|
||||
: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE must name the digest-pinned image}"
|
||||
: "${FRANK_LARGE_CONTAINER:?FRANK_LARGE_CONTAINER must name the container}"
|
||||
: "${FRANK_LARGE_PUBLISH:?FRANK_LARGE_PUBLISH must be host:container}"
|
||||
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR must name the payload dir}"
|
||||
: "${FRANK_LARGE_CHECKPOINT:?FRANK_LARGE_CHECKPOINT must name the .hgn inside it}"
|
||||
: "${FRANK_LARGE_CHECKPOINT_BYTES:?FRANK_LARGE_CHECKPOINT_BYTES must pin the size}"
|
||||
: "${FRANK_LARGE_TOKENIZER:?FRANK_LARGE_TOKENIZER must name the tokenizer file}"
|
||||
: "${HALOGEN_MODEL_ID:?HALOGEN_MODEL_ID must be the served alias (frank-large)}"
|
||||
: "${HALOGEN_SLOT_CTX:?HALOGEN_SLOT_CTX must be set}"
|
||||
: "${HALOGEN_KV_SLOTS:?HALOGEN_KV_SLOTS must be set (1 keeps the drafter on)}"
|
||||
: "${HALOGEN_DRAFTER:?HALOGEN_DRAFTER must be set (2 = DFlash2)}"
|
||||
|
||||
[[ -n "$LLAMA_BIN" ]] || { echo "frank-large: LLAMA_BIN is empty" >&2; exit 1; }
|
||||
[[ -x "$LLAMA_BIN" ]] || {
|
||||
echo "frank-large: llama-server '$LLAMA_BIN' is not executable;" >&2
|
||||
echo " build it with scripts/build-rocmfpx.sh" >&2
|
||||
# Capture podman's output BEFORE grepping: under `set -o pipefail` the
|
||||
# `podman images | grep -q` pattern races (grep exits on first match,
|
||||
# podman dies SIGPIPE 141, pipefail turns the 141 into "no match" — the
|
||||
# 2026-09-30 deploy refusal).
|
||||
DIGESTS="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$DIGESTS"; then
|
||||
echo "frank-large: image $FRANK_LARGE_IMAGE not in podman storage;" >&2
|
||||
echo " pull it: podman pull ${FRANK_LARGE_IMAGE%@*}" >&2
|
||||
exit 1
|
||||
}
|
||||
# Refuse anything but the real payload. A path that is not a readable file
|
||||
# makes llama-server hang rather than fail clean.
|
||||
[[ -r "$MODEL_FILE" ]] || {
|
||||
echo "frank-large: model file '$MODEL_FILE' is not readable;" >&2
|
||||
echo " fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2
|
||||
exit 1
|
||||
}
|
||||
# The MTP head is the performance feature: without it these weights decode at
|
||||
# memory-bandwidth speed (measured on the aligned sibling: 17.0 t/s with the
|
||||
# head against 7.6 without) and the endpoint loses the ~2.2x the head buys.
|
||||
[[ -r "$DRAFT_FILE" ]] || {
|
||||
echo "frank-large: MTP draft head '$DRAFT_FILE' is not readable;" >&2
|
||||
echo " fetch it with scripts/download-frank-large-kingjones-uncensored.sh" >&2
|
||||
exit 1
|
||||
}
|
||||
# The GGUF carries no vision tower; the projector file is the whole tower, so
|
||||
# a missing file means a vision request would 500 rather than degrade.
|
||||
# MMPROJ_FILE="" is the documented opt-out for a text-only endpoint.
|
||||
MM_ARGS=()
|
||||
if [[ -n "${MMPROJ_FILE:-}" ]]; then
|
||||
[[ -r "$MMPROJ_FILE" ]] || {
|
||||
echo "frank-large: mmproj '$MMPROJ_FILE' is not readable; set MMPROJ_FILE='' for text-only" >&2
|
||||
exit 1
|
||||
}
|
||||
MM_ARGS=(--mmproj "$MMPROJ_FILE")
|
||||
fi
|
||||
|
||||
# HIP/ROCm host environment. HIP reads bare env vars (TEMP, HOST,
|
||||
# GPU_MAX_HEAP_SIZE), which is why the tunables above are named TEMPERATURE
|
||||
# and BIND_ADDR. HSA_OVERRIDE_GFX_VERSION is what lets a runtime built for a
|
||||
# neighbouring gfx target run on this iGPU; the qualified pin targets gfx1151
|
||||
# natively and the override is a no-op for it.
|
||||
export ROCM_PATH
|
||||
export HIP_VISIBLE_DEVICES="$HIP_VISIBLE"
|
||||
export HSA_OVERRIDE_GFX_VERSION="$HSA_GFX_VERSION"
|
||||
export LD_LIBRARY_PATH="$(dirname "$LLAMA_BIN"):${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
|
||||
# The checkpoint gate is size-exact: a truncated curl exits 0 on some
|
||||
# failures and a partial .hgn loads far enough to look alive, so the byte
|
||||
# count from the model card is the contract (the fleet fetch law).
|
||||
CKPT="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_CHECKPOINT"
|
||||
if [ ! -f "$CKPT" ]; then
|
||||
echo "frank-large: checkpoint $CKPT missing;" >&2
|
||||
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
CKPT_SIZE="$(stat -c %s "$CKPT")"
|
||||
if [ "$CKPT_SIZE" != "$FRANK_LARGE_CHECKPOINT_BYTES" ]; then
|
||||
echo "frank-large: checkpoint is $CKPT_SIZE bytes, expected $FRANK_LARGE_CHECKPOINT_BYTES (partial download?)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec /usr/bin/env "$LLAMA_BIN" \
|
||||
--model "$MODEL_FILE" \
|
||||
"${MM_ARGS[@]}" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$BIND_ADDR" \
|
||||
--port "$LISTEN_PORT" \
|
||||
--jinja \
|
||||
--device ROCm0 \
|
||||
--n-gpu-layers "$NGL" \
|
||||
--parallel "$PARALLEL" \
|
||||
--ctx-size "$CTX_SIZE" \
|
||||
--batch-size "$BATCH_SIZE" \
|
||||
--ubatch-size "$UBATCH_SIZE" \
|
||||
--threads "$NUM_THREADS" \
|
||||
--threads-batch "$THREAD_BATCH" \
|
||||
--fit "$FIT" \
|
||||
--flash-attn "$FLASH_ATTN" \
|
||||
--direct-io \
|
||||
--cache-type-k "$CACHE_TYPE_K" \
|
||||
--cache-type-v "$CACHE_TYPE_V" \
|
||||
--ctx-checkpoints 16 \
|
||||
--cache-ram 8192 \
|
||||
--cache-idle-slots \
|
||||
--temp "$TEMPERATURE" \
|
||||
--top-p "$TOP_P" \
|
||||
--top-k "$TOP_K" \
|
||||
--min-p "$MIN_P" \
|
||||
--presence-penalty "$PRESENCE_PENALTY" \
|
||||
--reasoning on \
|
||||
--reasoning-format deepseek \
|
||||
--reasoning-budget -1 \
|
||||
--spec-type "$SPEC_TYPE" \
|
||||
--model-draft "$DRAFT_FILE" \
|
||||
--spec-draft-ngl "$SPEC_DRAFT_NGL" \
|
||||
--spec-draft-device "$SPEC_DRAFT_DEVICE" \
|
||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX" \
|
||||
--spec-draft-n-min "$SPEC_DRAFT_N_MIN" \
|
||||
--spec-draft-p-min "$SPEC_DRAFT_P_MIN" \
|
||||
--metrics
|
||||
# The tokenizer must be a regular file in a FLAT directory: pointing the
|
||||
# mount at a HuggingFace cache snapshot hands the container symlinks into
|
||||
# a sibling blobs/ that dangle inside the mount.
|
||||
TOK="$FRANK_LARGE_MODEL_DIR/$FRANK_LARGE_TOKENIZER"
|
||||
if [ ! -f "$TOK" ] || [ -L "$TOK" ]; then
|
||||
echo "frank-large: tokenizer $TOK missing or a symlink (HF cache snapshot?);" >&2
|
||||
echo " fetch it: bash scripts/download-frank-large-halogen.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# GPU access is mandatory but the node paths are variables so the seam can
|
||||
# drive the gate on a box without an AMD GPU.
|
||||
DEV_KFD="${FRANK_LARGE_DEV_KFD:-/dev/kfd}"
|
||||
DEV_DRI="${FRANK_LARGE_DEV_DRI:-/dev/dri}"
|
||||
if [ ! -e "$DEV_KFD" ] || [ ! -d "$DEV_DRI" ]; then
|
||||
echo "frank-large: $DEV_KFD or $DEV_DRI missing — no gfx1151 to serve on" >&2
|
||||
exit 1
|
||||
fi
|
||||
for CTR in ${FRANK_LARGE_LEGACY_CONTAINERS:-} "$FRANK_LARGE_CONTAINER"; do
|
||||
podman rm -f "$CTR" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
LOG="${FRANK_LARGE_LOG:-/home/dank/logs/frank_large.log}"
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
# for the udev-granted supplementary group, seccomp unconfined for the ROCm
|
||||
# ioctl surface, and unbounded memlock/nofile (the engine mmaps 33.4 GiB of
|
||||
# weights and wants a big fd budget). No comment lines inside the backslash
|
||||
# chain — a mid-chain # terminates the continuation (the 2026-10-05
|
||||
# exit-127 crash loop on the sibling serve script).
|
||||
podman run -d --name "$FRANK_LARGE_CONTAINER" \
|
||||
--publish "$FRANK_LARGE_PUBLISH" \
|
||||
--ipc=host \
|
||||
--ulimit memlock=-1:-1 --ulimit nofile=1048576:1048576 \
|
||||
--device "$DEV_KFD" --device "$DEV_DRI" \
|
||||
--group-add keep-groups \
|
||||
--security-opt seccomp=unconfined \
|
||||
-e HALOGEN_MODEL_ID="$HALOGEN_MODEL_ID" \
|
||||
-e HALOGEN_CHECKPOINT="/models/$FRANK_LARGE_CHECKPOINT" \
|
||||
-e HALOGEN_TOKENIZER="/tokenizer/$FRANK_LARGE_TOKENIZER" \
|
||||
-e HALOGEN_SLOT_CTX="$HALOGEN_SLOT_CTX" \
|
||||
-e HALOGEN_KV_SLOTS="${HALOGEN_KV_SLOTS:-1}" \
|
||||
-e HALOGEN_DRAFTER="$HALOGEN_DRAFTER" \
|
||||
-e HALOGEN_MAX_TOKENS_CAP="${HALOGEN_MAX_TOKENS_CAP:-65536}" \
|
||||
-e HALOGEN_QUEUE_TIMEOUT="${HALOGEN_QUEUE_TIMEOUT:-7200}" \
|
||||
${HALOGEN_CACHE_MB:+-e HALOGEN_CACHE_MB="$HALOGEN_CACHE_MB"} \
|
||||
-v "$FRANK_LARGE_MODEL_DIR:/models:ro" \
|
||||
-v "$FRANK_LARGE_MODEL_DIR:/tokenizer:ro" \
|
||||
"$FRANK_LARGE_IMAGE" \
|
||||
> "$LOG" 2>&1
|
||||
|
||||
# This script's lifetime IS the container's: podman wait returns the
|
||||
# container's exit code, so a crash is a nonzero ExecStart and the unit's
|
||||
# Restart=always is real. systemd stop goes through the unit's ExecStop
|
||||
# (podman wait alone cannot stop the child).
|
||||
exec podman wait "$FRANK_LARGE_CONTAINER"
|
||||
|
||||
+218
-231
@@ -1,265 +1,242 @@
|
||||
#!/usr/bin/env bats
|
||||
# Seam: frank-large serves kingjones777's UNCENSORED Qwen3.8-27B
|
||||
# Q6_0_ROCMFPX_AGENT (a ftype-114 ROCmFPX file) with the repo's MTP draft head
|
||||
# on the Strix Halo iGPU, via the ROCmFPX fork of llama.cpp — one session at
|
||||
# the native 262,144 window.
|
||||
# Seam: frank-large serves peonist-ai/halogen-qwen3.8-27b (a .hgn halogen
|
||||
# checkpoint) on the Strix Halo iGPU through the halogen CONTAINER
|
||||
# (ghcr.io/peonist-ai/halogen, digest-pinned), driven by the root unit's
|
||||
# serve script — one conversation of 200,700 tokens with the DFlash2
|
||||
# drafter ON. 2026-10-05 cutover from the ROCmFPX llama.cpp shape.
|
||||
#
|
||||
# This seam exists because the format is the constraint: GGML_TYPE_Q6_0_ROCMFPX
|
||||
# (tensor type 102) and GGML_TYPE_Q8_0_ROCMFPX (103) are understood by the
|
||||
# ROCmFPX fork and by nothing else on this host, so the runtime, the payload
|
||||
# and the deployer's gate have to agree
|
||||
# on paths that no other endpoint uses. On 2026-10-01 a serve-script gate and
|
||||
# its deployer twin disagreed during a cutover and the endpoint crash-looped;
|
||||
# both sides are pinned here.
|
||||
# This seam exists because the artifacts are a triangle that must agree on
|
||||
# exact strings the engine reads: the image DIGEST, the checkpoint file
|
||||
# name AND byte count inside the /models mount, and the tokenizer's
|
||||
# flat-file shape (a HF cache snapshot's symlinks dangle inside the
|
||||
# container). A serve gate and a deployer gate that disagree is the
|
||||
# 2026-10-01 crash-loop; both sides are pinned here, plus the one engine
|
||||
# law the fleet must never "optimize" away: HALOGEN_KV_SLOTS=1 is what
|
||||
# keeps speculative decoding on.
|
||||
#
|
||||
# Everything is tested from what an operator or a supervisor can observe — the
|
||||
# command line, the exit code, the deployer's error text — never from anything
|
||||
# inside llama.cpp.
|
||||
# Everything is tested from what an operator or supervisor can observe —
|
||||
# the command line podman would run, the exit code, the deployer's error
|
||||
# text — never from anything inside the engine.
|
||||
|
||||
setup() {
|
||||
load helpers
|
||||
|
||||
SERVE="$REPO_ROOT/scripts/serve-frank-large.sh"
|
||||
UNIT="$REPO_ROOT/scripts/frank-large.service"
|
||||
ENV_FILE="$REPO_ROOT/scripts/frank-large.env"
|
||||
DEPLOYER="$REPO_ROOT/scripts/deploy-remote.sh"
|
||||
BUILD="$REPO_ROOT/scripts/build-rocmfpx.sh"
|
||||
RUNTIME_STUB="/home/dank/GitPackages/kingjones-ROCmFPX/build/bin/llama-server"
|
||||
MODEL_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT.gguf"
|
||||
DRAFT_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mtp-Qwen3.8-27B-Q4_0.gguf"
|
||||
MMPROJ_STUB="/home/dank/models/kingjones-Qwen3.8-27B-Uncensored-ROCmFP4-STRIX-MTP/mmproj-Qwen3.8-27B-BF16.gguf"
|
||||
|
||||
ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)"
|
||||
SERVE="$ROOT/scripts/serve-frank-large.sh"
|
||||
ENV_FILE="$ROOT/scripts/frank-large.env"
|
||||
UNIT="$ROOT/scripts/frank-large.service"
|
||||
DEPLOYER="$ROOT/scripts/deploy-remote.sh"
|
||||
BUILD="$ROOT/scripts/build-rocmfpx.sh"
|
||||
TEST_TMP="$(mktemp -d)"
|
||||
SRC="$TEST_TMP/src"
|
||||
mkdir -p "$SRC"
|
||||
git -C "$SRC" init -q
|
||||
git -C "$SRC" -c user.email=t -c user.name=t commit -q --allow-empty -m x
|
||||
touch "$TEST_TMP/empty.env"
|
||||
}
|
||||
|
||||
teardown() {
|
||||
rm -rf "$TEST_TMP"
|
||||
}
|
||||
|
||||
# make_stub — an executable that prints its whole argv on one line, so
|
||||
# "what command line would the supervisor run" is observable output.
|
||||
make_stub() {
|
||||
cat > "$TEST_TMP/bin" <<'STUB'
|
||||
#!/bin/sh
|
||||
printf '%s\n' "$*"
|
||||
# podman_stub — a fake podman on PATH that prints the argv of `run` and
|
||||
# `wait` and answers `images` with whatever digests the test names, so
|
||||
# "what would the supervisor run" is observable output and the digest
|
||||
# gate is testable without a 3.5 GB image.
|
||||
podman_stub() {
|
||||
mkdir -p "$TEST_TMP/bin"
|
||||
cat > "$TEST_TMP/bin/podman" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
case "$1" in
|
||||
images) printf '%s\n' "${STUB_DIGESTS:-}" ;;
|
||||
rm) exit 0 ;;
|
||||
run|wait) echo "PODMAN $*"; exit "${STUB_WAIT_RC:-0}" ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
STUB
|
||||
chmod +x "$TEST_TMP/bin"
|
||||
chmod +x "$TEST_TMP/bin/podman"
|
||||
}
|
||||
|
||||
@test "the env file defines the deployment values" {
|
||||
grep -qE '^LLAMA_BIN=/' "$ENV_FILE"
|
||||
grep -qE '^MODEL_FILE=/' "$ENV_FILE"
|
||||
grep -qE '^DRAFT_FILE=/' "$ENV_FILE"
|
||||
grep -qE '^ALIAS=[a-z0-9-]+$' "$ENV_FILE"
|
||||
grep -qE '^LISTEN_PORT=[0-9]+$' "$ENV_FILE"
|
||||
grep -qE '^CTX_SIZE=[0-9]+$' "$ENV_FILE"
|
||||
# good_env — the minimum env the serve script demands, with the digest
|
||||
# stub agrees on and zero-byte artifacts passing the size gate.
|
||||
good_env() {
|
||||
: > "$TEST_TMP/empty.env"
|
||||
touch "$TEST_TMP/qwen.hgn" "$TEST_TMP/tok.gguf"
|
||||
GOOD_ENV=(
|
||||
"FRANK_LARGE_ENV_FILE=$TEST_TMP/empty.env"
|
||||
"FRANK_LARGE_IMAGE=ghcr.io/peonist-ai/halogen@sha256:cafe"
|
||||
"FRANK_LARGE_CONTAINER=frank-large-halogen"
|
||||
"FRANK_LARGE_LEGACY_CONTAINERS=frank-large"
|
||||
"FRANK_LARGE_DEV_KFD=$TEST_TMP/qwen.hgn"
|
||||
"FRANK_LARGE_DEV_DRI=$TEST_TMP"
|
||||
"FRANK_LARGE_LOG=$TEST_TMP/serve.log"
|
||||
"FRANK_LARGE_PUBLISH=7770:8731"
|
||||
"FRANK_LARGE_MODEL_DIR=$TEST_TMP"
|
||||
"FRANK_LARGE_CHECKPOINT=qwen.hgn"
|
||||
"FRANK_LARGE_CHECKPOINT_BYTES=0"
|
||||
"FRANK_LARGE_TOKENIZER=tok.gguf"
|
||||
"HALOGEN_MODEL_ID=frank-large"
|
||||
"HALOGEN_SLOT_CTX=200700"
|
||||
"HALOGEN_KV_SLOTS=1"
|
||||
"HALOGEN_DRAFTER=2"
|
||||
"STUB_DIGESTS=localhost/x ghcr.io/peonist-ai/halogen@sha256:cafe"
|
||||
)
|
||||
}
|
||||
|
||||
@test "the serve script fails when the runtime is missing" {
|
||||
# Only the model, the head and the projector exist; the fork was never
|
||||
# built. llama.cpp is not reached, so this fails fast and says what to
|
||||
# run instead of failing at bind time.
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/nope/llama-server" \
|
||||
"MODEL_FILE=$TEST_TMP/model.gguf" "DRAFT_FILE=$TEST_TMP/draft.gguf" \
|
||||
"MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
@test "the env file defines the halogen deployment values" {
|
||||
grep -qE '^FRANK_LARGE_IMAGE=ghcr\.io/peonist-ai/halogen@sha256:[0-9a-f]{64}$' "$ENV_FILE"
|
||||
grep -qE '^FRANK_LARGE_CONTAINER=frank-large-halogen$' "$ENV_FILE"
|
||||
grep -qE '^FRANK_LARGE_PUBLISH=7770:8731$' "$ENV_FILE"
|
||||
grep -qE '^FRANK_LARGE_CHECKPOINT=.*\.hgn$' "$ENV_FILE"
|
||||
grep -qE '^FRANK_LARGE_CHECKPOINT_BYTES=[0-9]+$' "$ENV_FILE"
|
||||
grep -qE '^HALOGEN_MODEL_ID=frank-large$' "$ENV_FILE"
|
||||
grep -qE '^HALOGEN_SLOT_CTX=[0-9]+$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the drafter stays on: slots must stay 1" {
|
||||
# halogen makes batching and speculation mutually exclusive — with
|
||||
# more than one slot the drafter is DISABLED and per-stream decode
|
||||
# falls to serial speed (the sibling engine measured 41 -> 14 t/s
|
||||
# when slots went up). frank-large was a one-session endpoint before
|
||||
# this swap too (PARALLEL=1), so slots=1 preserves the contract;
|
||||
# raising it is a product decision, never a default change.
|
||||
grep -qE '^HALOGEN_KV_SLOTS=1$' "$ENV_FILE"
|
||||
grep -qE '^HALOGEN_DRAFTER=2$' "$ENV_FILE"
|
||||
# The cap/timeout pair is coupled (FLAGS.md table): a request at the
|
||||
# cap must not outlast the queue timeout, or everyone queued behind
|
||||
# it eats a 503.
|
||||
grep -qE '^HALOGEN_MAX_TOKENS_CAP=[0-9]+$' "$ENV_FILE"
|
||||
grep -qE '^HALOGEN_QUEUE_TIMEOUT=[0-9]+$' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the serve script refuses when there is no GPU to serve on" {
|
||||
podman_stub
|
||||
good_env
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_LARGE_DEV_KFD="$TEST_TMP/absent-node" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"build it with scripts/build-rocmfpx.sh"* ]]
|
||||
[[ "$output" == *"no gfx1151 to serve on"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses to start without the model file" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/absent.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"is not readable"* ]]
|
||||
[[ "$output" == *"download-frank-large-kingjones-uncensored.sh"* ]]
|
||||
@test "the KV budget fits unified memory with margin" {
|
||||
# Pool arithmetic is the engine's documented product: slots x
|
||||
# slot_ctx x 64 KiB. Against it: 33.4 GiB of mapped weights, the
|
||||
# engine working set + auto prompt cache (~25 GiB of headroom
|
||||
# allowed), host baseline and frank-small co-tenancy. The box has
|
||||
# 121 GiB unified; the 2026-09-16 halogen era ran the same engine
|
||||
# class here on a LARGER pool with less co-tenancy slack.
|
||||
ctx="$(sed -nE 's/^HALOGEN_SLOT_CTX=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
slots="$(sed -nE 's/^HALOGEN_KV_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
pool_gib=$(( slots * ctx * 65536 / 1073741824 ))
|
||||
total_gib=$(( 34 + pool_gib + 25 + 15 ))
|
||||
[ "$total_gib" -le 121 ]
|
||||
}
|
||||
|
||||
@test "the serve script refuses to start without the MTP draft head" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/absent.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
@test "the serve script refuses when the image is not in storage" {
|
||||
podman_stub
|
||||
good_env
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" STUB_DIGESTS="" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"MTP draft head"* ]]
|
||||
[[ "$output" == *"not in podman storage"* ]]
|
||||
[[ "$output" == *"podman pull"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses a missing mmproj unless vision is opted out" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/absent.gguf" "$SERVE"
|
||||
@test "the serve script refuses a missing checkpoint and names the fetch script" {
|
||||
podman_stub
|
||||
good_env
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_LARGE_CHECKPOINT=absent.hgn "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"set MMPROJ_FILE='' for text-only"* ]]
|
||||
# The documented opt-out must actually start, without a --mmproj argument.
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=" "$SERVE"
|
||||
[[ "$output" == *"checkpoint"*"missing"* ]]
|
||||
[[ "$output" == *"download-frank-large-halogen.sh"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses a checkpoint of the wrong byte count" {
|
||||
# A truncated curl exit-0 artifact must not boot; the model card's
|
||||
# byte count is the contract (fleet fetch law).
|
||||
podman_stub
|
||||
good_env
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_LARGE_CHECKPOINT_BYTES=42 "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"expected 42"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script refuses a symlinked tokenizer" {
|
||||
# A HuggingFace cache snapshot's entries are symlinks into blobs/ and
|
||||
# dangle inside the container mount.
|
||||
podman_stub
|
||||
good_env
|
||||
ln -sf /nowhere/x "$TEST_TMP/link.gguf"
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" \
|
||||
FRANK_LARGE_TOKENIZER=link.gguf "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"missing or a symlink"* ]]
|
||||
}
|
||||
|
||||
@test "the render names the container, the port map and the engine shape" {
|
||||
podman_stub
|
||||
good_env
|
||||
run env PATH="$TEST_TMP/bin:$PATH" "${GOOD_ENV[@]}" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" != *"--mmproj"* ]]
|
||||
# `podman run` output is redirected to the log by design (the unit
|
||||
# journal must not absorb engine chatter); the stub's printed argv
|
||||
# lands there, so the render assertions read the same log.
|
||||
output="$output
|
||||
$(cat "$TEST_TMP/serve.log")"
|
||||
[[ "$output" == *"--name frank-large-halogen"* ]]
|
||||
[[ "$output" == *"--publish 7770:8731"* ]]
|
||||
[[ "$output" == *"-e HALOGEN_MODEL_ID=frank-large"* ]]
|
||||
[[ "$output" == *"-e HALOGEN_SLOT_CTX=200700"* ]]
|
||||
[[ "$output" == *"-e HALOGEN_KV_SLOTS=1"* ]]
|
||||
[[ "$output" == *"-e HALOGEN_DRAFTER=2"* ]]
|
||||
[[ "$output" == *"-e HALOGEN_CHECKPOINT=/models/qwen.hgn"* ]]
|
||||
[[ "$output" == *"--device $TEST_TMP/qwen.hgn"* ]]
|
||||
[[ "$output" == *"ghcr.io/peonist-ai/halogen@sha256:cafe"* ]]
|
||||
# The unit's restart policy is real because the script becomes
|
||||
# `podman wait`: the container's exit code is the service's exit.
|
||||
[[ "$output" == *"PODMAN wait frank-large-halogen"* ]]
|
||||
}
|
||||
|
||||
@test "the serve script fails when LLAMA_BIN is empty" {
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env LLAMA_BIN= "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"LLAMA_BIN is empty"* ]]
|
||||
@test "the shipped env pins the fleet's window and shape" {
|
||||
# 200700 is the per-conversation hard ceiling (a longer prompt is a
|
||||
# 400 naming the limit, never a silent truncation), and the harness
|
||||
# compaction math must fit prompt + budget under it: OMP compacts at
|
||||
# 70% = 140490, and 140490 + 57344 = 197834 <= 200700.
|
||||
grep -qE '^HALOGEN_SLOT_CTX=200700$' "$ENV_FILE"
|
||||
ctx=200700
|
||||
[ $(( ctx * 70 / 100 + 57344 )) -le "$ctx" ]
|
||||
}
|
||||
|
||||
@test "the shipped env points at the ROCmFPX runtime and the AGENT payload" {
|
||||
grep -qE "^LLAMA_BIN=$RUNTIME_STUB$" "$ENV_FILE"
|
||||
grep -qE "^MODEL_FILE=$MODEL_STUB$" "$ENV_FILE"
|
||||
grep -qE "^DRAFT_FILE=$DRAFT_STUB$" "$ENV_FILE"
|
||||
grep -qE "^MMPROJ_FILE=$MMPROJ_STUB$" "$ENV_FILE"
|
||||
# HIP reads bare HOST/TEMP/GPU_MAX_HEAP_SIZE out of the environment that
|
||||
# `set -a` creates; a tunable named HOST leaks into the runtime.
|
||||
@test "no HIP-sensitive bare names leak through the env file" {
|
||||
# HIP reads bare env vars out of a `set -a` source; a tunable named
|
||||
# HOST or TEMP used to send the JIT cache to path 1.0 and segfault
|
||||
# every load (kept from the llama.cpp era — the container gets only
|
||||
# the explicit -e list, but the host env still reaches podman itself).
|
||||
! grep -qE '^(HOST|TEMP|GPU_MAX_HEAP_SIZE)=' "$ENV_FILE"
|
||||
}
|
||||
|
||||
@test "the argv contract keeps the fleet's port and alias" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" \
|
||||
ALIAS=frank-large BIND_ADDR=127.0.0.1 LISTEN_PORT=7770 "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
# The caller's values must survive the env file, or a single-run operator
|
||||
# override silently does nothing.
|
||||
[[ "$output" == *"--host 127.0.0.1"* ]]
|
||||
[[ "$output" == *"--port 7770"* ]]
|
||||
[[ "$output" == *"--alias frank-large"* ]]
|
||||
# Vision is wired: the projector is the whole vision tower.
|
||||
[[ "$output" == *"--mmproj $TEST_TMP/mmproj.gguf"* ]]
|
||||
@test "the deployer gates exactly the artifacts the serve script opens" {
|
||||
# Serve and deploy must name the SAME image digest, checkpoint and
|
||||
# tokenizer: the 2026-10-01 crash-loop was exactly these twins
|
||||
# disagreeing mid-cutover.
|
||||
img="$(sed -nE 's/^FRANK_LARGE_IMAGE=(.*)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_LARGE_IMAGE=\"\${LLM_LARGE_IMAGE:-$img}\"" "$DEPLOYER"
|
||||
dir="$(sed -nE 's/^FRANK_LARGE_MODEL_DIR=(.*)$/\1/p' "$ENV_FILE")"
|
||||
ckpt="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT=(.*)$/\1/p' "$ENV_FILE")"
|
||||
tok="$(sed -nE 's/^FRANK_LARGE_TOKENIZER=(.*)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_LARGE_CKPT=\"\${LLM_LARGE_CKPT:-$dir/$ckpt}\"" "$DEPLOYER"
|
||||
grep -qF "LLM_LARGE_TOKENIZER=\"\${LLM_LARGE_TOKENIZER:-$dir/$tok}\"" "$DEPLOYER"
|
||||
bytes="$(sed -nE 's/^FRANK_LARGE_CHECKPOINT_BYTES=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
grep -qF "LLM_LARGE_CKPT_BYTES=\"\${LLM_LARGE_CKPT_BYTES:-$bytes}\"" "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the shipped argv is one full native window on one session" {
|
||||
grep -qE '^CTX_SIZE=262144$' "$ENV_FILE"
|
||||
grep -qE '^PARALLEL=1$' "$ENV_FILE"
|
||||
# 262144 IS the model's trained window (qwen35.context_length in the GGUF),
|
||||
# so 1 slot gets the whole thing — no truncation, no yarn scaling.
|
||||
#
|
||||
# Budget, from the GGUF header: 23.15 GiB of weights + 1.57 GiB draft head
|
||||
# + 0.87 GiB projector + 16 full-attention layers x 4 kv heads x 256 dim x
|
||||
# 2 (K+V) x 2 B = 64 KiB/token of f16 KV (~16 GiB at 262,144) + ~0.14 GiB
|
||||
# of DeltaNet state + a couple GiB of compute buffers, against the iGPU's
|
||||
# ~114 GiB GTT aperture. Change CTX_SIZE and this arithmetic, not a boot
|
||||
# log, is what tells you it no longer fits.
|
||||
ctx="$(sed -nE 's/^CTX_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
||||
kv_gib=$((ctx * 65536 / 1073741824))
|
||||
total_gib=$((24 + 2 + 1 + kv_gib + 3))
|
||||
[ "$total_gib" -le 114 ]
|
||||
@test "the deployer points a missing artifact at the halogen fetch script" {
|
||||
grep -qE 'download-frank-large-halogen\.sh' "$DEPLOYER"
|
||||
grep -qE 'podman pull' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the MTP draft contract matches the model card's measured optimum" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
# The head is the model's own nextn layer, loaded as a separate draft
|
||||
# model because this checkpoint ships it standalone (no blk.<n>.nextn.*
|
||||
# tensors in the target file).
|
||||
[[ "$output" == *"--spec-type draft-mtp"* ]]
|
||||
[[ "$output" == *"--model-draft $TEST_TMP/draft.gguf"* ]]
|
||||
# n-max 4 is the measured optimum; n-max 8 measured WORSE (acceptance
|
||||
# fell to 0.757 and the extra verification cost outran the savings), so
|
||||
# the flag is a measurement, not a wish.
|
||||
[[ "$output" == *"--spec-draft-n-max 4"* ]]
|
||||
[[ "$output" == *"--spec-draft-n-min 0"* ]]
|
||||
[[ "$output" == *"--spec-draft-p-min 0.0"* ]]
|
||||
# The draft runs on the iGPU beside the target: no device-to-device hop
|
||||
# on a link that measures single-digit GB/s.
|
||||
[[ "$output" == *"--spec-draft-device ROCm0"* ]]
|
||||
[[ "$output" == *"--spec-draft-ngl 99"* ]]
|
||||
}
|
||||
|
||||
@test "KV and placement stay unquantized with flash attention on" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
[[ "$output" == *"--cache-type-k f16"* ]]
|
||||
[[ "$output" == *"--cache-type-v f16"* ]]
|
||||
# ROCmFPx quantized KV has no kernel on this branch, and the endpoint has
|
||||
# the aperture to keep KV unquantized anyway.
|
||||
[[ "$output" == *"--flash-attn on"* ]]
|
||||
# --fit's placement heuristics do not know that 311 tensors of type 103
|
||||
# belong on the iGPU, so the whole stack is placed explicitly instead.
|
||||
[[ "$output" == *"--fit off"* ]]
|
||||
[[ "$output" == *"--n-gpu-layers 999"* ]]
|
||||
}
|
||||
|
||||
@test "the client contract wires thinking and template kwargs" {
|
||||
make_stub
|
||||
touch "$TEST_TMP/model.gguf" "$TEST_TMP/draft.gguf" "$TEST_TMP/mmproj.gguf"
|
||||
run env "LLAMA_BIN=$TEST_TMP/bin" "MODEL_FILE=$TEST_TMP/model.gguf" \
|
||||
"DRAFT_FILE=$TEST_TMP/draft.gguf" "MMPROJ_FILE=$TEST_TMP/mmproj.gguf" "$SERVE"
|
||||
[ "$status" -eq 0 ]
|
||||
# Without --jinja, llama.cpp silently ignores the harness's
|
||||
# chat_template_kwargs, so thinking control disappears from the API.
|
||||
[[ "$output" == *"--jinja"* ]]
|
||||
# The harness reads `reasoning_content` for this endpoint, which is what
|
||||
# the deepseek emission format produces (vLLM's `reasoning` is not it).
|
||||
[[ "$output" == *"--reasoning-format deepseek"* ]]
|
||||
# The template needs an unbounded thinking budget.
|
||||
[[ "$output" == *"--reasoning-budget -1"* ]]
|
||||
}
|
||||
|
||||
@test "the deployer gates exactly the paths the serve script opens" {
|
||||
# A gate that checks a different file than the engine opens is the
|
||||
# 2026-10-01 crash-loop: deploy passed, the unit restart-looped. Read the
|
||||
# engine's own paths out of the env file and require the deployer's
|
||||
# defaults to name the same ones.
|
||||
for pair in "LLAMA_BIN:LLM_LARGE_RUNTIME" \
|
||||
"MODEL_FILE:LLM_LARGE_MODEL" \
|
||||
"DRAFT_FILE:LLM_LARGE_DRAFT" \
|
||||
"MMPROJ_FILE:LLM_LARGE_MMPROJ"; do
|
||||
env_key="${pair%%:*}"
|
||||
dep_key="${pair##*:}"
|
||||
want="$(sed -nE "s|^${env_key}=(.*)\$|\1|p" "$ENV_FILE")"
|
||||
[ -n "$want" ]
|
||||
got="$(grep -oE "${dep_key}:-[^}]+" "$DEPLOYER" | head -1 | sed "s|^${dep_key}:-||")"
|
||||
[ "$got" = "$want" ]
|
||||
done
|
||||
}
|
||||
|
||||
@test "the deployer points a missing runtime at the fork build script" {
|
||||
grep -qE 'build it: bash scripts/build-rocmfpx\.sh' "$DEPLOYER"
|
||||
grep -qE 'fetch it: bash scripts/download-frank-large-kingjones-uncensored\.sh' "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the stray-process sweep cannot take frank-small down with frank-large" {
|
||||
# frank-small serves a different GGUF over llama-server on the SAME host.
|
||||
# A sweep keyed on the binary name (`pkill -f llama-server`) would kill
|
||||
# the 3090 endpoint too, so it is keyed on frank-large's own payload file.
|
||||
grep -q "pkill -f 'Qwen3.8-27B-Uncensored-Q6_0_ROCMFPX_AGENT'" "$DEPLOYER"
|
||||
@test "the stray sweep cannot take frank-small or the deployer down" {
|
||||
# frank-small also listens on this host (7771, llama-server) and the
|
||||
# deployer's own command line contains "frank-large"; only the FULL
|
||||
# container name matches a stray manual halogen run and nothing else.
|
||||
grep -q "pkill -f 'frank-large-halogen'" "$DEPLOYER"
|
||||
! grep -qE "pkill -f 'llama-server'" "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the build script refuses a source tree that is not the pinned commit" {
|
||||
# The pin is load-bearing: the older builds of this fork line segfault in
|
||||
# ggml_compute_forward_set_rows on this exact payload. A runtime that
|
||||
# differs from what was qualified must be heard about at deploy time.
|
||||
wrong="$(git -C "$SRC" rev-parse HEAD)"
|
||||
run env ROCMFPX_SRC_DIR="$SRC" ROCMFPX_COMMIT="ffff$wrong" \
|
||||
"$BUILD" --verify
|
||||
[ "$status" -ne 0 ]
|
||||
[[ "$output" == *"ROCMFPX_COMMIT"*mismatch* ]]
|
||||
}
|
||||
|
||||
@test "the build script pins a real commit and the gfx1151 target" {
|
||||
grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
|
||||
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
||||
grep -qE 'DGPU_TARGETS=' "$BUILD"
|
||||
# The webui asset fetch half-failed on this box and aborted the link at
|
||||
# 98%; the build must not depend on it.
|
||||
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
||||
! grep -qE "pkill -f 'frank-large'" "$DEPLOYER"
|
||||
}
|
||||
|
||||
@test "the deployed unit keeps the fleet's restart and boot-persistence shape" {
|
||||
@@ -267,15 +244,25 @@ STUB
|
||||
grep -qE '^WantedBy=' "$UNIT"
|
||||
grep -qE '^ExecStart=.*/serve-frank-large\.sh$' "$UNIT"
|
||||
grep -qE '^User=dank$' "$UNIT"
|
||||
# The payload is 26 GiB of mmap'd weights plus pinned staging buffers; an
|
||||
# RLIMIT smaller than the pinned set fails the load in a way that looks
|
||||
# like an engine crash.
|
||||
grep -qE '^LimitMEMLOCK=infinity$' "$UNIT"
|
||||
# The service's lifetime is the container's; stopping the service
|
||||
# must stop the container by its real name.
|
||||
grep -qE '^ExecStop=.*podman stop .* frank-large-halogen$' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the unit points at no retired frank-large runtime path" {
|
||||
# The host keeps every retired payload and build on disk, so a stale path
|
||||
# in a unit still resolves — to a runtime nobody has measured in weeks.
|
||||
@test "the unit runs the halogen runtime and no retired path" {
|
||||
grep -q 'halogen' "$UNIT"
|
||||
# Every retired payload/build stays on the host, so a stale path in
|
||||
# a unit still resolves — to a runtime nobody has measured in weeks.
|
||||
! grep -qE 'GitPackages/(ROCmFPX-kairic|llama.cpp-ik_llama|llama-builds|ciru-orca)' "$UNIT"
|
||||
! grep -q 'halogen' "$UNIT"
|
||||
! grep -qE 'kingjones-ROCmFPX' "$UNIT"
|
||||
}
|
||||
|
||||
@test "the rollback door stays qualified: the ROCmFPX build pin survives" {
|
||||
# Rolling back this swap means rebuilding the fork the old payload
|
||||
# needs; the pin (real commit, gfx1151, no webui fetch) is what makes
|
||||
# that door trustworthy months later.
|
||||
grep -qE 'ROCMFPX_COMMIT=.*[0-9a-f]{40}' "$BUILD"
|
||||
grep -qE 'GPU_TARGET=.*gfx1151' "$BUILD"
|
||||
grep -qE 'DLLAMA_BUILD_WEBUI=OFF' "$BUILD"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user