frank-primary: vLLM 0.30 UVA-offload shape (pre-deploy commit)
The llama.cpp b11030 trio becomes the vLLM 0.30 trio ahead of the FP8 cutover on :7770 — same port, same alias, no client moves. Serve shape: locally built ROCm gfx1201 image (localhost/frank-primary:vllm030) execs via podman as dank from the root unit (LimitMEMLOCK=infinity legalizes the pin). UVA offload: --offload-backend uva pins the PLE family (47.75 GiB, 138 tensors: 131 n-gram table shards + 7 per-layer conv/proj) + routed experts (114.86 GiB, 150,528 tensors) in host RAM — 162.60 GiB combined, ~81.3 GiB per rank under TP=2, hence --cpu-offload-gb 82 (80/card strands 2.6 GiB of experts on the cards). Parsers per the host's own Ornith-era precedent: qwen3_xml tools + qwen3 reasoning (vLLM 0.30 emits the channel as `reasoning` — reasoning_content is renamed server-side) + enable_thinking/preserve_thinking template kwargs. Spec decode stays env-wired and OFF at bring-up. Deployer deploy_llm_primary rewritten: image gate, index-driven shard-count gate (131), stale-container cleanup, port wait, and the false-green hardening — health is only green when the unit's own MainPID is live behind the answer (2026-09-26 lesson). Seam tests rewritten to the vLLM contract (15 gates): offload arithmetic, parser wiring, no-llama.cpp-ism, deployer gates. Suite 349/349 green.
This commit is contained in:
Frank committed
1 parent
82960c77db
commit
ecaad36654
5 files changed
+378
-425
No files matched your search
+54
-79
@@ -1183,98 +1183,73 @@ deploy_llm_small() {
|
|||||||
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
||||||
}
|
}
|
||||||
|
|
||||||
# deploy_llm_primary — frank-primary is unsloth llama.cpp (the b11030-mix
|
# deploy_llm_primary — frank-primary is vLLM 0.30 (locally built ROCm image,
|
||||||
# prebuilt, ROCm gfx120X, bundled runtime) serving Qwen3.8-Flash-Next at
|
# gfx1201) serving Qwen3.8-Flash-Next-FP8 on the two R9700s as a ROOT unit
|
||||||
# Q8_0 on the two R9700s, as a ROOT unit — the same shape as frank-large on
|
# whose serve script execs a rootless-podman container as dank. Same port and
|
||||||
# Dankclaw: a host binary, no container image, no generated unit.
|
# alias as the llama.cpp endpoint it replaces (2026-09-27 cutover); the
|
||||||
# relink_unit points /etc/systemd/system/frank-primary.service at the repo
|
# harness models.yml and homepage cards are unchanged.
|
||||||
# file and boot persistence is that unit's [Install] section.
|
|
||||||
#
|
#
|
||||||
# The 2026-09-26 swap that replaced the Lucebox DS4.1 Quadlet on this port.
|
# Gates: the image must exist in dank's rootless storage, and the payload
|
||||||
# The payload is six Q8_0 split parts plus the shared MTP head and the
|
# must be complete — every shard named by the model's own
|
||||||
# projector, so the gate walks every artifact at its declared byte count (a
|
# model.safetensors.index.json present on disk (131 shards, 172.78 GiB;
|
||||||
# partial download must never be served — and stat -c %s, never wc -c, which
|
# per-shard sha256 + byte-count verification is download-frank-primary-fp8.sh's
|
||||||
# read the whole 377 GiB lucebox file for minutes on 2026-09-26).
|
# job at fetch time, never re-hashed here).
|
||||||
LLM_PRIMARY_RUNTIME=/home/dank/GitPackages/unsloth-llama-b11030/llama-server
|
LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030
|
||||||
LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-GGUF
|
LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8
|
||||||
LLM_PRIMARY_PART_SIZES=(
|
LLM_PRIMARY_CONTAINER=frank-primary-vllm
|
||||||
10946624
|
|
||||||
682434912
|
|
||||||
54400261312
|
|
||||||
49446841216
|
|
||||||
49668930400
|
|
||||||
34015618784
|
|
||||||
)
|
|
||||||
LLM_PRIMARY_DRAFT="$LLM_PRIMARY_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
|
||||||
LLM_PRIMARY_DRAFT_SIZE=2786568256
|
|
||||||
LLM_PRIMARY_MMPROJ="$LLM_PRIMARY_ROOT/mmproj-BF16.gguf"
|
|
||||||
LLM_PRIMARY_MMPROJ_SIZE=907542944
|
|
||||||
|
|
||||||
# The lucebox quadlet held this port until the 2026-09-26 cutover. Its
|
|
||||||
# generated user unit is host state, not repo state: stop it and move the
|
|
||||||
# installed .container into dated home trash (the same retire pattern as
|
|
||||||
# retire_halogen_large_quadlet). The image and the 377 GiB payload stay on
|
|
||||||
# disk as the rollback target.
|
|
||||||
retire_lucebox_primary_quadlet() {
|
|
||||||
local trash="$HOME/.AGENTTRASH/20260926-llamacpp-cutover/quadlet"
|
|
||||||
systemctl --user stop frank-primary.service 2>/dev/null || true
|
|
||||||
if [[ -e "$quadlet_unit_dir/frank-primary.container" ]]; then
|
|
||||||
mkdir -p "$trash"
|
|
||||||
mv -- "$quadlet_unit_dir/frank-primary.container" "$trash/frank-primary.container"
|
|
||||||
echo "retired quadlet frank-primary.container -> $trash"
|
|
||||||
fi
|
|
||||||
systemctl --user daemon-reload
|
|
||||||
}
|
|
||||||
|
|
||||||
deploy_llm_primary() {
|
deploy_llm_primary() {
|
||||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||||
# missing never becomes healthy, and a green deploy would leave the
|
# missing never becomes healthy, and a green deploy would leave the
|
||||||
# endpoint dark with nothing in the deploy log to say why.
|
# endpoint dark with nothing in the log to say why.
|
||||||
local missing=0 i part want_size got_size
|
if ! podman image inspect "$LLM_PRIMARY_IMAGE" >/dev/null 2>&1; then
|
||||||
if [[ ! -x "$LLM_PRIMARY_RUNTIME" ]]; then
|
echo "ERROR: image absent: $LLM_PRIMARY_IMAGE" >&2
|
||||||
echo "ERROR: runtime absent at $LLM_PRIMARY_RUNTIME" >&2
|
echo "frank-primary: build the fp030 image first (see the runbook)" >&2
|
||||||
echo "frank-primary: fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2
|
|
||||||
missing=1
|
|
||||||
fi
|
|
||||||
for i in 1 2 3 4 5 6; do
|
|
||||||
part="$LLM_PRIMARY_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-0000$i-of-00006.gguf"
|
|
||||||
want_size="${LLM_PRIMARY_PART_SIZES[$((i-1))]}"
|
|
||||||
got_size="$(stat -c %s "$part" 2>/dev/null || echo 0)"
|
|
||||||
if [[ "$got_size" != "$want_size" ]]; then
|
|
||||||
echo "ERROR: part $i absent or incomplete at $part (have $got_size, want $want_size)" >&2
|
|
||||||
missing=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
for part in "$LLM_PRIMARY_DRAFT:$LLM_PRIMARY_DRAFT_SIZE" \
|
|
||||||
"$LLM_PRIMARY_MMPROJ:$LLM_PRIMARY_MMPROJ_SIZE"; do
|
|
||||||
local file="${part%%:*}" want="${part##*:}"
|
|
||||||
got_size="$(stat -c %s "$file" 2>/dev/null || echo 0)"
|
|
||||||
if [[ "$got_size" != "$want" ]]; then
|
|
||||||
echo "ERROR: artifact absent or incomplete at $file (have $got_size, want $want)" >&2
|
|
||||||
missing=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
[[ "$missing" -eq 0 ]] || {
|
|
||||||
echo "frank-primary: run scripts/download-frank-primary.sh on this host" >&2
|
|
||||||
return 1
|
return 1
|
||||||
}
|
fi
|
||||||
|
local want_shards have_shards
|
||||||
|
want_shards="$(python3 - "$LLM_PRIMARY_ROOT/model.safetensors.index.json" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
print(len(set(json.load(open(sys.argv[1]))["weight_map"].values())))
|
||||||
|
PY
|
||||||
|
)" || return 1
|
||||||
|
have_shards="$(find "$LLM_PRIMARY_ROOT" -maxdepth 1 -name '*.safetensors' | wc -l)"
|
||||||
|
if [[ -z "$want_shards" || "$have_shards" != "$want_shards" ]]; then
|
||||||
|
echo "ERROR: payload incomplete at $LLM_PRIMARY_ROOT (have $have_shards shards, index names $want_shards)" >&2
|
||||||
|
echo "frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
retire_lucebox_primary_quadlet
|
# A stale container with the same name makes `podman run --name` fail at
|
||||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
# start; remove any leftover from a failed previous attempt.
|
||||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
podman rm -f "$LLM_PRIMARY_CONTAINER" >/dev/null 2>&1 || true
|
||||||
|
# On a re-deploy the predecessor is THIS unit (the llama.cpp unit until
|
||||||
|
# the 2026-09-27 cutover, the vLLM unit after it): stop it before
|
||||||
|
# waiting on the port, or the wait deadlocks on the server it just
|
||||||
|
# stopped serving.
|
||||||
run_privileged systemctl stop frank-primary.service 2>/dev/null || true
|
run_privileged systemctl stop frank-primary.service 2>/dev/null || true
|
||||||
# Nothing starts until the port is released: the lucebox cutover on
|
# Nothing starts until the port is released: the lucebox cutover
|
||||||
# 2026-09-26 and the frank-large cutover on 2026-09-16 both failed
|
# (2026-09-26) and the frank-large cutover (2026-09-16) both failed
|
||||||
# exactly here when the next engine bound 7770 a second too early.
|
# exactly here when the next engine bound 7770 a second too early.
|
||||||
wait_port_free 7770 || return 1
|
wait_port_free 7770 || return 1
|
||||||
relink_unit frank-primary
|
relink_unit frank-primary
|
||||||
run_privileged systemctl daemon-reload
|
run_privileged systemctl daemon-reload
|
||||||
run_privileged systemctl enable frank-primary.service
|
run_privileged systemctl enable frank-primary.service
|
||||||
run_privileged systemctl restart frank-primary.service
|
run_privileged systemctl restart frank-primary.service
|
||||||
# Cold start page-ins 175 GiB of expert pages and builds the graphs;
|
# Cold start: triton/inductor compile on first boot plus reading+pinning
|
||||||
# 1800s parity with the lucebox HealthStartPeriod this port has always
|
# ~173 GiB of weights. Parity with every predecessor on this port.
|
||||||
# needed.
|
wait_for_health frank-primary http://127.0.0.1:7770/health 1800 || return 1
|
||||||
wait_for_health frank-primary http://127.0.0.1:7770/health 1800
|
# False-green hardening (the 2026-09-26 lesson): a bare curl against the
|
||||||
|
# port cannot tell WHO answers — a stray probe holding :7770 satisfies
|
||||||
|
# it while the unit is stopped. The deploy is only green if the unit
|
||||||
|
# itself is active with a live MainPID behind the health answer.
|
||||||
|
local main_pid
|
||||||
|
main_pid="$(run_privileged systemctl show -p MainPID --value frank-primary.service)"
|
||||||
|
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||||
|
echo "FAIL: :7770 answered but frank-primary.service has no MainPID — a stray process holds the port" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "frank-primary: health answered by unit MainPID $main_pid"
|
||||||
}
|
}
|
||||||
|
|
||||||
wait_for_health() {
|
wait_for_health() {
|
||||||
|
|||||||
+84
-101
@@ -1,113 +1,96 @@
|
|||||||
# frank-primary — unsloth llama.cpp (b11030-mix prebuilt, ROCm gfx120X)
|
# frank-primary.env — tunables for the vLLM 0.30 frank-primary endpoint
|
||||||
# serving unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 on the two Radeon AI PRO
|
# (Qwen3.8-Flash-Next-FP8, 2x R9700 gfx1201, dankserver :7770).
|
||||||
# R9700s (gfx1201), with the shared Q8_0 MTP drafter head, the BF16 vision
|
|
||||||
# projector, and the draft-mtp + ngram-mod spec chain.
|
|
||||||
#
|
#
|
||||||
# These are the tunables read by scripts/serve-frank-primary.sh. The flag
|
# Paths and numbers only — NEVER credentials. Do not name a tunable TEMP, HOST
|
||||||
# list is owned by the serve script; this file only sets fleet values.
|
# or GPU_MAX_HEAP_SIZE: the HIP runtime reads those bare names (2026-08-13).
|
||||||
#
|
#
|
||||||
# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP
|
# Container paths: the model tree is mounted at /models (host
|
||||||
# runtime reads those bare (the 2026-08-13 TEMP incident). The bind address
|
# /home/dank/models); the compile caches live under /cache (host
|
||||||
# is BIND_HOST, not HOST — frank-large's env predates that lesson and gets
|
# /home/dank/.cache/frank-primary-vllm, keyed on model + stack + graph knobs —
|
||||||
# away with it; do not copy the shape.
|
# never share it across configurations).
|
||||||
|
|
||||||
# The payload (scripts/download-frank-primary.sh): the serve script refuses
|
# --- identity ---------------------------------------------------------------
|
||||||
# to start without all six Q8_0 parts, the MTP head and the projector.
|
SERVED_NAME=frank-primary
|
||||||
MODEL_ROOT=/home/dank/models/Qwen3.8-Flash-Next-GGUF
|
|
||||||
|
|
||||||
# The runtime: the unsloth b11030-mix prebuilt for gfx120X (the same
|
|
||||||
# b11030-mix build frank-large runs on gfx1151 — it bundles its own ROCm
|
|
||||||
# runtime, no host SDK gate).
|
|
||||||
RUNTIME_DIR=/home/dank/GitPackages/unsloth-llama-b11030
|
|
||||||
|
|
||||||
# Fleet endpoint: the port and alias the lucebox DS4.1 quadlet held before
|
|
||||||
# the 2026-09-26 swap, so no client, probe or models.yml entry changes
|
|
||||||
# address.
|
|
||||||
BIND_HOST=0.0.0.0
|
BIND_HOST=0.0.0.0
|
||||||
PORT=7770
|
PORT=7770
|
||||||
|
|
||||||
# Four conversations, each the model's full native 262144 window.
|
# --- model ------------------------------------------------------------------
|
||||||
# --parallel divides --ctx-size into slots — the pool is 4x262144 cells
|
# Host-side dir: /home/dank/models/Qwen3.8-Flash-Next-FP8 (131 safetensors
|
||||||
# (~23 GiB at the V-quantized law below, split across both cards). Slots
|
# shards, 172.78 GiB, verified against model.safetensors.index.json).
|
||||||
# DIVIDE the pool; they never multiply it. Per-slot window stays the
|
MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8
|
||||||
# full native 262144, so no client or models.yml change is implied.
|
|
||||||
CONTEXT_SIZE=1048576
|
|
||||||
PARALLEL_SLOTS=4
|
|
||||||
|
|
||||||
# Spec chain: draft-mtp (the shared Q8_0 head) at depth 2, with the
|
# --- placement --------------------------------------------------------------
|
||||||
# draftless ngram-mod matcher chained behind it. Depth 2 is the measured
|
# Fixed 2-card tensor parallel (the R9700 pair; the Raphael iGPU is index 2
|
||||||
# optimum class for this pair at long contexts (the retired Dankclaw
|
# and excluded by the visibility pins).
|
||||||
# Flash-Next deployment ran exactly this head at depth 2 with 0.83-0.92
|
ROCR_VISIBLE_DEVICES=0,1
|
||||||
# acceptance); each extra verified position re-batches the CPU expert
|
HIP_VISIBLE_DEVICES=0,1
|
||||||
# reads, so depth buys less than bandwidth here.
|
TP=2
|
||||||
SPEC_DRAFT_N_MAX=2
|
# 0.97 of 31.86 GiB/card (34,208,743,424 B) — leaves ~1 GiB for the ROCm
|
||||||
|
# baseline. Raise only with a measured reason.
|
||||||
|
GPU_UTIL=0.97
|
||||||
|
# The model's native window.
|
||||||
|
CTX_SIZE=262144
|
||||||
|
# Sequences the scheduler may hold. 8 is the proven batch shape on this host
|
||||||
|
# (radiance era); revisit with a bench after bring-up.
|
||||||
|
MAX_SEQS=8
|
||||||
|
|
||||||
# Device placement. -ngl 999 puts every layer's non-overridden tensors on
|
# --- weight offloading (the UVA shape) ---------------------------------------
|
||||||
# the GPUs; the big overrides then move the bandwidth-huge tensors to host
|
# --offload-backend uva: matched weights live in PINNED host RAM and are read
|
||||||
# RAM (computed there — the x4-pair measurement: streaming over PCIe loses,
|
# in place over PCIe by the kernels (zero-copy), instead of being resident
|
||||||
# computing in RAM does not care). OT_OVERRIDES is ONE comma-separated
|
# on the cards. is_uva_available() = is_pin_memory_available() on ROCm; the
|
||||||
# <tensor-pattern>=<backend> list passed as a single --override-tensor
|
# pin needs the unit's LimitMEMLOCK=infinity (a root unit grants it via
|
||||||
# flag (b11030 logs DEPRECATED for repeated flags and wants the joined
|
# PID1 — the dankserver user session defaults to 8 MiB and cannot raise it).
|
||||||
# form), and the loader applies the FIRST matching pattern per tensor —
|
# If pinning is unavailable the offloader falls back to a copy path — the
|
||||||
# so the block-specific patterns MUST precede the broad exps=CPU sweep.
|
# load log MUST be checked for the UVA path, not just /health.
|
||||||
#
|
# --cpu-offload-gb is PER GPU (GiB). 2x R9700 total pinned budget ≈ 160 GiB of
|
||||||
# Hot expert blocks stay GPU-resident (2026-09-26): -ot granularity is the
|
# the 246 GiB host RAM; the rest covers the OS, llama.cpp-era page cache
|
||||||
# whole per-block expert tensor (512 experts share one tensor), so "critical
|
# drain and the pinned staging buffers. Do not raise without checking
|
||||||
# experts" = blocks co-located on the card that holds their layers.
|
# MemAvailable at load time.
|
||||||
# Measured cost: 2.42 GiB/block on ROCm0, 2.49 GiB/block on ROCm1.
|
# --cpu-offload-params: name SEGMENTS (dot-delimited components) of tensors
|
||||||
# Blocks 0-4 on ROCm0 + 45-47 on ROCm1: card0 runs at ~2.5 GiB free at
|
# to offload. Measured from the payload headers (2026-09-26): `ple` matches
|
||||||
# boot / ~2.1 GB under load (vision graph included) — the operator
|
# 138 tensors = 131 n-gram table shards + 7 per-layer conv/proj weights,
|
||||||
# directed the fill PAST the 3 GB checkpoint floor on 2026-09-26
|
# 47.75 GiB; `experts` matches 150,528 tensors, 114.86 GiB. Combined
|
||||||
# ("shift a few more experts"); measured, battery-verified. Card1 keeps
|
# 162.60 GiB — with TP=2 each rank needs ~81.3 GiB of pinned host RAM,
|
||||||
# ~3.4 GB. Card1 CANNOT take another block: 3.31 GiB free − 2.49 GiB =
|
# hence OFFLOAD_GB=82 per GPU (164 GiB total; 80/card left 2.6 GiB of
|
||||||
# 0.82 GiB, and the vision graph + boot drift alone exceed that.
|
# experts resident, inflating the on-card weight share). Resident after
|
||||||
# * exps=CPU — the remaining routed experts (blk.*.ffn_*_exps.weight,
|
# the full offload: 10.16 GiB of dense/attention/MTP/ViT/lm_head weights.
|
||||||
# ~100 GiB at Q8_0 even with 7 blocks GPU-side), read once per token;
|
# Matching is greedy-up-to-budget in traversal order; unmatched params are
|
||||||
# the CPU backend computes them from page cache.
|
# never offloaded.
|
||||||
# * per_layer_token_embd.weight=CPU — the 51B-element PLE n-gram table
|
OFFLOAD_BACKEND=uva
|
||||||
# (dims [160, 320001536], Q8_0, 54.4 GiB — the whole of part 3):
|
OFFLOAD_GB=82
|
||||||
# sparse row lookups, tiny PCIe traffic from CPU RAM. It doubles as
|
OFFLOAD_PARAMS=ple,experts
|
||||||
# the shared MTP head's borrow tensor — GPU-resident, the draft load
|
|
||||||
# crashes with "invalid vector subscript" (llama.cpp PR #28243) — so
|
|
||||||
# this override is REQUIRED, not just a placement choice. NEVER give
|
|
||||||
# it a GPU target (the 23 -> 13.4 t/s regression) and never --no-mmap
|
|
||||||
# (the table must stay page-cache-backed; anonymous memory
|
|
||||||
# cgroup-kills silently).
|
|
||||||
NGL=999
|
|
||||||
OT_OVERRIDES='blk\.[0-4]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU'
|
|
||||||
TENSOR_SPLIT=1,1
|
|
||||||
|
|
||||||
# The drafter MUST sit on the SECOND R9700 (ROCm1): with -ts 1,1 the main
|
# --- engine -----------------------------------------------------------------
|
||||||
# model's output head lands on the last-layers card, and the shared MTP head
|
# No aiter wheel in this image (gfx9-only upstream); the AMD qwen4_exp path
|
||||||
# BORROWS it. --override-tensor sets has_tensor_overrides(), which pins
|
# runs its in-tree triton kernels. Deterministic rather than relying on the
|
||||||
# n_copies==1 in the draft's scheduler — a borrowed output.weight sitting in
|
# availability probe.
|
||||||
# a ROCm1 buffer then aborts the draft graph with ggml-backend.cpp:941
|
VLLM_ROCM_USE_AITER=0
|
||||||
# "pre-allocated tensor (output.weight) in a buffer (ROCm1) that cannot run
|
# Compile/inductor/triton caches (container /cache, TRITON_CACHE_DIR etc.)
|
||||||
# the operation" when the draft is elsewhere (verified live 2026-09-26: the
|
CACHE_ROOT=/cache
|
||||||
# ROCm0 placement crash-looped 4x at ~12 s, ROCm1 loaded and served).
|
|
||||||
SPEC_DRAFT_DEVICE=ROCm1
|
|
||||||
|
|
||||||
# Batch geometry: logical batch 8192 with ubatch 512 — the measured optimum
|
# --- parsers / template (the harness wire contract) ---------------------------
|
||||||
# class on this fleet's ROCm stack (a bigger logical batch is free; compute
|
# vLLM 0.30 emits the thinking channel as `reasoning` (the deprecated
|
||||||
# buffers reserve at max graph size). ubatch 512 balances prefill
|
# `reasoning_content` is renamed server-side, chat_completion/protocol.py:554).
|
||||||
# throughput against the asymmetric compute-buffer cost on the card that
|
# The qwen3 reasoning parser extracts the template's think-block tags into
|
||||||
# holds the output layers and the drafter.
|
# that channel; the qwen3_xml tool parser structures tool-call XML output into
|
||||||
BATCH_SIZE=8192
|
# tool_calls (auto-tool-choice enables the structured path for API clients).
|
||||||
UBATCH_SIZE=512
|
# The family precedent is the host's own Ornith-era serve scripts
|
||||||
|
# (~/vllm/failedAttempts/OrnithFP8.sh); the FP8 tokenizer's template
|
||||||
|
# consumes enable_thinking + preserve_thinking (verified in tokenizer_config).
|
||||||
|
TOOL_PARSER=qwen3_xml
|
||||||
|
REASONING_PARSER=qwen3
|
||||||
|
CHAT_TEMPLATE_KWARGS='{"enable_thinking": true, "preserve_thinking": true}'
|
||||||
|
|
||||||
# KV cache types: V quantized to q8_0, K held at the f16 default. The V
|
# --- speculative decode ------------------------------------------------------
|
||||||
# half is what quantizes on this build: main KV drops 24,576 -> 18,816
|
# The hybrid stack (GatedDeltaNet + QSA short-conv) uses the v1 engine's
|
||||||
# B/token (12 of 48 layers carry KV; the 36 SSM layers are constant-state
|
# mamba-capable attention backends, negotiated by the model, not by env.
|
||||||
# and never scale with ctx). NOTE the shared MTP drafter's KV ignores the
|
# MTP drafting comes from the model's own mtp.* weights via
|
||||||
# V type (measured: draft pool 2,048 MiB = the f16 law at 1,048,576
|
# --speculative-config, only when SPEC_CONFIG is non-empty. Bring-up runs
|
||||||
# cells) — one layer's worth, so the miss costs ~0.5 GiB on ROCm1.
|
# with it empty on purpose: measure the base shape clean, then A/B; never
|
||||||
CACHE_TYPE_K=f16
|
# change two knobs at once.
|
||||||
CACHE_TYPE_V=q8_0
|
SPEC_CONFIG=
|
||||||
|
|
||||||
# Prompt-cache budget: 16 GiB. The V-quantized KV pool at four full
|
|
||||||
# windows is ~23 GiB spread across both cards (the hybrid DeltaNet/QSA
|
# --- container ---------------------------------------------------------------
|
||||||
# pattern keeps even 4x262144 a fraction of a full-ctx x all-layers
|
IMAGE=localhost/frank-primary:vllm030
|
||||||
# pool); the prompt cache is the RAM-side spend that actually moves
|
CONTAINER_NAME=frank-primary-vllm
|
||||||
# prefill on repeat traffic. Host RAM budget: ~90 GiB of expert pages +
|
|
||||||
# this cache + the server's own arenas inside 246 GiB.
|
|
||||||
CACHE_RAM=16384
|
|
||||||
@@ -1,19 +1,35 @@
|
|||||||
|
# frank-primary.service — vLLM 0.30 endpoint (Qwen3.8-Flash-Next-FP8,
|
||||||
|
# UVA offload shape) on the 2x R9700 pair, dankserver :7770.
|
||||||
|
#
|
||||||
|
# Replaces the llama.cpp frank-primary unit of 2026-09-26 (same port, same
|
||||||
|
# served alias; harness models.yml unchanged). The serve script execs a
|
||||||
|
# podman container (image localhost/frank-primary:vllm030) as dank.
|
||||||
|
#
|
||||||
|
# WHY A ROOT UNIT (not a rootless quadlet): the UVA offload pins ~160 GiB of
|
||||||
|
# host RAM (PLE n-gram table + routed experts). Pinned pages are charged
|
||||||
|
# against RLIMIT_MEMLOCK; dank's login session on this host carries 8 MiB and
|
||||||
|
# a user quadlet is capped by the user manager's own hard limit — raising
|
||||||
|
# that means restarting user@1000.service and bouncing every user container
|
||||||
|
# on the host. A root unit grants LimitMEMLOCK=infinity to the dank process
|
||||||
|
# via PID1, no user-manager restart, and the image stays in dank's rootless
|
||||||
|
# podman storage (podman run as dank reads ~/.local/share/containers).
|
||||||
[Unit]
|
[Unit]
|
||||||
Description=unsloth llama.cpp (b11030-mix, ROCm gfx120X) Qwen3.8-Flash-Next Q8_0, shared-Q8_0 MTP + ngram-mod spec chain, BF16 vision, 262144 ctx / 1 slot, root systemd unit (dankserver, frank-primary, 2x Radeon AI PRO R9700)
|
Description=frank-primary — vLLM 0.30 Qwen3.8-Flash-Next-FP8 (UVA offload) on 2x R9700
|
||||||
After=network-online.target
|
After=network-online.target
|
||||||
Wants=network-online.target
|
Wants=network-online.target
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
Type=simple
|
|
||||||
User=dank
|
User=dank
|
||||||
Group=dank
|
Group=dank
|
||||||
# The model is mmap'd (175 GiB of expert pages + the 51B-param n-gram
|
# The UVA offloader pins the offloaded weights in host RAM; an RLIMIT_MEMLOCK
|
||||||
# table live in page cache); an RLIMIT_MEMLOCK smaller than what the
|
# smaller than the pinned set fails the load mid-start. PID1 grants infinity
|
||||||
# runtime pins would fail the load mid-start. Parity with the other LLM
|
# here — parity with the fleet's other LLM units.
|
||||||
# units on this fleet.
|
|
||||||
LimitMEMLOCK=infinity
|
LimitMEMLOCK=infinity
|
||||||
Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
Environment="FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env"
|
||||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh
|
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-primary.sh
|
||||||
|
# Cold start: Triton/inductor compile (several minutes on first boot) plus
|
||||||
|
# reading+pinning ~173 GiB of weights. Parity with the radiance-era timeout.
|
||||||
|
TimeoutStartSec=1800
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|
||||||
|
|||||||
+106
-123
@@ -1,140 +1,123 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# Serve unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 as frank-primary on
|
# serve-frank-primary.sh — frank-primary endpoint (vLLM 0.30,
|
||||||
# Dankserver's two Radeon AI PRO R9700s (gfx1201), using the unsloth
|
# Qwen3.8-Flash-Next-FP8 on the 2x R9700 pair, dankserver :7770).
|
||||||
# b11030-mix prebuilt (ROCm gfx120X, bundled runtime).
|
|
||||||
#
|
#
|
||||||
# Shape: one stream, the model's full 262144 context, all non-overridden
|
# Validates the image, the model payload and the env tunables, then execs the
|
||||||
# tensors on the GPUs (--n-gpu-layers 999 across both cards), with the
|
# container. The unit (scripts/frank-primary.service, root systemd, User=dank,
|
||||||
# routed experts and the 51B-param n-gram embedding table computed in host
|
# LimitMEMLOCK=infinity) is what makes the UVA pin budget legal — a plain
|
||||||
# RAM via --override-tensor (the x4-pair law: streaming over PCIe loses,
|
# shell here carries the 8 MiB session limit and the pinned offload fails.
|
||||||
# computing in RAM does not care). mmap stays ON (the PLE table must stay
|
|
||||||
# page-cache-backed; --no-mmap forces it into anonymous memory and the
|
|
||||||
# cgroup kill is silent).
|
|
||||||
#
|
#
|
||||||
# Spec chain: draft-mtp (the shared Q8_0 head, loaded via -md) chained with
|
# OFFLOAD SHAPE (the only legal fit for a ~125B-param FP8 model on 2x 32 GiB
|
||||||
# the draftless ngram-mod matcher. The shared head borrows the target's
|
# cards): --offload-backend uva pins the PLE n-gram table (51.2B params) and
|
||||||
# embedding tensors, which is why per_layer_token_embd.weight must be
|
# the routed experts in host RAM and reads them in place over PCIe. The
|
||||||
# CPU-resident (see frank-primary.env). depth 2.
|
# first start MUST be watched for the UVA engagement log (not just /health):
|
||||||
|
# a silent fallback to the copy path turns the endpoint into a slideshow.
|
||||||
#
|
#
|
||||||
# Vision: the BF16 projector, loaded explicitly via --mmproj (this build
|
# NO --model-draft, no draft GGUF: vLLM's MTP drafting comes from the model's
|
||||||
# never autoloads projectors — the frank-medium 2026-09-17 trap: a server
|
# own mtp.* weights via --speculative-config, and only when SPEC_CONFIG is
|
||||||
# started without it comes up healthy and text-only, and answers image
|
# non-empty. Bring-up runs with it empty on purpose.
|
||||||
# requests with 500).
|
|
||||||
#
|
|
||||||
# Template: the GGUF's embedded chat template via --jinja. The Qwen3.8
|
|
||||||
# family template gates reasoning_effort to {xhigh, medium, low} — the
|
|
||||||
# harness folds its levels onto that set in models.yml
|
|
||||||
# (reasoningEffortMap). If the embedded template ever falls back to
|
|
||||||
# peg-native and bloats prompts (~73,600 tokens for a ~250-token request),
|
|
||||||
# switch to --chat-template-file scripts/qwen-sharp-v22.4.0.jinja the way
|
|
||||||
# frank-large does.
|
|
||||||
#
|
|
||||||
# Tunables come from frank-primary.env unless FRANK_PRIMARY_ENV_FILE
|
|
||||||
# overrides it; anything not set there falls through to the defaults below.
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
||||||
if [[ -f "$ENV_FILE" ]]; then
|
|
||||||
set -a
|
|
||||||
source "$ENV_FILE"
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
|
|
||||||
# --- defaults (used when the env file does not set them) ------------------
|
if [ ! -f "$ENV_FILE" ]; then
|
||||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
echo "serve-frank-primary: env file missing: $ENV_FILE" >&2
|
||||||
: "${MODEL_ROOT:=/home/dank/models/Qwen3.8-Flash-Next-GGUF}"
|
|
||||||
: "${BIND_HOST:=0.0.0.0}"
|
|
||||||
: "${PORT:=7770}"
|
|
||||||
: "${CONTEXT_SIZE:=262144}"
|
|
||||||
: "${PARALLEL_SLOTS:=1}"
|
|
||||||
: "${SPEC_DRAFT_N_MAX:=2}"
|
|
||||||
: "${NGL:=999}"
|
|
||||||
: "${OT_OVERRIDES:=exps=CPU,per_layer_token_embd.weight=CPU}"
|
|
||||||
: "${TENSOR_SPLIT:=1,1}"
|
|
||||||
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
|
||||||
: "${BATCH_SIZE:=8192}"
|
|
||||||
: "${UBATCH_SIZE:=512}"
|
|
||||||
: "${CACHE_TYPE_V:=f16}"
|
|
||||||
: "${CACHE_TYPE_K:=f16}"
|
|
||||||
: "${VERBOSITY:=4}"
|
|
||||||
|
|
||||||
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
|
||||||
MODEL_PARTS=(
|
|
||||||
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf"
|
|
||||||
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00002-of-00006.gguf"
|
|
||||||
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00003-of-00006.gguf"
|
|
||||||
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00004-of-00006.gguf"
|
|
||||||
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00005-of-00006.gguf"
|
|
||||||
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00006-of-00006.gguf"
|
|
||||||
)
|
|
||||||
DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
|
||||||
MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf"
|
|
||||||
|
|
||||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
||||||
echo "runtime not found at $SERVER_BIN" >&2
|
|
||||||
echo " fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
# Every part is checked: part 1 is the -m target and parts 2-6 are followed
|
set -a
|
||||||
# by the split-file loader, so a missing tail part fails at LOAD time with
|
# shellcheck disable=SC1090
|
||||||
# a confusing partial-mmap error unless it is caught here.
|
source "$ENV_FILE"
|
||||||
for f in "${MODEL_PARTS[@]}" "$DRAFTER_FILE" "$MMPROJ_FILE"; do
|
set +a
|
||||||
if [[ ! -f "$f" ]]; then
|
|
||||||
echo "missing asset: $f" >&2
|
|
||||||
echo " run scripts/download-frank-primary.sh on this host" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# --- env (the unsloth build's bundled ROCm runtime) -----------------------
|
: "${SERVED_NAME:?SERVED_NAME must be set}"
|
||||||
# Both R9700s (gfx1201, HIP devices 0 and 1); the host's Raphael iGPU is
|
: "${BIND_HOST:?BIND_HOST must be set}"
|
||||||
# device 2 and must stay hidden from the enumeration.
|
: "${PORT:?PORT must be set}"
|
||||||
export HIP_VISIBLE_DEVICES=0,1
|
: "${MODEL_DIR:?MODEL_DIR must be set}"
|
||||||
export LD_LIBRARY_PATH="$RUNTIME_DIR:${LD_LIBRARY_PATH:-}"
|
: "${TP:?TP must be set}"
|
||||||
|
: "${GPU_UTIL:?GPU_UTIL must be set}"
|
||||||
|
: "${CTX_SIZE:?CTX_SIZE must be set}"
|
||||||
|
: "${MAX_SEQS:?MAX_SEQS must be set}"
|
||||||
|
: "${OFFLOAD_BACKEND:?OFFLOAD_BACKEND must be set}"
|
||||||
|
: "${OFFLOAD_GB:?OFFLOAD_GB must be set}"
|
||||||
|
: "${OFFLOAD_PARAMS:?OFFLOAD_PARAMS must be set}"
|
||||||
|
: "${VLLM_ROCM_USE_AITER:?VLLM_ROCM_USE_AITER must be set}"
|
||||||
|
: "${IMAGE:?IMAGE must be set}"
|
||||||
|
: "${CONTAINER_NAME:?CONTAINER_NAME must be set}"
|
||||||
|
: "${ROCR_VISIBLE_DEVICES:?ROCR_VISIBLE_DEVICES must be set}"
|
||||||
|
: "${TOOL_PARSER:?TOOL_PARSER must be set}"
|
||||||
|
: "${REASONING_PARSER:?REASONING_PARSER must be set}"
|
||||||
|
: "${CHAT_TEMPLATE_KWARGS:?CHAT_TEMPLATE_KWARGS must be set}"
|
||||||
|
: "${CACHE_ROOT:?CACHE_ROOT must be set}"
|
||||||
|
|
||||||
# --- the serve command ---------------------------------------------------
|
if [ "$OFFLOAD_BACKEND" != "uva" ]; then
|
||||||
ARGS=(
|
echo "serve-frank-primary: OFFLOAD_BACKEND=$OFFLOAD_BACKEND — this deployment is only qualified for uva" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! podman image inspect "$IMAGE" > /dev/null 2>&1; then
|
||||||
|
echo "serve-frank-primary: image missing: $IMAGE — build it before serving" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The model payload is verified by the fetch/verify step (131 shards, index
|
||||||
|
# cross-checked). The serve-time check is the load-bearing minimum: the
|
||||||
|
# wrapper config and the shard index must exist, or the container start is a
|
||||||
|
# guaranteed 20-minute timeout instead of a fast failure.
|
||||||
|
HOST_MODEL_DIR="/home/dank/models/${MODEL_DIR#/models/}"
|
||||||
|
if [ ! -f "$HOST_MODEL_DIR/config.json" ] || [ ! -f "$HOST_MODEL_DIR/model.safetensors.index.json" ]; then
|
||||||
|
echo "serve-frank-primary: model payload incomplete at $HOST_MODEL_DIR (config.json or index missing)" >&2
|
||||||
|
echo "serve-frank-primary: run scripts/download-frank-primary-fp8.sh on this host" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -e /dev/kfd ] || [ ! -d /dev/dri ]; then
|
||||||
|
echo "serve-frank-primary: /dev/kfd or /dev/dri missing — GPU stack not up" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p /home/dank/.cache/frank-primary-vllm
|
||||||
|
|
||||||
|
SERVE_ARGS=(serve "$MODEL_DIR"
|
||||||
|
--served-model-name "$SERVED_NAME"
|
||||||
--host "$BIND_HOST"
|
--host "$BIND_HOST"
|
||||||
--port "$PORT"
|
--port "$PORT"
|
||||||
--model "${MODEL_PARTS[0]}"
|
--tensor-parallel-size "$TP"
|
||||||
--alias frank-primary
|
--gpu-memory-utilization "$GPU_UTIL"
|
||||||
--ctx-size "$CONTEXT_SIZE"
|
--max-model-len "$CTX_SIZE"
|
||||||
--parallel "$PARALLEL_SLOTS"
|
--max-num-seqs "$MAX_SEQS"
|
||||||
-lv "$VERBOSITY"
|
--offload-backend "$OFFLOAD_BACKEND"
|
||||||
--metrics
|
--cpu-offload-gb "$OFFLOAD_GB"
|
||||||
--n-gpu-layers "$NGL"
|
--cpu-offload-params "$OFFLOAD_PARAMS"
|
||||||
--tensor-split "$TENSOR_SPLIT"
|
--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER"
|
||||||
--jinja
|
--reasoning-parser "$REASONING_PARSER"
|
||||||
--reasoning auto
|
--default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"
|
||||||
--reasoning-format deepseek
|
|
||||||
--reasoning-budget -1
|
|
||||||
--cache-type-k "$CACHE_TYPE_K"
|
|
||||||
--cache-type-v "$CACHE_TYPE_V"
|
|
||||||
--batch-size "$BATCH_SIZE"
|
|
||||||
--ubatch-size "$UBATCH_SIZE"
|
|
||||||
--cache-ram "$CACHE_RAM"
|
|
||||||
# Vision: explicit, never autoloaded by this build
|
|
||||||
--mmproj "$MMPROJ_FILE"
|
|
||||||
# Spec chain: the shared MTP head + the draftless ngram matcher
|
|
||||||
--spec-type draft-mtp,ngram-mod
|
|
||||||
--model-draft "$DRAFTER_FILE"
|
|
||||||
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
|
||||||
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
|
||||||
# Greedy server default: deterministic e2e, and the harness overrides
|
|
||||||
# sampling per-request (models.yml compat.extraBody carries Qwen's
|
|
||||||
# recommended temp 1.0 / top_p 0.95 / top_k 20).
|
|
||||||
--temp 0
|
|
||||||
--top-p 1.0
|
|
||||||
--top-k 0
|
|
||||||
--min-p 0.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Comma-separated <pattern>=<backend> pairs, ONE flag: b11030 logs
|
# SPEC_CONFIG (optional, JSON) is how MTP drafting is enabled after the
|
||||||
# DEPRECATED for repeated --override-tensor and wants the joined list.
|
# bring-up measurement: --speculative-config '{"method":"mtp",...}'.
|
||||||
# The loader applies the FIRST matching pattern per tensor, so the
|
if [ -n "${SPEC_CONFIG:-}" ]; then
|
||||||
# specific block patterns must precede the broad exps=CPU sweep in
|
SERVE_ARGS+=(--speculative-config "$SPEC_CONFIG")
|
||||||
# frank-primary.env.
|
fi
|
||||||
ARGS+=(--override-tensor "$OT_OVERRIDES")
|
|
||||||
|
|
||||||
exec "$SERVER_BIN" "${ARGS[@]}"
|
exec podman run --rm \
|
||||||
|
--name "$CONTAINER_NAME" \
|
||||||
|
--userns=keep-id \
|
||||||
|
--network=host \
|
||||||
|
--ipc=host \
|
||||||
|
--security-opt seccomp=unconfined \
|
||||||
|
--cap-add SYS_PTRACE \
|
||||||
|
--device /dev/kfd \
|
||||||
|
--device /dev/dri \
|
||||||
|
--group-add keep-groups \
|
||||||
|
-e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" \
|
||||||
|
-e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES" \
|
||||||
|
-e VLLM_ROCM_USE_AITER="$VLLM_ROCM_USE_AITER" \
|
||||||
|
-e TRITON_CACHE_DIR="$CACHE_ROOT/triton" \
|
||||||
|
-e TORCHINDUCTOR_CACHE_DIR="$CACHE_ROOT/inductor" \
|
||||||
|
-e VLLM_CACHE_DIR="$CACHE_ROOT/vllm" \
|
||||||
|
-e TRITON_ALLOW_NON_CONST_GLOBALS=1 \
|
||||||
|
--volume /home/dank/models:/models \
|
||||||
|
--volume /home/dank/.cache/frank-primary-vllm:"$CACHE_ROOT" \
|
||||||
|
"$IMAGE" \
|
||||||
|
vllm "${SERVE_ARGS[@]}"
|
||||||
+112
-116
@@ -1,16 +1,17 @@
|
|||||||
#!/usr/bin/env bats
|
#!/usr/bin/env bats
|
||||||
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
|
# Seam: frank-primary — the Dankserver LLM endpoint on :7770.
|
||||||
#
|
#
|
||||||
# This file asserts the llama.cpp ROOT-UNIT shape (the 2026-09-26 swap that
|
# This file asserts the vLLM 0.30 ROOT-UNIT shape (the 2026-09-27 swap that
|
||||||
# replaced the Lucebox DS4.1 Quadlet): the unsloth b11030-mix prebuilt on
|
# replaced the llama.cpp b11030 endpoint): a locally built ROCm gfx1201
|
||||||
# the two R9700s (gfx1201) serving unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0
|
# image (localhost/frank-primary:vllm030) serving Qwen3.8-Flash-Next-FP8
|
||||||
# with the shared Q8_0 MTP head, the BF16 projector, and the
|
# (Qwen4ExpForConditionalGeneration, 131 shards, 172.78 GiB) on the two
|
||||||
# draft-mtp + ngram-mod chain. The seam pins what the flags alone do not
|
# R9700s with the UVA offload shape. The seam pins what the flags alone do
|
||||||
# say: the CPU-override placement law (experts and the 51B n-gram table
|
# not say: the offload budget arithmetic (the bandwidth-huge tensors — the
|
||||||
# are COMPUTED in host RAM, never streamed), mmap staying on, the
|
# 47.75 GiB PLE family and the 114.86 GiB routed experts — are pinned in
|
||||||
# shared-head borrow requirement (per_layer_token_embd on CPU), the
|
# host RAM and read in place, never streamed), the memlock law that makes
|
||||||
# explicit projector (never autoloaded by this build), HIP env hygiene,
|
# the pin legal, the parser contract that keeps the harness wire formats
|
||||||
# and the deployer's byte-count gates over a six-part payload.
|
# (tool_calls + the reasoning channel) working, HIP env hygiene, and the
|
||||||
|
# deployer's image/payload gates with the false-green MainPID check.
|
||||||
|
|
||||||
load helpers
|
load helpers
|
||||||
|
|
||||||
@@ -27,8 +28,10 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|||||||
|
|
||||||
@test "frank-primary is boot-persistent, self-healing and unprivileged" {
|
@test "frank-primary is boot-persistent, self-healing and unprivileged" {
|
||||||
# Root unit parity with frank-large/frank-small: enable is the [Install]
|
# Root unit parity with frank-large/frank-small: enable is the [Install]
|
||||||
# section, the restart policy survives a crash, and the mmap'd payload
|
# section, the restart policy survives a crash, and the ~164 GiB UVA pin
|
||||||
# needs an unbounded memlock.
|
# needs an unbounded memlock (a root unit grants it via PID1; the dank
|
||||||
|
# session's own hard limit is 8 MiB and cannot be raised without
|
||||||
|
# restarting user@1000.service).
|
||||||
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
grep -q '^WantedBy=multi-user.target$' "$UNIT"
|
||||||
grep -q '^Restart=on-failure$' "$UNIT"
|
grep -q '^Restart=on-failure$' "$UNIT"
|
||||||
grep -q '^RestartSec=' "$UNIT"
|
grep -q '^RestartSec=' "$UNIT"
|
||||||
@@ -41,99 +44,96 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|||||||
grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT"
|
grep -q 'FRANK_PRIMARY_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-primary.env' "$UNIT"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "frank-primary drafts with the shared MTP head chained behind ngram-mod" {
|
@test "the serve shape is vLLM: no llama.cpp-ism survives the cutover" {
|
||||||
# The chain is draft-mtp (context-backed) + ngram-mod (draftless): the
|
# Clean cutover: the GGUF era's flags would silently no-op or crash a vLLM
|
||||||
# only legal way to layer two spec implementations in one process. The
|
# server, so their absence is part of the contract.
|
||||||
# shared Q8_0 head borrows the target's embedding tensors — which is
|
! grep -q -- '--model-draft' "$SERVE"
|
||||||
# why per_layer_token_embd.weight must be CPU-resident (a GPU-resident
|
! grep -q -- '--spec-type' "$SERVE"
|
||||||
# borrow crashes the draft load with "invalid vector subscript").
|
! grep -q -- '--mmproj' "$SERVE"
|
||||||
grep -q -- '--spec-type draft-mtp,ngram-mod' "$SERVE"
|
! grep -q -- '--override-tensor' "$SERVE"
|
||||||
grep -q '^SPEC_DRAFT_N_MAX=2$' "$ENV_FILE"
|
! grep -q '\.gguf' "$SERVE"
|
||||||
grep -q -- '--spec-draft-n-max "$SPEC_DRAFT_N_MAX"' "$SERVE"
|
! grep -q '\.gguf' "$ENV_FILE"
|
||||||
grep -q 'DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"' "$SERVE"
|
|
||||||
grep -q -- '--spec-draft-device "$SPEC_DRAFT_DEVICE"' "$SERVE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the bandwidth-huge tensors are computed in host RAM, not streamed" {
|
@test "the bandwidth-huge tensors are pinned in host RAM, not streamed" {
|
||||||
# The x4-pair law: streaming experts over PCIe loses (3.7 tok/s) while
|
# The UVA shape (the x4-pair law: streaming experts over PCIe loses,
|
||||||
# computing them in RAM does not care (20+ tok/s on the same model
|
# computing them in host RAM does not care): --offload-backend uva pins
|
||||||
# shape). exps=CPU plus the n-gram embedding table on CPU; the PLE
|
# the matched weights with zero-copy reads. Measured from the payload
|
||||||
# table must NEVER carry a GPU target (the 23 -> 13.4 t/s regression).
|
# headers (2026-09-26): `ple` matches 138 tensors (131 n-gram table
|
||||||
# ONE joined flag (b11030 deprecates repeated --override-tensor), and
|
# shards + 7 per-layer conv/proj weights) = 47.75 GiB; `experts` matches
|
||||||
# first-match-wins means the block-specific patterns must precede the
|
# 150,528 tensors = 114.86 GiB; combined 162.60 GiB. cpu-offload-gb is
|
||||||
# broad exps=CPU sweep — the exact value pins the whole law.
|
# PER GPU: under TP=2 each rank needs ~81.3 GiB, hence 82 — 80/card
|
||||||
grep -q -- '--override-tensor "$OT_OVERRIDES"' "$SERVE"
|
# would strand 2.6 GiB of experts on the cards.
|
||||||
! grep -q -- '--no-mmap' "$SERVE"
|
grep -q '^OFFLOAD_BACKEND=uva$' "$ENV_FILE"
|
||||||
! grep -q '^MMPROJ_ON_GPU' "$ENV_FILE"
|
grep -q '^OFFLOAD_GB=82$' "$ENV_FILE"
|
||||||
ot="$(sed -nE "s/^OT_OVERRIDES='(.*)'$/\1/p" "$ENV_FILE")"
|
grep -q '^OFFLOAD_PARAMS=ple,experts$' "$ENV_FILE"
|
||||||
[ "$ot" = 'blk\.[0-4]\.ffn_(up|gate|down)_exps=ROCm0,blk\.4[5-7]\.ffn_(up|gate|down)_exps=ROCm1,exps=CPU,per_layer_token_embd.weight=CPU' ]
|
grep -q -- '--offload-backend "$OFFLOAD_BACKEND"' "$SERVE"
|
||||||
|
grep -q -- '--cpu-offload-gb "$OFFLOAD_GB"' "$SERVE"
|
||||||
|
grep -q -- '--cpu-offload-params "$OFFLOAD_PARAMS"' "$SERVE"
|
||||||
|
# auto/prefetch are the copy paths: this deployment is only qualified
|
||||||
|
# for uva, so the serve script refuses anything else.
|
||||||
|
grep -q 'only qualified for uva' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the KV pool is V-quantized, K held at f16" {
|
@test "the engine shape: TP=2 over both R9700s, full native window, batch 8" {
|
||||||
# V q8_0 halves nothing (it is 1/4 of the per-token law: 18,816 of the
|
grep -q '^TP=2$' "$ENV_FILE"
|
||||||
# 24,576 B/token f16/f16 cost) but it is the deploy's stated shape; the
|
grep -q '^GPU_UTIL=0.97$' "$ENV_FILE"
|
||||||
# flags must be env-wired tunables so the quant can be reverted
|
grep -q '^CTX_SIZE=262144$' "$ENV_FILE"
|
||||||
# without touching the serve script.
|
grep -q '^MAX_SEQS=8$' "$ENV_FILE"
|
||||||
grep -q '^CACHE_TYPE_K=f16$' "$ENV_FILE"
|
grep -q -- '--tensor-parallel-size "$TP"' "$SERVE"
|
||||||
grep -q '^CACHE_TYPE_V=q8_0$' "$ENV_FILE"
|
grep -q -- '--gpu-memory-utilization "$GPU_UTIL"' "$SERVE"
|
||||||
grep -q -- '--cache-type-k "$CACHE_TYPE_K"' "$SERVE"
|
grep -q -- '--max-model-len "$CTX_SIZE"' "$SERVE"
|
||||||
grep -q -- '--cache-type-v "$CACHE_TYPE_V"' "$SERVE"
|
grep -q -- '--max-num-seqs "$MAX_SEQS"' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the projector is explicit, never autoloaded" {
|
@test "the payload is the FP8 tree, vision served from the model's own config" {
|
||||||
# This build never autoloads mmproj from the model directory: without
|
# Qwen4ExpForConditionalGeneration: vLLM auto-loads the ViT and the
|
||||||
# the flag the server comes up healthy and text-only, answering image
|
# mtp.* weights from the payload config — no explicit projector flag
|
||||||
# requests with 500 (the frank-medium 2026-09-17 trap).
|
# exists in this runtime (the llama.cpp --mmproj trap does not apply).
|
||||||
grep -q -- '--mmproj "$MMPROJ_FILE"' "$SERVE"
|
grep -q '^MODEL_DIR=/models/Qwen3.8-Flash-Next-FP8$' "$ENV_FILE"
|
||||||
grep -q 'MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf"' "$SERVE"
|
grep -q '^IMAGE=localhost/frank-primary:vllm030$' "$ENV_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "frank-primary is vision-serving but holds the single fleet port" {
|
@test "frank-primary holds the single fleet port under its own alias" {
|
||||||
grep -q '^PORT=7770$' "$ENV_FILE"
|
grep -q '^PORT=7770$' "$ENV_FILE"
|
||||||
grep -q '^BIND_HOST=0.0.0.0$' "$ENV_FILE"
|
grep -q '^BIND_HOST=0.0.0.0$' "$ENV_FILE"
|
||||||
grep -q -- '--alias frank-primary' "$SERVE"
|
grep -q '^SERVED_NAME=frank-primary$' "$ENV_FILE"
|
||||||
|
grep -q -- '--served-model-name "$SERVED_NAME"' "$SERVE"
|
||||||
grep -q -- '--host "$BIND_HOST"' "$SERVE"
|
grep -q -- '--host "$BIND_HOST"' "$SERVE"
|
||||||
grep -q -- '--port "$PORT"' "$SERVE"
|
grep -q -- '--port "$PORT"' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "four streams, each the model's full native window" {
|
@test "tool_calls and the reasoning channel are wired for the harness" {
|
||||||
# --parallel divides --ctx-size into slots; the contract is four
|
# The wire contract: vLLM 0.30 emits the thinking channel as `reasoning`
|
||||||
# conversations each holding the FULL native 262144 window, so the
|
# (the deprecated `reasoning_content` is renamed server-side), the qwen3
|
||||||
# per-slot window (ctx / parallel) equals the native size.
|
# parser extracts the template's think block into it, and qwen3_xml +
|
||||||
local ctx parallel
|
# auto-tool-choice structure tool calls. Precedent: the host's own
|
||||||
ctx="$(sed -nE 's/^CONTEXT_SIZE=([0-9]+)$/\1/p' "$ENV_FILE")"
|
# Ornith-era scripts (~/vllm/failedAttempts/OrnithFP8.sh); the FP8
|
||||||
parallel="$(sed -nE 's/^PARALLEL_SLOTS=([0-9]+)$/\1/p' "$ENV_FILE")"
|
# template consumes enable_thinking/preserve_thinking (verified in
|
||||||
[ -n "$ctx" ] && [ -n "$parallel" ]
|
# tokenizer_config.json).
|
||||||
[ "$parallel" -eq 4 ]
|
grep -q '^TOOL_PARSER=qwen3_xml$' "$ENV_FILE"
|
||||||
[ "$((ctx / parallel))" -eq 262144 ]
|
grep -q '^REASONING_PARSER=qwen3$' "$ENV_FILE"
|
||||||
grep -q -- '--ctx-size "$CONTEXT_SIZE"' "$SERVE"
|
grep -q -- '--enable-auto-tool-choice --tool-call-parser "$TOOL_PARSER"' "$SERVE"
|
||||||
grep -q -- '--parallel "$PARALLEL_SLOTS"' "$SERVE"
|
grep -q -- '--reasoning-parser "$REASONING_PARSER"' "$SERVE"
|
||||||
|
# The template kwargs are JSON: the env value must be single-quoted or
|
||||||
|
# bash splits it and CHAT_TEMPLATE_KWARGS arrives empty (the -ot lesson).
|
||||||
|
grep -q "^CHAT_TEMPLATE_KWARGS='{\"enable_thinking\": true, \"preserve_thinking\": true}'$" "$ENV_FILE"
|
||||||
|
grep -q -- '--default-chat-template-kwargs "$CHAT_TEMPLATE_KWARGS"' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "reasoning is extracted to reasoning_content and tools are legal" {
|
@test "speculative decode is env-wired and OFF at bring-up" {
|
||||||
# The harness contract: llama.cpp endpoints emit reasoning_content
|
# MTP drafting comes from the model's own mtp.* weights via
|
||||||
# (the deepseek format is the one that fills it), and tools require
|
# --speculative-config; bring-up measures the base shape clean, so
|
||||||
# --jinja (the ik_llama 500 lesson). The Qwen3.8 template gates
|
# SPEC_CONFIG is empty and the flag is only added when non-empty.
|
||||||
# reasoning_effort to {xhigh, medium, low} — folded in models.yml.
|
grep -q '^SPEC_CONFIG=$' "$ENV_FILE"
|
||||||
grep -q -- '--jinja' "$SERVE"
|
grep -q -- '--speculative-config "$SPEC_CONFIG"' "$SERVE"
|
||||||
grep -q -- '--reasoning auto' "$SERVE"
|
|
||||||
grep -q -- '--reasoning-format deepseek' "$SERVE"
|
|
||||||
grep -q -- '--reasoning-budget -1' "$SERVE"
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the greedy server default is explicit" {
|
|
||||||
# Deterministic e2e probes; the harness overrides sampling per request.
|
|
||||||
grep -q -- '--temp 0' "$SERVE"
|
|
||||||
grep -q -- '--top-p 1.0' "$SERVE"
|
|
||||||
grep -q -- '--top-k 0' "$SERVE"
|
|
||||||
grep -q -- '--min-p 0.0' "$SERVE"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "both R9700s are visible and the Raphael iGPU is not" {
|
@test "both R9700s are visible and the Raphael iGPU is not" {
|
||||||
# The host has a third (Raphael) GPU the server must never place
|
grep -q '^ROCR_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
|
||||||
# tensors on; the bundled ROCm runtime is the only one linked.
|
grep -q '^HIP_VISIBLE_DEVICES=0,1$' "$ENV_FILE"
|
||||||
grep -q 'export HIP_VISIBLE_DEVICES=0,1$' "$SERVE"
|
grep -q -- '-e ROCR_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"' "$SERVE"
|
||||||
grep -q 'export LD_LIBRARY_PATH="$RUNTIME_DIR' "$SERVE"
|
grep -q -- '-e HIP_VISIBLE_DEVICES="$HIP_VISIBLE_DEVICES"' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "no bare HIP-reserved env var is defined" {
|
@test "no bare HIP-reserved env var is defined" {
|
||||||
@@ -145,40 +145,36 @@ REMOTE_DEPLOY="$REPO_ROOT/scripts/deploy-remote.sh"
|
|||||||
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
! grep -qE '^(export )?(TEMP|HOST|GPU_MAX_HEAP_SIZE)=' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the serve script refuses to start on a half-finished download" {
|
@test "the serve script refuses to start on a half-finished payload or missing image" {
|
||||||
# Six split parts plus the head and the projector: a missing tail part
|
# A payload missing its wrapper config or shard index turns the
|
||||||
# fails at LOAD time with a confusing partial-mmap error, so every
|
# container start into a guaranteed 20-minute timeout; the checks fail
|
||||||
# artifact is checked before the server execs.
|
# fast with a pointer at the fetch script instead.
|
||||||
grep -q 'missing asset' "$SERVE"
|
grep -q 'config.json or index missing' "$SERVE"
|
||||||
grep -q 'download-frank-primary.sh' "$SERVE"
|
grep -q 'download-frank-primary-fp8.sh' "$SERVE"
|
||||||
|
grep -q 'podman image inspect' "$SERVE"
|
||||||
}
|
}
|
||||||
|
|
||||||
@test "the prompt-cache cap is set below the RAM the host can spare" {
|
@test "the deployer gates the image and the full shard set before starting" {
|
||||||
grep -q '^CACHE_RAM=16384$' "$ENV_FILE"
|
# The deployer refuses when the image or any index-named shard is
|
||||||
grep -q -- '--cache-ram "$CACHE_RAM"' "$SERVE"
|
# absent (never serve a partial download), clears any stale container
|
||||||
}
|
# that would break `podman run --name`, and waits out the port before
|
||||||
|
# the new engine binds it.
|
||||||
@test "the deployer retires the lucebox quadlet, gates every part and waits out the port" {
|
grep -q 'LLM_PRIMARY_IMAGE=localhost/frank-primary:vllm030' "$REMOTE_DEPLOY"
|
||||||
# The cutover order: the user-scope quadlet is stopped and its
|
grep -q 'LLM_PRIMARY_ROOT=/home/dank/models/Qwen3.8-Flash-Next-FP8' "$REMOTE_DEPLOY"
|
||||||
# generated .container moved to dated home trash BEFORE the root unit
|
grep -q 'weight_map' "$REMOTE_DEPLOY"
|
||||||
# binds 7770 (nothing starts until the port is released — the 2026-09-26
|
grep -q 'download-frank-primary-fp8.sh' "$REMOTE_DEPLOY"
|
||||||
# and 2026-09-16 cutover failures were both here).
|
grep -q "podman rm -f \"\$LLM_PRIMARY_CONTAINER\"" "$REMOTE_DEPLOY"
|
||||||
grep -q 'retire_lucebox_primary_quadlet' "$REMOTE_DEPLOY"
|
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
||||||
grep -q 'systemctl --user stop frank-primary.service' "$REMOTE_DEPLOY"
|
|
||||||
grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
|
grep -q 'relink_unit frank-primary' "$REMOTE_DEPLOY"
|
||||||
grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
|
grep -q 'systemctl enable frank-primary.service' "$REMOTE_DEPLOY"
|
||||||
grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
|
grep -q 'systemctl restart frank-primary.service' "$REMOTE_DEPLOY"
|
||||||
grep -q 'wait_port_free 7770' "$REMOTE_DEPLOY"
|
|
||||||
grep -q '127.0.0.1:7770/health' "$REMOTE_DEPLOY"
|
|
||||||
}
|
|
||||||
|
|
||||||
@test "the deployer gates the six-part payload at declared byte counts" {
|
|
||||||
# stat -c %s, never wc -c — wc read the whole 377 GiB lucebox file for
|
|
||||||
# minutes on 2026-09-26 before it could reach the cutover. A partial
|
|
||||||
# download must never be served.
|
|
||||||
grep -q 'LLM_PRIMARY_PART_SIZES' "$REMOTE_DEPLOY"
|
|
||||||
grep -q 'stat -c %s' "$REMOTE_DEPLOY"
|
|
||||||
grep -q '2786568256' "$REMOTE_DEPLOY"
|
|
||||||
grep -q '907542944' "$REMOTE_DEPLOY"
|
|
||||||
grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY"
|
grep -q 'wait_for_health frank-primary http://127.0.0.1:7770/health 1800' "$REMOTE_DEPLOY"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@test "the deployer will not report green on a stray port-holder" {
|
||||||
|
# The 2026-09-26 false green: a leftover probe held :7770 and satisfied
|
||||||
|
# the health curl while the real unit was stopped. The deploy is only
|
||||||
|
# green when the unit's own MainPID is live behind the health answer.
|
||||||
|
grep -q 'systemctl show -p MainPID --value frank-primary.service' "$REMOTE_DEPLOY"
|
||||||
|
grep -q 'stray process holds the port' "$REMOTE_DEPLOY"
|
||||||
|
}
|
||||||
Reference in new issue
Block a user