frank-large: cutover to vLLM TheRock gfx1151, MiMo-V2.6 BF16 + cross-model MTP
Engine: digest-pinned docker.io/kyuz0/vllm-therock-gfx1151 (vLLM 0.30, TheRock ROCm 10, Triton GDN kernels); the serve script gates the image by REPO DIGEST, the payload (config + index + exact shard count), the spec draft (BOTH weight layouts: single model.safetensors for dflash, sharded index for the Qwen3.5-9B MTP head), and whole /dev/dri, sweeps the bench identity vllm-mimo, then exec podman wait so a container death trips Restart=always. Shape: 4 x 262144 at util 0.45 (KV grows only on the 8 full-attention layers), MTP 2 draft tokens. Measured 2026-10-01 window (frank-large stopped): base 13.1 t/s decode (~94% of the LPDDR5x BF16 roof), DFlash-8 17.4, MTP-2 19.8 with 71%/44% per-position acceptance - the cross-model head works because vLLM reads the draft from a separate path while MiMo ships zero mtp.* tensors. Spec arms do not preserve greedy bf16 text (batched-verify reduction order). The seam was rewritten around the new laws (18 tests): unit self-healing, digest gating, engine wiring, the measured spec table, the retired gufo gates, the deployer MainPID check. frank_small port test follows the PORT -> FRANK_LARGE_PUBLISH rename. Rollback: tag frank-large-gufo-q4kxl-20261001.
This commit is contained in:
Frank committed
1 parent
b3313340fa
commit
dab483c79f
6 files changed
+487
-516
No files matched your search
+75
-52
@@ -1077,27 +1077,20 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built
|
||||
# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter
|
||||
# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a
|
||||
# ROOT unit: the runtime is a native binary linked against the host ROCm,
|
||||
# so there is no container image to install and no generated unit to dodge.
|
||||
# relink_unit points /etc/systemd/system/frank-large.service at the repo
|
||||
# file and boot persistence is that unit's [Install] section.
|
||||
#
|
||||
# The runtime gate matters as much as the payload gate: gufo is built from
|
||||
# source on this host (there is no prebuilt release), so a machine whose
|
||||
# /home/dank/GitPackages tree was never built has a unit that can never
|
||||
# become healthy, and a green deploy would leave the endpoint dark.
|
||||
#
|
||||
# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had
|
||||
# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the
|
||||
# same as every cutover since 2026-09-16: nothing starts until 7770 is
|
||||
# released.
|
||||
LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}"
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
# frank-large is vLLM (the digest-pinned kyuz0 TheRock gfx1151 image) serving
|
||||
# the BF16 MiMo-V2.6-Distill-Qwen-9B payload with cross-model MTP spec decode
|
||||
# (Qwen/Qwen3.5-9B head, 2 draft tokens) on the Strix Halo iGPU. 2026-10-01
|
||||
# cutover from gufo; the gufo-era gates (built-from-source binary, GGUF
|
||||
# target + DFlash2 file) are retired with it — rollback is git tag
|
||||
# frank-large-gufo-q4kxl-20261001, and the gufo runtime and payloads stay on
|
||||
# disk. The service drives `podman run` directly from its serve script, so
|
||||
# the gates here mirror deploy_llm_primary's: image present by REPO DIGEST in
|
||||
# dank's rootless storage, payload complete (config.json + index + 4 shards),
|
||||
# and the spec draft present whenever FRANK_LARGE_SPEC names one. The port
|
||||
# gate is the same as every cutover since 2026-09-16: nothing starts until
|
||||
# 7770 is released.
|
||||
LLM_LARGE_CONTAINER=frank-large-mimo
|
||||
LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1130,51 +1123,81 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the deploy log to say why.
|
||||
if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then
|
||||
echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
# endpoint dark. Gates mirror deploy_llm_primary.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
local image model_dir shards count spec spec_dir digests
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-large env file absent: $env_file" >&2
|
||||
return 1
|
||||
fi
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2
|
||||
missing=1
|
||||
image="$(sed -n 's/^FRANK_LARGE_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_LARGE_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
shards="$(sed -n 's/^FRANK_LARGE_MODEL_SHARDS=//p' "$env_file" | tail -1)"
|
||||
spec="$(sed -n 's/^FRANK_LARGE_SPEC=//p' "$env_file" | tail -1)"
|
||||
spec_dir="$(sed -n 's/^FRANK_LARGE_SPEC_DIR=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then
|
||||
echo "ERROR: frank-large.env missing FRANK_LARGE_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2
|
||||
return 1
|
||||
fi
|
||||
# Capture podman's output BEFORE grepping (the SIGPIPE race, 2026-09-30).
|
||||
digests="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${image#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-large: fetch it (podman pull $image)" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/config.json" || ! -f "$model_dir/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: payload incomplete: $model_dir (config.json / weight-map index)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)"
|
||||
if [[ "$count" != "$shards" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "$spec" ]]; then
|
||||
if [[ -z "$spec_dir" || ! -f "$spec_dir/model.safetensors" ]]; then
|
||||
echo "ERROR: spec draft absent for FRANK_LARGE_SPEC=$spec: $spec_dir/model.safetensors" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
fi
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
||||
# Then sweep a stray manual run -- that is how a bench harness leaves the
|
||||
# port busy. Match the DRAFTER path, which every frank-large command
|
||||
# line carries via --dflash-model and which is UNCHANGED by the 2026-09-28
|
||||
# payload swap: the target moved to a different repository while the
|
||||
# drafter stayed put, so keying on the target filename would have quietly
|
||||
# stopped matching. It is also the right key in principle — it names this
|
||||
# endpoint's speculative drafter, not a binary. Match that, not a binary
|
||||
# name: frank-small on this host serves a different GGUF over llama-server
|
||||
# with no DFlash2 drafter and must not be killed. The pattern never
|
||||
# appears in the deploying shell's own command line, so -f is safe here
|
||||
# (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
# later and reported "Address already in use". Gate on the port, not on
|
||||
# hope.
|
||||
# A stopped unit can leave its container running (the cgroup-kill vs
|
||||
# conmon --rm race); the orphan holds the port and the GPU. The
|
||||
# bench-window identity (vllm-mimo) dies here too — that is how a bench
|
||||
# harness leaves the port busy.
|
||||
local ctr
|
||||
for ctr in $LLM_LARGE_LEGACY_CONTAINERS "$LLM_LARGE_CONTAINER"; do
|
||||
podman rm -f "$ctr" >/dev/null 2>&1 || true
|
||||
done
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16 and 2026-09-26 cutovers both failed exactly here).
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
# Warm boot is ~200-240 s (measured 2026-10-01); cold torch compile is
|
||||
# longer. Parity with frank-primary's 1800 s allowance.
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1800 || return 1
|
||||
# False-green hardening: the port answering proves nothing about WHO
|
||||
# answers — the unit must be active with a live MainPID.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-large.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-large.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-large: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090,
|
||||
|
||||
+92
-58
@@ -1,63 +1,97 @@
|
||||
# frank-large — gufo (gufo-org/gufo) serving
|
||||
# Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) on the Strix Halo iGPU.
|
||||
# frank-large.env — vLLM production config for frank-large (Dankclaw iGPU :7770).
|
||||
#
|
||||
# These are the tunables read by scripts/serve-frank-large.sh. The flag
|
||||
# list is owned by the serve script; this file only sets fleet values.
|
||||
# 2026-10-01 cutover from gufo (git tag frank-large-gufo-q4kxl-20261001).
|
||||
# Target: XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B in FULL BF16 precision
|
||||
# (17.53 GiB, 4 shards) — the same architecture frank-small serves as an
|
||||
# INT8 W8A16 AutoRound conversion on the 3090, here unquantized. The engine
|
||||
# is the kyuz0 TheRock vLLM build, the only fleet-carried runtime path for
|
||||
# gfx1151 (official vLLM images are CUDA-only).
|
||||
#
|
||||
# DO NOT name a new tunable TEMP, HOST or GPU_MAX_HEAP_SIZE: the HIP
|
||||
# runtime reads those bare (the 2026-08-13 TEMP incident). HOST is the one
|
||||
# exception because the serve script itself consumes it for --host.
|
||||
|
||||
# The target payload (scripts/download-frank-large-q4.sh): the serve script
|
||||
# refuses to start without the main GGUF and the DFlash2 drafter.
|
||||
# MEASURED 2026-10-01 bench window on this host (identical client, one
|
||||
# session, temperature 0, 28,647-token prefill / 512-token decode):
|
||||
#
|
||||
# 2026-10-01 swap (Frank): unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL, 16.35 GiB,
|
||||
# replacing ajgazin's abliterated Swift-1.5 UD-Q8_K_XL (29.3 GiB). Two
|
||||
# consequences worth knowing at a glance:
|
||||
# * this is the VANILLA (non-abliterated) Qwen3.8-27B — the endpoint is no
|
||||
# longer uncensored, which was one of the reasons the Swift payload was
|
||||
# chosen on 2026-09-28;
|
||||
# * ~13 GiB of weights became available for KV, which is what makes four
|
||||
# full-sized f16 sessions fit.
|
||||
# gufo needs no template for either: its Qwen renderer is compiled in.
|
||||
MODEL_DIR=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL
|
||||
MODEL_FILE=/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf
|
||||
|
||||
# The DFlash2 drafter is UNCHANGED by the model swap and still lives beside
|
||||
# the previous target, so it carries its own directory. It has a sliding
|
||||
# window of 2048, which is what keeps its KV small (~40 MiB).
|
||||
DRAFTER_DIR=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL
|
||||
DRAFTER_FILE=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf
|
||||
|
||||
# The gufo runtime, built from source by scripts/build-gufo.sh.
|
||||
RUNTIME_DIR=/home/dank/GitPackages/gufo-v0.1.1
|
||||
|
||||
# Fleet endpoint: the same port and alias every frank-large has served, so
|
||||
# no client, probe or models.yml entry changes address.
|
||||
ALIAS=frank-large
|
||||
HOST=0.0.0.0
|
||||
PORT=7770
|
||||
|
||||
# Four sessions, 163840 context each (Frank, 2026-10-01: "4x 160k"). The
|
||||
# context is per session and the KV is UNQUANTIZED (f16) — gufo has no KV
|
||||
# quantization lever at all (execution_policy.hpp pins kv_cache_storage to
|
||||
# kFp16), so "unquantized" is inherent rather than configured.
|
||||
# arm prefill decode acceptance (per position)
|
||||
# BF16 base 17,876 t/s 13.1 t/s —
|
||||
# BF16 + DFlash 8 tok 7,976 t/s 17.4 t/s pos0 59% pos1 26%; 14.8% token rate
|
||||
# BF16 + MTP 2 tok 7,380 t/s 19.8 t/s pos0 71% pos1 44%; 57.4% token rate
|
||||
#
|
||||
# The arithmetic that makes this fit: f16 KV costs 64 KiB/token on this model
|
||||
# (16 full-attention layers x 4 KV heads x 256 head_dim x 2 (K,V) x 2 bytes;
|
||||
# the other 48 layers are linear-attention with a per-SEQUENCE fp32 state of
|
||||
# ~144 MiB, not a per-token cache). 4 x 163840 = 655,360 tokens = 40.0 GiB of
|
||||
# KV, and each session's 163,840 tokens is exactly 10 GiB. Budget: weights
|
||||
# 16.35 + KV 40.0 + drafter 1.07 + linear state 0.56 + runtime ~4 = ~62 GiB
|
||||
# against the 114 GiB GTT — roughly 50 GiB of headroom, so the session count
|
||||
# or the per-session context can both grow from here.
|
||||
CONTEXT_SIZE=163840
|
||||
SESSIONS=4
|
||||
# Decode laws: 13.1 t/s is ~94% of the Strix Halo LPDDR5x roof for BF16
|
||||
# weights (17.5 GiB/token against ~256 GB/s) — a quant, not a bug; the same
|
||||
# INT8 payload decodes 83.9 t/s on the 3090 because it moves ~12 GiB
|
||||
# through ~936 GB/s. Speculative arms are the ONLY way past the wall, and
|
||||
# both drafters pay ~55-59% of prefill (draft passes share the compute
|
||||
# queues). Frank's call enabled MTP: the thinking-heavy traffic on this
|
||||
# endpoint spends its wall clock in decode.
|
||||
#
|
||||
# MTP IS CROSS-MODEL: the head comes from Qwen/Qwen3.5-9B (the distill's
|
||||
# source model, 15 mtp.* tensors, identical 4096-hidden/248320-vocab
|
||||
# geometry) — MiMo itself ships ZERO mtp tensors, which is why frank-small's
|
||||
# SGLang NEXTN is a 0.00-acceptance ghost (SGLang reads the head from the
|
||||
# TARGET checkpoint; vLLM reads the draft from a separate path). The 71%/44%
|
||||
# per-position acceptance is the distillation preserving Qwen3.5 hidden-state
|
||||
# structure; it is measured, not assumed.
|
||||
#
|
||||
# NUMERICS NOTE (measured): with a spec drafter enabled, temperature-0 text
|
||||
# DIFFERS from the no-spec run (greedy sha 1318aeb6 base vs 0208b45d dflash
|
||||
# vs 20c41c38 mtp): batched verification changes the bf16 reduction order,
|
||||
# so argmax near-ties diverge and cascade. Spec decode on vLLM is
|
||||
# distribution-preserving, not bitwise-greedy-preserving. The gufo
|
||||
# greedy==AR law does not carry over.
|
||||
#
|
||||
# HIP reads bare env vars TEMP, HOST, GPU_MAX_HEAP_SIZE — every key here is
|
||||
# FRANK_LARGE_-prefixed so none can leak.
|
||||
|
||||
# DFlash2 spec decode (a block-diffusion drafter). gufo sizes the draft
|
||||
# block itself; DRAFT_TOKENS caps the proposals per step (gufo's default is
|
||||
# 7, chosen over llama.cpp's fixed depth 4 because gufo's adaptive policy
|
||||
# shortens the block when acceptance drops) and DRAFT_POLICY is the block
|
||||
# length controller.
|
||||
DRAFT_TOKENS=7
|
||||
DRAFT_POLICY=adaptive
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
# Digest-pinned (the :latest tag of this fork moves with no notice):
|
||||
# image ID 2d24a8c5a3fd, vLLM 0.30.0 / torch 2.11.0 / ROCm 10.0.0 (TheRock),
|
||||
# verified to ship vllm/v1/spec_decode/dflash.py, the qwen3_5 model registry
|
||||
# entry (Qwen3_5ForConditionalGeneration), and the Triton GDN backend.
|
||||
FRANK_LARGE_IMAGE=docker.io/kyuz0/vllm-therock-gfx1151@sha256:98c1f3ac9833e7fa45e9a82fb7a4315e6d95daaf2548a211c8da3a871b17533a
|
||||
|
||||
# --- payload ----------------------------------------------------------------
|
||||
FRANK_LARGE_MODEL_DIR=/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-BF16
|
||||
FRANK_LARGE_MODEL_SHARDS=4
|
||||
|
||||
# --- identity ---------------------------------------------------------------
|
||||
# Same port and served alias as every frank-large predecessor; the harness
|
||||
# models.yml id is unchanged. The bench-window container (vllm-mimo) is
|
||||
# swept before every start.
|
||||
FRANK_LARGE_CONTAINER=frank-large-mimo
|
||||
FRANK_LARGE_LEGACY_CONTAINERS="vllm-mimo"
|
||||
FRANK_LARGE_SERVED_NAME=frank-large
|
||||
FRANK_LARGE_PUBLISH=7770:7770
|
||||
|
||||
# --- engine shape (2026-10-01 window) ----------------------------------------
|
||||
# Full native window 262,144 at UTIL 0.45 booted clean on gfx1151 in all
|
||||
# three arms (~200-240 s warm start). KV grows only on the 8 full-attention
|
||||
# layers (32 KiB/token bf16: 8 x 2 x 4 heads x 256 dim x 2 B); the 24 GDN
|
||||
# layers keep a FIXED ~25 MiB state per running sequence, so the pool is
|
||||
# generous and sessions are cheap. SEQS 4 preserves the four-session
|
||||
# contract the gufo shape ran; vLLM shares the pool across them by
|
||||
# scheduler, unlike gufo's per-session carve.
|
||||
FRANK_LARGE_MAXLEN=262144
|
||||
FRANK_LARGE_SEQS=4
|
||||
FRANK_LARGE_UTIL=0.45
|
||||
|
||||
# --- speculative decode: cross-model MTP (measured +51% decode) ---------------
|
||||
# mtp beats dflash here (19.8 vs 17.4 t/s) and needs 2 draft tokens, not 8:
|
||||
# acceptance past position 1 is 44% and falling (DFlash's own curve decays
|
||||
# 59/26/14/7). num_speculative_tokens beyond 2 is UNMEASURED — do not raise
|
||||
# it without a fresh A/B. To flip drafters, set FRANK_LARGE_SPEC=dflash and
|
||||
# repoint SPEC_DIR (the DFlash drafter stays on disk).
|
||||
# The gufo-era law "spec decode is absent" was a frank-primary finding; this
|
||||
# endpoint's traffic is decode-dominated thinking, which is the regime where
|
||||
# spec wins (prefill costs ~55% in every arm).
|
||||
FRANK_LARGE_SPEC=mtp
|
||||
FRANK_LARGE_SPEC_DIR=/home/dank/models/Qwen3.5-9B
|
||||
FRANK_LARGE_SPEC_TOKENS=2
|
||||
|
||||
# --- client contract ----------------------------------------------------------
|
||||
# vLLM emits the thinking text in `reasoning` (NOT llama.cpp/gufo's
|
||||
# `reasoning_content`) — the harness models.yml compat block for this
|
||||
# provider must set reasoningContentField accordingly. The qwen3 parser is
|
||||
# what keeps thinking out of `content`. Tools ride the qwen3_coder parser
|
||||
# with auto tool choice. --language-model-only drops the vision tower (the
|
||||
# endpoint stays text-only; the tower can come back by dropping the flag).
|
||||
FRANK_LARGE_REASONING_PARSER=qwen3
|
||||
FRANK_LARGE_TOOL_PARSER=qwen3_coder
|
||||
@@ -1,20 +1,46 @@
|
||||
# frank-large.service — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the Strix
|
||||
# Halo gfx1151 iGPU, :7770. 2026-10-01 cutover from gufo (tag
|
||||
# frank-large-gufo-q4kxl-20261001 restores the Q4_K_XL + DFlash2 shape).
|
||||
#
|
||||
# WHY A ROOT UNIT: parity with frank-primary/frank-small — the unit is
|
||||
# relink_unit'd from this repo into /etc/systemd/system/, PID1 grants
|
||||
# LimitMEMLOCK=infinity (vLLM pins host buffers), and OOM protection lands
|
||||
# on the engine instead of the co-tenants. The container runs in dank's
|
||||
# ROOTLESS podman storage.
|
||||
#
|
||||
# The serve script drives `podman run` directly (image entrypoint is
|
||||
# overridden to `vllm serve`) and blocks on the container, so a container
|
||||
# death is a non-zero ExecStart. NOTE from frank-primary's experience:
|
||||
# vLLM's API server can drain and exit 0 after an engine fault, so the
|
||||
# restart policy MUST be always, not on-failure.
|
||||
[Unit]
|
||||
Description=gufo (gufo-org/gufo, ROCm gfx1151) Qwen3.8-27B UD-Q8_K_XL, DFlash2 spec decode, 262144 ctx / 1 session, root systemd unit (dankclaw, frank-large, Strix Halo iGPU)
|
||||
Description=frank-large — vLLM MiMo-V2.6-Distill-Qwen-9B (BF16) on the gfx1151 iGPU
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
# Never let the start-rate limiter give up after a crash loop.
|
||||
StartLimitIntervalSec=0
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=dank
|
||||
Group=dank
|
||||
# The engine pins staging buffers; keep the memlock budget unbounded so an
|
||||
# RLIMIT smaller than the pinned set cannot fail the load. Parity with the
|
||||
# other LLM units on this fleet.
|
||||
LimitMEMLOCK=infinity
|
||||
Environment="FRANK_LARGE_ENV_FILE=/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
ExecStart=/home/dank/serverAppConfigs/scripts/serve-frank-large.sh
|
||||
Restart=on-failure
|
||||
# systemd stop: podman wait alone cannot stop the container child; stop it
|
||||
# explicitly or the engine outlives the unit holding the GPU and port.
|
||||
ExecStop=/usr/bin/podman stop -t 120 frank-large-mimo
|
||||
# Boot cost: ~200-240 s warm (measured 2026-10-01 bench window); cold torch
|
||||
# compile is longer. Health stays 503 until the engine is up — do not kill.
|
||||
TimeoutStartSec=1800
|
||||
# ExecStop allows 120 s of drain; the stop timeout must exceed it (the
|
||||
# frank-primary 2026-10-01 lesson: a short TimeoutStopSec records 'Failed
|
||||
# with result timeout' on every stop).
|
||||
TimeoutStopSec=180
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# The engine holds 17.5 GiB of weights plus the KV pool in the GTT
|
||||
# aperture; it is the last process on this host the kernel should pick.
|
||||
OOMScoreAdjust=-500
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
+130
-102
@@ -1,121 +1,149 @@
|
||||
#!/usr/bin/env bash
|
||||
# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on
|
||||
# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the
|
||||
# inference engine built specifically for this hardware.
|
||||
# serve-frank-large.sh — frank-large: MiMo-V2.6-Distill-Qwen-9B (BF16) under
|
||||
# vLLM on the Strix Halo gfx1151 iGPU, :7770.
|
||||
#
|
||||
# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source
|
||||
# against the host ROCm by scripts/build-gufo.sh. It replaced the stock
|
||||
# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client
|
||||
# contract did not move.
|
||||
# 2026-10-01 cutover from gufo (Qwen3.8-27B GGUF + DFlash2). The runtime is
|
||||
# the digest-pinned kyuz0 TheRock image (the fork that carries the gfx1151
|
||||
# build: ROCm 10 / torch 2.11 / vLLM 0.30 with the Triton Gated DeltaNet
|
||||
# kernels); like frank-primary, the service drives `podman run` directly and
|
||||
# then blocks on the container, so a container death is a non-zero ExecStart
|
||||
# and the unit's Restart=always is what self-heals.
|
||||
#
|
||||
# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's
|
||||
# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally
|
||||
# interchangeable for this engine — same `qwen35` architecture, the same 866
|
||||
# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the
|
||||
# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune
|
||||
# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which
|
||||
# is not an OSI licence.
|
||||
#
|
||||
# Shape: one session, full 262144 context (the model's native window). The
|
||||
# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV +
|
||||
# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this
|
||||
# context measured 66.5 GiB of the 116.7 GiB device.
|
||||
#
|
||||
# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block
|
||||
# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the
|
||||
# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048
|
||||
# (small draft KV). It is UNCHANGED by the model swap and still lives beside
|
||||
# the previous target, which is why DRAFTER_DIR is its own tunable.
|
||||
#
|
||||
# Sampling: greedy (--temperature 0) server-side, as every frank-large has
|
||||
# served. The harness overrides it per request via extraBody (1.0 / 0.95 /
|
||||
# top-k 20).
|
||||
#
|
||||
# Template: gufo compiles the official Qwen Jinja into the binary and takes
|
||||
# NO --chat-template-file — and it is not a Jinja engine at all
|
||||
# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the
|
||||
# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law
|
||||
# was given up when this endpoint moved to gufo on 2026-09-28, by decision:
|
||||
# the terse system-prompt injection, the in-band <|think_*|> control, the
|
||||
# json|xml tool_call_format switch and the tool-argument truncation are all
|
||||
# absent. gufo's own renderer keeps the effort instruction strings and
|
||||
# preserve_thinking, which are the parts the harness depends on. Accepted
|
||||
# 2026-09-28; recovering the terse prompt from the client side is the
|
||||
# cheapest partial restore.
|
||||
#
|
||||
# Text-only: gufo auto-discovers a mmproj beside the model and this payload
|
||||
# has none (the base repo ships one separately, unused), so the endpoint
|
||||
# stays text-only.
|
||||
#
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
||||
# it; anything not set there falls through to the defaults below.
|
||||
# The engine flags are the ones measured in the 2026-10-01 bench window on
|
||||
# this host (see frank-large.env for the numbers). Every tunable arrives via
|
||||
# frank-large.env; keys are FRANK_LARGE_-prefixed because HIP reads bare
|
||||
# TEMP/HOST/GPU_MAX_HEAP_SIZE from the environment.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-/home/dank/serverAppConfigs/scripts/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- defaults (used when the env file does not set them) ------------------
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CONTEXT_SIZE:=262144}"
|
||||
: "${SESSIONS:=1}"
|
||||
: "${DRAFT_TOKENS:=7}"
|
||||
: "${DRAFT_POLICY:=adaptive}"
|
||||
: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}"
|
||||
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}"
|
||||
: "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}"
|
||||
: "${FRANK_LARGE_CONTAINER:=frank-large-mimo}"
|
||||
: "${FRANK_LARGE_LEGACY_CONTAINERS:=vllm-mimo}"
|
||||
: "${FRANK_LARGE_SERVED_NAME:=frank-large}"
|
||||
: "${FRANK_LARGE_PUBLISH:=7770:7770}"
|
||||
: "${FRANK_LARGE_MAXLEN:=262144}"
|
||||
: "${FRANK_LARGE_SEQS:=4}"
|
||||
: "${FRANK_LARGE_UTIL:=0.45}"
|
||||
: "${FRANK_LARGE_REASONING_PARSER:=qwen3}"
|
||||
: "${FRANK_LARGE_TOOL_PARSER:=qwen3_coder}"
|
||||
: "${FRANK_LARGE_SPEC:=}"
|
||||
: "${FRANK_LARGE_SPEC_TOKENS:=8}"
|
||||
: "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}"
|
||||
|
||||
GUFO_BIN="$RUNTIME_DIR/bin/gufo"
|
||||
|
||||
# The target GGUF and the drafter are named explicitly because they do not
|
||||
# share a directory any more: the target is a different repository's artifact
|
||||
# and keeps its own long filename, while the DFlash2 drafter is unchanged and
|
||||
# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to
|
||||
# the deployed payload, so the env file only has to state them when it moves.
|
||||
: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
|
||||
: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}"
|
||||
: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
|
||||
if [[ ! -x "$GUFO_BIN" ]]; then
|
||||
echo "runtime not found at $GUFO_BIN" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
# --- gates (fail FAST, before touching the running stack) ------------------
|
||||
# Image gate by REPO DIGEST, not the human-readable ref: rootless podman
|
||||
# resolves the tag from the network, and the whole point of the pin is to
|
||||
# boot exactly these bytes offline. Capture first, grep second — piping
|
||||
# straight into `grep -q` dies of the SIGPIPE race under pipefail.
|
||||
if ! digests="$(podman images --format '{{.RepoDigests}}')" ||
|
||||
! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $FRANK_LARGE_IMAGE" >&2
|
||||
echo "frank-large: fetch it (podman pull $FRANK_LARGE_IMAGE)" >&2
|
||||
exit 1
|
||||
fi
|
||||
for f in "$MODEL_FILE" "$DRAFTER_FILE"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2
|
||||
|
||||
# Payload gate: config + weight-map index + the exact shard count. A partial
|
||||
# payload fails the load minutes later, after the port has been freed and
|
||||
# the old server stopped.
|
||||
if [[ ! -f "$FRANK_LARGE_MODEL_DIR/config.json" ]]; then
|
||||
echo "ERROR: payload config missing: $FRANK_LARGE_MODEL_DIR/config.json" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -f "$FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: weight-map index missing: $FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
shards_found=$(find "$FRANK_LARGE_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)
|
||||
if [[ "$shards_found" != "$FRANK_LARGE_MODEL_SHARDS" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $FRANK_LARGE_MODEL_DIR (found $shards_found, want $FRANK_LARGE_MODEL_SHARDS)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$FRANK_LARGE_SPEC" ]]; then
|
||||
: "${FRANK_LARGE_SPEC_DIR:?FRANK_LARGE_SPEC_DIR not set (host dir holding the draft weights)}"
|
||||
# dflash ships ONE model.safetensors; the Qwen3.5-9B MTP head ships as
|
||||
# shards behind model.safetensors.index.json. Either shape is a real
|
||||
# draft; config.json must be present in both, and a directory with
|
||||
# neither weight layout is not a draft at all.
|
||||
if [[ ! -f "$FRANK_LARGE_SPEC_DIR/config.json" ]] ||
|
||||
{ [[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors" ]] &&
|
||||
[[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors.index.json" ]]; }; then
|
||||
echo "ERROR: spec draft absent or partial: $FRANK_LARGE_SPEC_DIR (FRANK_LARGE_SPEC=$FRANK_LARGE_SPEC)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The models tree is mounted read-only at /models; the engine must be
|
||||
# given the CONTAINER path to the draft, not the host path.
|
||||
FRANK_LARGE_SPEC_CONTAINER_DIR="/models/${FRANK_LARGE_SPEC_DIR##*/}"
|
||||
fi
|
||||
|
||||
# The engine needs the RENDER nodes too, not just /dev/kfd: mounting the
|
||||
# whole /dev/dri directory (never individual renderD nodes) is the
|
||||
# verified-working shape (Dankserver 2026-09-29: individual render nodes
|
||||
# enumerate the device but cannot create a context).
|
||||
for dev in /dev/kfd /dev/dri; do
|
||||
if [[ ! -e "$dev" ]]; then
|
||||
echo "ERROR: $dev missing; gfx1151 cannot serve without it" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# --- env -------------------------------------------------------------------
|
||||
# gufo links against the host ROCm through its RPATH (/opt/rocm ->
|
||||
# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is
|
||||
# already gfx1151 — `gufo diagnose` reports it natively — so no
|
||||
# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt
|
||||
# llama.cpp runtime and can only misreport the arch here.
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "$FRANK_LARGE_SPEC" ]]; then
|
||||
SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}")
|
||||
fi
|
||||
|
||||
# --- the serve command ---------------------------------------------------
|
||||
# Server options precede the subcommand, model options follow it.
|
||||
exec "$GUFO_BIN" serve \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--sessions "$SESSIONS" \
|
||||
llm \
|
||||
--model "$MODEL_FILE" \
|
||||
--served-model-name "$ALIAS" \
|
||||
--context "$CONTEXT_SIZE" \
|
||||
--speculative dflash2 \
|
||||
--dflash-model "$DRAFTER_FILE" \
|
||||
--draft-tokens "$DRAFT_TOKENS" \
|
||||
--draft-policy "$DRAFT_POLICY" \
|
||||
--temperature 0 \
|
||||
--think auto \
|
||||
--preserve-thinking on
|
||||
# Remove stale container identities BEFORE starting: a crashed unit can
|
||||
# leave the name registered forever after, and the bench-window identity
|
||||
# (vllm-mimo) holds the GPU and port when a window ends without teardown.
|
||||
for name in $FRANK_LARGE_LEGACY_CONTAINERS "$FRANK_LARGE_CONTAINER"; do
|
||||
if podman ps -a --format '{{.Names}}' | grep -qx "$name"; then
|
||||
echo "removing stale container: $name"
|
||||
podman rm -f "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
|
||||
# --device /dev/mem exists for the fleet's other GPU containers; the GTT
|
||||
# aperture (114 GiB) needs no size knob. Models mount read-only wholesale so
|
||||
# spec drafts resolve from disk, and the HF cache mount keeps tokenizer
|
||||
# metadata out of the container's ephemeral layer.
|
||||
podman run -d \
|
||||
--name "$FRANK_LARGE_CONTAINER" \
|
||||
--device /dev/kfd --device /dev/dri --device /dev/mem \
|
||||
--security-opt seccomp=unconfined --group-add keep-groups \
|
||||
--ulimit memlock=-1:-1 \
|
||||
-v "$FRANK_LARGE_MODELS_DIR:/models:ro" \
|
||||
-v /home/dank/models/hf-cache:/root/.cache/huggingface \
|
||||
-e HSA_ENABLE_SDMA=0 \
|
||||
-p "$FRANK_LARGE_PUBLISH" \
|
||||
--entrypoint vllm \
|
||||
"$FRANK_LARGE_IMAGE" \
|
||||
serve "$FRANK_LARGE_MODEL_DIR" \
|
||||
--served-model-name "$FRANK_LARGE_SERVED_NAME" \
|
||||
--host 0.0.0.0 --port "${FRANK_LARGE_PUBLISH##*:}" \
|
||||
--dtype bfloat16 \
|
||||
--max-model-len "$FRANK_LARGE_MAXLEN" \
|
||||
--max-num-seqs "$FRANK_LARGE_SEQS" \
|
||||
--gpu-memory-utilization "$FRANK_LARGE_UTIL" \
|
||||
--reasoning-parser "$FRANK_LARGE_REASONING_PARSER" \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \
|
||||
--language-model-only \
|
||||
--trust-remote-code \
|
||||
"${SPEC_ARGS[@]}"
|
||||
|
||||
# Block on the container so a container death returns non-zero from ExecStart
|
||||
# and the unit's Restart=always fires. systemd stop goes through the unit's
|
||||
# ExecStop (podman stop -t); podman wait unblocks when the container is gone.
|
||||
exec podman wait "$FRANK_LARGE_CONTAINER"
|
||||
Reference in new issue
Block a user