frank-large: cutover to vLLM TheRock gfx1151, MiMo-V2.6 BF16 + cross-model MTP
Engine: digest-pinned docker.io/kyuz0/vllm-therock-gfx1151 (vLLM 0.30, TheRock ROCm 10, Triton GDN kernels); the serve script gates the image by REPO DIGEST, the payload (config + index + exact shard count), the spec draft (BOTH weight layouts: single model.safetensors for dflash, sharded index for the Qwen3.5-9B MTP head), and whole /dev/dri, sweeps the bench identity vllm-mimo, then exec podman wait so a container death trips Restart=always. Shape: 4 x 262144 at util 0.45 (KV grows only on the 8 full-attention layers), MTP 2 draft tokens. Measured 2026-10-01 window (frank-large stopped): base 13.1 t/s decode (~94% of the LPDDR5x BF16 roof), DFlash-8 17.4, MTP-2 19.8 with 71%/44% per-position acceptance - the cross-model head works because vLLM reads the draft from a separate path while MiMo ships zero mtp.* tensors. Spec arms do not preserve greedy bf16 text (batched-verify reduction order). The seam was rewritten around the new laws (18 tests): unit self-healing, digest gating, engine wiring, the measured spec table, the retired gufo gates, the deployer MainPID check. frank_small port test follows the PORT -> FRANK_LARGE_PUBLISH rename. Rollback: tag frank-large-gufo-q4kxl-20261001.
This commit is contained in:
Frank committed
1 parent
b3313340fa
commit
dab483c79f
6 files changed
+487
-516
No files matched your search
+75
-52
@@ -1077,27 +1077,20 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built
|
||||
# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter
|
||||
# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a
|
||||
# ROOT unit: the runtime is a native binary linked against the host ROCm,
|
||||
# so there is no container image to install and no generated unit to dodge.
|
||||
# relink_unit points /etc/systemd/system/frank-large.service at the repo
|
||||
# file and boot persistence is that unit's [Install] section.
|
||||
#
|
||||
# The runtime gate matters as much as the payload gate: gufo is built from
|
||||
# source on this host (there is no prebuilt release), so a machine whose
|
||||
# /home/dank/GitPackages tree was never built has a unit that can never
|
||||
# become healthy, and a green deploy would leave the endpoint dark.
|
||||
#
|
||||
# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had
|
||||
# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the
|
||||
# same as every cutover since 2026-09-16: nothing starts until 7770 is
|
||||
# released.
|
||||
LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}"
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-UD-Q8_K_XL.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
# frank-large is vLLM (the digest-pinned kyuz0 TheRock gfx1151 image) serving
|
||||
# the BF16 MiMo-V2.6-Distill-Qwen-9B payload with cross-model MTP spec decode
|
||||
# (Qwen/Qwen3.5-9B head, 2 draft tokens) on the Strix Halo iGPU. 2026-10-01
|
||||
# cutover from gufo; the gufo-era gates (built-from-source binary, GGUF
|
||||
# target + DFlash2 file) are retired with it — rollback is git tag
|
||||
# frank-large-gufo-q4kxl-20261001, and the gufo runtime and payloads stay on
|
||||
# disk. The service drives `podman run` directly from its serve script, so
|
||||
# the gates here mirror deploy_llm_primary's: image present by REPO DIGEST in
|
||||
# dank's rootless storage, payload complete (config.json + index + 4 shards),
|
||||
# and the spec draft present whenever FRANK_LARGE_SPEC names one. The port
|
||||
# gate is the same as every cutover since 2026-09-16: nothing starts until
|
||||
# 7770 is released.
|
||||
LLM_LARGE_CONTAINER=frank-large-mimo
|
||||
LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1130,51 +1123,81 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the deploy log to say why.
|
||||
if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then
|
||||
echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
# endpoint dark. Gates mirror deploy_llm_primary.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
local image model_dir shards count spec spec_dir digests
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-large env file absent: $env_file" >&2
|
||||
return 1
|
||||
fi
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2
|
||||
missing=1
|
||||
image="$(sed -n 's/^FRANK_LARGE_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_LARGE_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
shards="$(sed -n 's/^FRANK_LARGE_MODEL_SHARDS=//p' "$env_file" | tail -1)"
|
||||
spec="$(sed -n 's/^FRANK_LARGE_SPEC=//p' "$env_file" | tail -1)"
|
||||
spec_dir="$(sed -n 's/^FRANK_LARGE_SPEC_DIR=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then
|
||||
echo "ERROR: frank-large.env missing FRANK_LARGE_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2
|
||||
return 1
|
||||
fi
|
||||
# Capture podman's output BEFORE grepping (the SIGPIPE race, 2026-09-30).
|
||||
digests="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${image#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-large: fetch it (podman pull $image)" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/config.json" || ! -f "$model_dir/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: payload incomplete: $model_dir (config.json / weight-map index)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)"
|
||||
if [[ "$count" != "$shards" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "$spec" ]]; then
|
||||
if [[ -z "$spec_dir" || ! -f "$spec_dir/model.safetensors" ]]; then
|
||||
echo "ERROR: spec draft absent for FRANK_LARGE_SPEC=$spec: $spec_dir/model.safetensors" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
fi
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
||||
# Then sweep a stray manual run -- that is how a bench harness leaves the
|
||||
# port busy. Match the DRAFTER path, which every frank-large command
|
||||
# line carries via --dflash-model and which is UNCHANGED by the 2026-09-28
|
||||
# payload swap: the target moved to a different repository while the
|
||||
# drafter stayed put, so keying on the target filename would have quietly
|
||||
# stopped matching. It is also the right key in principle — it names this
|
||||
# endpoint's speculative drafter, not a binary. Match that, not a binary
|
||||
# name: frank-small on this host serves a different GGUF over llama-server
|
||||
# with no DFlash2 drafter and must not be killed. The pattern never
|
||||
# appears in the deploying shell's own command line, so -f is safe here
|
||||
# (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
# later and reported "Address already in use". Gate on the port, not on
|
||||
# hope.
|
||||
# A stopped unit can leave its container running (the cgroup-kill vs
|
||||
# conmon --rm race); the orphan holds the port and the GPU. The
|
||||
# bench-window identity (vllm-mimo) dies here too — that is how a bench
|
||||
# harness leaves the port busy.
|
||||
local ctr
|
||||
for ctr in $LLM_LARGE_LEGACY_CONTAINERS "$LLM_LARGE_CONTAINER"; do
|
||||
podman rm -f "$ctr" >/dev/null 2>&1 || true
|
||||
done
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16 and 2026-09-26 cutovers both failed exactly here).
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
# Warm boot is ~200-240 s (measured 2026-10-01); cold torch compile is
|
||||
# longer. Parity with frank-primary's 1800 s allowance.
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1800 || return 1
|
||||
# False-green hardening: the port answering proves nothing about WHO
|
||||
# answers — the unit must be active with a live MainPID.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-large.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-large.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-large: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090,
|
||||
|
||||
Reference in new issue
Block a user