frank-large: roll back to gufo Qwen3.8-27B Q4_K_XL, one session x 262144 f16 KV
frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144, asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
This commit is contained in:
Frank
+77
-100
@@ -1077,20 +1077,27 @@ deploy_anubis() {
|
||||
wait_container_healthy anubis 60 || return 1
|
||||
}
|
||||
|
||||
# frank-large is vLLM (the digest-pinned kyuz0 TheRock gfx1151 image) serving
|
||||
# the BF16 MiMo-V2.6-Distill-Qwen-9B payload with cross-model MTP spec decode
|
||||
# (Qwen/Qwen3.5-9B head, 2 draft tokens) on the Strix Halo iGPU. 2026-10-01
|
||||
# cutover from gufo; the gufo-era gates (built-from-source binary, GGUF
|
||||
# target + DFlash2 file) are retired with it — rollback is git tag
|
||||
# frank-large-gufo-q4kxl-20261001, and the gufo runtime and payloads stay on
|
||||
# disk. The service drives `podman run` directly from its serve script, so
|
||||
# the gates here mirror deploy_llm_primary's: image present by REPO DIGEST in
|
||||
# dank's rootless storage, payload complete (config.json + index + 4 shards),
|
||||
# and the spec draft present whenever FRANK_LARGE_SPEC names one. The port
|
||||
# gate is the same as every cutover since 2026-09-16: nothing starts until
|
||||
# 7770 is released.
|
||||
LLM_LARGE_CONTAINER=frank-large-mimo
|
||||
LLM_LARGE_LEGACY_CONTAINERS="vllm-mimo"
|
||||
# frank-large is gufo (gufo-org/gufo), the Strix Halo inference engine built
|
||||
# from the pinned v0.1.1 tag by scripts/build-gufo.sh, serving
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL with the DFlash2 drafter
|
||||
# (z-lab/Qwen3.8-27B-DFlash2-GGUF:Q4_K_M) on the Strix Halo iGPU. It is a
|
||||
# ROOT unit: the runtime is a native binary linked against the host ROCm,
|
||||
# so there is no container image to install and no generated unit to dodge.
|
||||
# relink_unit points /etc/systemd/system/frank-large.service at the repo
|
||||
# file and boot persistence is that unit's [Install] section.
|
||||
#
|
||||
# The runtime gate matters as much as the payload gate: gufo is built from
|
||||
# source on this host (there is no prebuilt release), so a machine whose
|
||||
# /home/dank/GitPackages tree was never built has a unit that can never
|
||||
# become healthy, and a green deploy would leave the endpoint dark.
|
||||
#
|
||||
# It replaces the stock llama.cpp prebuilt (unsloth b11030), which had
|
||||
# replaced the Kairic Edge runtime (ciru-ai/ROCmFPX). The port gate is the
|
||||
# same as every cutover since 2026-09-16: nothing starts until 7770 is
|
||||
# released.
|
||||
LLM_LARGE_RUNTIME="${LLM_LARGE_RUNTIME:-/home/dank/GitPackages/gufo-v0.1.1/bin/gufo}"
|
||||
LLM_LARGE_MODEL="${LLM_LARGE_MODEL:-/home/dank/models/Qwen3.8-27B-UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
|
||||
LLM_LARGE_DRAFT="${LLM_LARGE_DRAFT:-/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
|
||||
retire_halogen_large_quadlet() {
|
||||
local trash="$HOME/.AGENTTRASH/20260919-orca-cutover/quadlet"
|
||||
@@ -1123,127 +1130,97 @@ wait_port_free() {
|
||||
}
|
||||
|
||||
deploy_llm_large() {
|
||||
# Refuse rather than restart-loop: a unit whose image or payload is
|
||||
# Refuse rather than restart-loop. A unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark. Gates mirror deploy_llm_primary.
|
||||
local env_file="/home/dank/serverAppConfigs/scripts/frank-large.env"
|
||||
local image model_dir shards count spec spec_dir digests
|
||||
if [[ ! -f "$env_file" ]]; then
|
||||
echo "ERROR: frank-large env file absent: $env_file" >&2
|
||||
# endpoint dark with nothing in the deploy log to say why.
|
||||
if [[ ! -x "$LLM_LARGE_RUNTIME" ]]; then
|
||||
echo "ERROR: gufo runtime absent at $LLM_LARGE_RUNTIME" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
image="$(sed -n 's/^FRANK_LARGE_IMAGE=//p' "$env_file" | tail -1)"
|
||||
model_dir="$(sed -n 's/^FRANK_LARGE_MODEL_DIR=//p' "$env_file" | tail -1)"
|
||||
shards="$(sed -n 's/^FRANK_LARGE_MODEL_SHARDS=//p' "$env_file" | tail -1)"
|
||||
spec="$(sed -n 's/^FRANK_LARGE_SPEC=//p' "$env_file" | tail -1)"
|
||||
spec_dir="$(sed -n 's/^FRANK_LARGE_SPEC_DIR=//p' "$env_file" | tail -1)"
|
||||
if [[ -z "$image" || -z "$model_dir" || -z "$shards" ]]; then
|
||||
echo "ERROR: frank-large.env missing FRANK_LARGE_IMAGE/MODEL_DIR/MODEL_SHARDS" >&2
|
||||
return 1
|
||||
fi
|
||||
# Capture podman's output BEFORE grepping (the SIGPIPE race, 2026-09-30).
|
||||
digests="$(podman images --format '{{.RepoDigests}}')"
|
||||
if ! grep -qF "${image#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $image" >&2
|
||||
echo "frank-large: fetch it (podman pull $image)" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$model_dir/config.json" || ! -f "$model_dir/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: payload incomplete: $model_dir (config.json / weight-map index)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
count="$(find "$model_dir" -maxdepth 1 -name 'model-*-of-*.safetensors' 2>/dev/null | wc -l)"
|
||||
if [[ "$count" != "$shards" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $model_dir (found $count, want $shards)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "$spec" ]]; then
|
||||
# Both draft weight layouts are real: dflash ships a single
|
||||
# model.safetensors, the Qwen3.5-9B MTP head ships sharded behind
|
||||
# an index. Mirrors the serve script's gate (2026-10-01: the
|
||||
# single-file check rejected the deployed mtp arm).
|
||||
if [[ -z "$spec_dir" || ! -f "$spec_dir/config.json" ]] ||
|
||||
{ [[ ! -f "$spec_dir/model.safetensors" ]] &&
|
||||
[[ ! -f "$spec_dir/model.safetensors.index.json" ]]; }; then
|
||||
echo "ERROR: spec draft absent or partial for FRANK_LARGE_SPEC=$spec: $spec_dir" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2
|
||||
return 1
|
||||
local missing=0 f
|
||||
for f in "$LLM_LARGE_MODEL" "$LLM_LARGE_DRAFT"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "ERROR: payload absent at $f" >&2
|
||||
echo "frank-large: fetch it: bash scripts/download-frank-large.sh" >&2
|
||||
missing=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
[[ "$missing" -eq 0 ]] || return 1
|
||||
|
||||
retire_halogen_large_quadlet
|
||||
# On a re-deploy the predecessor is THIS unit: stop it before waiting on
|
||||
# the port, or the wait deadlocks on the server it just stopped serving.
|
||||
# Then sweep a stray manual run -- that is how a bench harness leaves the
|
||||
# port busy. Match the DRAFTER path, which every frank-large command
|
||||
# line carries via --dflash-model and which is UNCHANGED by the 2026-09-28
|
||||
# payload swap: the target moved to a different repository while the
|
||||
# drafter stayed put, so keying on the target filename would have quietly
|
||||
# stopped matching. It is also the right key in principle — it names this
|
||||
# endpoint's speculative drafter, not a binary. Match that, not a binary
|
||||
# name: frank-small on this host serves a different GGUF over llama-server
|
||||
# with no DFlash2 drafter and must not be killed. The pattern never
|
||||
# appears in the deploying shell's own command line, so -f is safe here
|
||||
# (it is not in `bash deploy-remote.sh`).
|
||||
run_privileged systemctl stop frank-large.service 2>/dev/null || true
|
||||
# A stopped unit can leave its container running (the cgroup-kill vs
|
||||
# conmon --rm race); the orphan holds the port and the GPU. The
|
||||
# bench-window identity (vllm-mimo) dies here too — that is how a bench
|
||||
# harness leaves the port busy.
|
||||
local ctr
|
||||
for ctr in $LLM_LARGE_LEGACY_CONTAINERS "$LLM_LARGE_CONTAINER"; do
|
||||
podman rm -f "$ctr" >/dev/null 2>&1 || true
|
||||
done
|
||||
# Nothing may start until the predecessor has released the port
|
||||
# (2026-09-16 and 2026-09-26 cutovers both failed exactly here).
|
||||
pkill -f 'Qwen3.8-27B-DFlash2-Q4_K_M' 2>/dev/null || true
|
||||
# Nothing may start until the predecessor has released the port. On
|
||||
# 2026-09-16 the cutover failed exactly here: the old server logged
|
||||
# "Deactivated successfully", the next engine tried to bind 7770 a second
|
||||
# later and reported "Address already in use". Gate on the port, not on
|
||||
# hope.
|
||||
wait_port_free 7770 || return 1
|
||||
relink_unit frank-large
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-large.service
|
||||
run_privileged systemctl restart frank-large.service
|
||||
# Warm boot is ~200-240 s (measured 2026-10-01); cold torch compile is
|
||||
# longer. Parity with frank-primary's 1800 s allowance.
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1800 || return 1
|
||||
# False-green hardening: the port answering proves nothing about WHO
|
||||
# answers — the unit must be active with a live MainPID.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-large.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7770 answered but frank-large.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-large: health answered by unit MainPID $main_pid"
|
||||
wait_for_health frank-large http://127.0.0.1:7770/health 1200
|
||||
}
|
||||
|
||||
# deploy_llm_small — frank-small is SGLang (python3.12 venv) on the RTX 3090,
|
||||
# as a ROOT unit: the runtime is a self-contained venv, so there is no
|
||||
# container image to install and no generated unit to avoid. relink_unit
|
||||
# points /etc/systemd/system/frank-small.service at the repo file and boot
|
||||
# deploy_llm_small — frank-small is llama.cpp (master build, CUDA sm_86) on
|
||||
# the RTX 3090 as a ROOT unit: the runtime is a native binary tree under
|
||||
# ~/GitPackages/llama-master/build-cuda, so there is no container image and
|
||||
# no generated unit to avoid. relink_unit points
|
||||
# /etc/systemd/system/frank-small.service at the repo file and boot
|
||||
# persistence is that unit's [Install] section.
|
||||
#
|
||||
# It is a SEPARATE endpoint on port 7771; frank-large keeps 7770. The 3090 is
|
||||
# not shared: frank-large's engine runs on the gfx1151 iGPU.
|
||||
LLM_SMALL_MODEL_DIR="${LLM_SMALL_MODEL_DIR:-/home/dank/models/MiMo-V2.6-Distill-Qwen-9B-INT8-W8A16-AutoRound}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/sglang-cu128/bin/sglang}"
|
||||
LLM_SMALL_MODEL_FILE="${LLM_SMALL_MODEL_FILE:-/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
|
||||
LLM_SMALL_RUNTIME="${LLM_SMALL_RUNTIME:-/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
||||
|
||||
deploy_llm_small() {
|
||||
# Refuse rather than restart-loop: a unit whose runtime or payload is
|
||||
# missing never becomes healthy, and a green deploy would leave the
|
||||
# endpoint dark with nothing in the log to say why.
|
||||
if [[ ! -x "$LLM_SMALL_RUNTIME" ]]; then
|
||||
echo "ERROR: SGLang runtime absent at $LLM_SMALL_RUNTIME" >&2
|
||||
echo "frank-small: build the venv (python3.12 -m venv /home/dank/GitPackages/sglang-cu128" >&2
|
||||
echo "frank-small: and pip install 'sglang[all]' into it) first." >&2
|
||||
echo "ERROR: llama.cpp runtime absent at $LLM_SMALL_RUNTIME" >&2
|
||||
echo "frank-small: build it (cmake -B build-cuda -DGGML_CUDA=ON" >&2
|
||||
echo "frank-small: -DCMAKE_CUDA_ARCHITECTURES=86 -DCMAKE_CUDA_HOST_COMPILER=/usr/bin/gcc-13" >&2
|
||||
echo "frank-small: && cmake --build build-cuda --target llama-server) first." >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! -f "$LLM_SMALL_MODEL_DIR/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: payload index absent at $LLM_SMALL_MODEL_DIR" >&2
|
||||
echo "frank-small: run scripts/download-frank-small.sh first;" >&2
|
||||
if [[ ! -f "$LLM_SMALL_MODEL_FILE" ]]; then
|
||||
echo "ERROR: payload GGUF absent at $LLM_SMALL_MODEL_FILE" >&2
|
||||
echo "frank-small: fetch bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF Q6_K first;" >&2
|
||||
echo "frank-small: refusing to start a unit that cannot load." >&2
|
||||
return 1
|
||||
fi
|
||||
local count
|
||||
count="$(find "$LLM_SMALL_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)"
|
||||
if [[ "$count" -ne 8 ]]; then
|
||||
echo "ERROR: payload shard count wrong: $LLM_SMALL_MODEL_DIR (found $count, want 8)" >&2
|
||||
return 1
|
||||
fi
|
||||
relink_unit frank-small
|
||||
run_privileged systemctl daemon-reload
|
||||
run_privileged systemctl enable frank-small.service
|
||||
run_privileged systemctl restart frank-small.service
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900
|
||||
wait_for_health frank-small http://127.0.0.1:7771/health 900 || return 1
|
||||
# False-green hardening (frank-large's lesson): the port answering
|
||||
# proves nothing about WHO answers — the unit must be active with a
|
||||
# live MainPID, or a stray SGLang holdover is masquerading as deployed.
|
||||
local main_pid
|
||||
main_pid="$(run_privileged systemctl show -p MainPID --value frank-small.service)"
|
||||
if [[ -z "$main_pid" || "$main_pid" == "0" ]]; then
|
||||
echo "FAIL: :7771 answered but frank-small.service has no MainPID — a stray process holds the port" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "frank-small: health answered by unit MainPID $main_pid"
|
||||
}
|
||||
|
||||
# deploy_llm_primary — frank-primary is the tcclaviger RDNA4 vLLM fork
|
||||
|
||||
Reference in New Issue
Block a user