frank-large: roll back to gufo Qwen3.8-27B Q4_K_XL, one session x 262144 f16 KV
frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144, asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
This commit is contained in:
Frank committed
1 parent
e452a1afbf
commit
8551bb006b
11 files changed
+729
-816
No files matched your search
+101
-179
@@ -1,199 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
# serve-frank-large.sh — frank-large: MiMo-V2.6-Distill-Qwen-9B (BF16) under
|
||||
# vLLM on the Strix Halo gfx1151 iGPU, :7770.
|
||||
# Serve Swift-1.5-Qwen3.8-27B-Uncensored (abliterated) as frank-large on
|
||||
# Dankclaw's Strix Halo iGPU (gfx1151), using gufo (gufo-org/gufo) — the
|
||||
# inference engine built specifically for this hardware.
|
||||
#
|
||||
# 2026-10-01 cutover from gufo (Qwen3.8-27B GGUF + DFlash2). The runtime is
|
||||
# the digest-pinned kyuz0 TheRock image (the fork that carries the gfx1151
|
||||
# build: ROCm 10 / torch 2.11 / vLLM 0.30 with the Triton Gated DeltaNet
|
||||
# kernels); like frank-primary, the service drives `podman run` directly and
|
||||
# then blocks on the container, so a container death is a non-zero ExecStart
|
||||
# and the unit's Restart=always is what self-heals.
|
||||
# Runtime: /home/dank/GitPackages/gufo-v0.1.1/bin/gufo, built from source
|
||||
# against the host ROCm by scripts/build-gufo.sh. It replaced the stock
|
||||
# llama.cpp prebuilt (unsloth b11030) on 2026-09-28; port, alias and client
|
||||
# contract did not move.
|
||||
#
|
||||
# The engine flags are the ones measured in the 2026-10-01 bench window on
|
||||
# this host (see frank-large.env for the numbers). Every tunable arrives via
|
||||
# frank-large.env; keys are FRANK_LARGE_-prefixed because HIP reads bare
|
||||
# TEMP/HOST/GPU_MAX_HEAP_SIZE from the environment.
|
||||
# Payload: ajgazin/Swift-1.5-Qwen3.8-27B-Uncensored-Dynamic-MTP-GGUF's
|
||||
# UD-Q8_K_XL, fetched by scripts/download-frank-large-swift.sh. It replaced
|
||||
# unsloth/Qwen3.8-27B-GGUF:UD-Q8_K_XL on 2026-09-28. The two are structurally
|
||||
# interchangeable for this engine — same `qwen35` architecture, the same 866
|
||||
# tensors in the same order, the same per-tensor Q8_0/F32/BF16 layout, the
|
||||
# same MTP head at blk.64 — so the swap needed no engine change. The fine-tune
|
||||
# is abliterated (uncensored) and carries the SWIFT OPEN LICENSE 1.0, which
|
||||
# is not an OSI licence.
|
||||
#
|
||||
# Shape: one session, full 262144 context (the model's native window). The
|
||||
# 27B dense weights (16.35 GiB at UD-Q4_K_XL) + the DFlash2 drafter (1.07) + KV +
|
||||
# compute buffers fit the iGPU's GTT aperture (~114 GiB); a load at this
|
||||
# context measured 66.5 GiB of the 116.7 GiB device.
|
||||
#
|
||||
# Spec method: DFlash2 (a block-diffusion drafter), gufo's adaptive block
|
||||
# policy, capped at DRAFT_TOKENS proposals per step. The drafter is the
|
||||
# z-lab Qwen3.8-27B-DFlash2-Q4_K_M and it has a sliding window of 2048
|
||||
# (small draft KV). It is UNCHANGED by the model swap and still lives beside
|
||||
# the previous target, which is why DRAFTER_DIR is its own tunable.
|
||||
#
|
||||
# Sampling: greedy (--temperature 0) server-side, as every frank-large has
|
||||
# served. The harness overrides it per request via extraBody (1.0 / 0.95 /
|
||||
# top-k 20).
|
||||
#
|
||||
# Template: gufo compiles the official Qwen Jinja into the binary and takes
|
||||
# NO --chat-template-file — and it is not a Jinja engine at all
|
||||
# (src/models/qwen/chat_template.cpp is a hand-written C++ renderer), so the
|
||||
# fleet's scripts/qwen-sharp-v22.4.0.jinja cannot be supplied to it. That law
|
||||
# was given up when this endpoint moved to gufo on 2026-09-28, by decision:
|
||||
# the terse system-prompt injection, the in-band <|think_*|> control, the
|
||||
# json|xml tool_call_format switch and the tool-argument truncation are all
|
||||
# absent. gufo's own renderer keeps the effort instruction strings and
|
||||
# preserve_thinking, which are the parts the harness depends on. Accepted
|
||||
# 2026-09-28; recovering the terse prompt from the client side is the
|
||||
# cheapest partial restore.
|
||||
#
|
||||
# Text-only: gufo auto-discovers a mmproj beside the model and this payload
|
||||
# has none (the base repo ships one separately, unused), so the endpoint
|
||||
# stays text-only.
|
||||
#
|
||||
# Tunables come from frank-large.env unless FRANK_LARGE_ENV_FILE overrides
|
||||
# it; anything not set there falls through to the defaults below.
|
||||
set -euo pipefail
|
||||
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-/home/dank/serverAppConfigs/scripts/frank-large.env}"
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="${FRANK_LARGE_ENV_FILE:-$SCRIPT_DIR/frank-large.env}"
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# --- runtime switch -----------------------------------------------------------
|
||||
# llama: bare llama-server (master build with qwen35 GDN kernels) on GGUF
|
||||
# weights. vllm (default): the digest-pinned TheRock container below.
|
||||
FRANK_LARGE_RUNTIME="${FRANK_LARGE_RUNTIME:-vllm}"
|
||||
if [[ "$FRANK_LARGE_RUNTIME" == "llama" ]]; then
|
||||
: "${FRANK_LARGE_LLAMA_BIN:?FRANK_LARGE_LLAMA_BIN not set}"
|
||||
: "${FRANK_LARGE_GGUF:?FRANK_LARGE_GGUF not set (target GGUF)}"
|
||||
: "${FRANK_LARGE_DRAFT_GGUF:=}"
|
||||
: "${FRANK_LARGE_LLAMA_CTX:=1048576}" # 4 slots x 262,144, f16 KV
|
||||
for f in "$FRANK_LARGE_LLAMA_BIN" "$FRANK_LARGE_GGUF"; do
|
||||
[[ -x "$f" || -f "$f" ]] || { echo "ERROR: missing $f" >&2; exit 1; }
|
||||
done
|
||||
DRAFT_ARGS=()
|
||||
if [[ -n "$FRANK_LARGE_DRAFT_GGUF" ]]; then
|
||||
[[ -f "$FRANK_LARGE_DRAFT_GGUF" ]] || { echo "ERROR: missing draft $FRANK_LARGE_DRAFT_GGUF" >&2; exit 1; }
|
||||
DRAFT_ARGS=(--model-draft "$FRANK_LARGE_DRAFT_GGUF" --spec-type draft-dflash)
|
||||
fi
|
||||
# ComfyUI co-tenancy on this iGPU: gufo-era law, 2 resident queues.
|
||||
export GPU_MAX_HW_QUEUES=2
|
||||
FRANK_LARGE_PORT="${FRANK_LARGE_PUBLISH:-7770:7770}"
|
||||
FRANK_LARGE_PORT="${FRANK_LARGE_PORT##*:}"
|
||||
exec "$FRANK_LARGE_LLAMA_BIN" \
|
||||
--model "$FRANK_LARGE_GGUF" \
|
||||
"${DRAFT_ARGS[@]}" \
|
||||
--alias "${FRANK_LARGE_SERVED_NAME:=frank-large}" \
|
||||
--host 0.0.0.0 --port "$FRANK_LARGE_PORT" \
|
||||
--parallel 4 --ctx-size "$FRANK_LARGE_LLAMA_CTX" \
|
||||
--n-gpu-layers 99 --flash-attn on --jinja \
|
||||
--reasoning auto --reasoning-format deepseek \
|
||||
--cont-batching
|
||||
fi
|
||||
# --- defaults (used when the env file does not set them) ------------------
|
||||
: "${RUNTIME_DIR:=/home/dank/GitPackages/gufo-v0.1.1}"
|
||||
: "${MODEL_DIR:=/home/dank/models/Swift-1.5-Qwen3.8-27B-Uncensored-MTP-UD-Q8_K_XL}"
|
||||
: "${ALIAS:=frank-large}"
|
||||
: "${HOST:=0.0.0.0}"
|
||||
: "${PORT:=7770}"
|
||||
: "${CONTEXT_SIZE:=262144}"
|
||||
: "${SESSIONS:=1}"
|
||||
: "${DRAFT_TOKENS:=7}"
|
||||
: "${DRAFT_POLICY:=adaptive}"
|
||||
|
||||
# --- below: vllm runtime ------------------------------------------------------
|
||||
: "${FRANK_LARGE_IMAGE:?FRANK_LARGE_IMAGE not set (digest-pinned kyuz0 vllm-therock image)}"
|
||||
: "${FRANK_LARGE_MODEL_DIR:?FRANK_LARGE_MODEL_DIR not set}"
|
||||
: "${FRANK_LARGE_MODEL_SHARDS:?FRANK_LARGE_MODEL_SHARDS not set}"
|
||||
: "${FRANK_LARGE_CONTAINER:=frank-large-mimo}"
|
||||
: "${FRANK_LARGE_LEGACY_CONTAINERS:=vllm-mimo}"
|
||||
: "${FRANK_LARGE_SERVED_NAME:=frank-large}"
|
||||
: "${FRANK_LARGE_PUBLISH:=7770:7770}"
|
||||
: "${FRANK_LARGE_MAXLEN:=262144}"
|
||||
: "${FRANK_LARGE_SEQS:=4}"
|
||||
: "${FRANK_LARGE_UTIL:=0.45}"
|
||||
: "${FRANK_LARGE_REASONING_PARSER:=qwen3}"
|
||||
: "${FRANK_LARGE_TOOL_PARSER:=qwen3_coder}"
|
||||
: "${FRANK_LARGE_SPEC:=}"
|
||||
: "${FRANK_LARGE_SPEC_TOKENS:=8}"
|
||||
: "${FRANK_LARGE_MODELS_DIR:=${FRANK_LARGE_MODEL_DIR%/*}}"
|
||||
FRANK_LARGE_QUANT="${FRANK_LARGE_QUANT:-}"
|
||||
FRANK_LARGE_QUANT_CONFIG="${FRANK_LARGE_QUANT_CONFIG:-}"
|
||||
GUFO_BIN="$RUNTIME_DIR/bin/gufo"
|
||||
|
||||
# --- gates (fail FAST, before touching the running stack) ------------------
|
||||
# Image gate by REPO DIGEST, not the human-readable ref: rootless podman
|
||||
# resolves the tag from the network, and the whole point of the pin is to
|
||||
# boot exactly these bytes offline. Capture first, grep second — piping
|
||||
# straight into `grep -q` dies of the SIGPIPE race under pipefail.
|
||||
if ! digests="$(podman images --format '{{.RepoDigests}}')" ||
|
||||
! grep -qF "${FRANK_LARGE_IMAGE#*@}" <<<"$digests"; then
|
||||
echo "ERROR: pinned runtime image absent from rootless storage: $FRANK_LARGE_IMAGE" >&2
|
||||
echo "frank-large: fetch it (podman pull $FRANK_LARGE_IMAGE)" >&2
|
||||
# The target GGUF and the drafter are named explicitly because they do not
|
||||
# share a directory any more: the target is a different repository's artifact
|
||||
# and keeps its own long filename, while the DFlash2 drafter is unchanged and
|
||||
# still lives beside the previous target. MODEL_FILE/DRAFTER_FILE default to
|
||||
# the deployed payload, so the env file only has to state them when it moves.
|
||||
: "${MODEL_FILE:=$MODEL_DIR/Qwen3.8-27B-UD-Q4_K_XL.gguf}"
|
||||
: "${DRAFTER_DIR:=/home/dank/models/Qwen3.8-27B-UD-Q8_K_XL}"
|
||||
: "${DRAFTER_FILE:=$DRAFTER_DIR/Qwen3.8-27B-DFlash2-Q4_K_M.gguf}"
|
||||
|
||||
if [[ ! -x "$GUFO_BIN" ]]; then
|
||||
echo "runtime not found at $GUFO_BIN" >&2
|
||||
echo "frank-large: build it: bash scripts/build-gufo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Payload gate: config + weight-map index + the exact shard count. A partial
|
||||
# payload fails the load minutes later, after the port has been freed and
|
||||
# the old server stopped.
|
||||
if [[ ! -f "$FRANK_LARGE_MODEL_DIR/config.json" ]]; then
|
||||
echo "ERROR: payload config missing: $FRANK_LARGE_MODEL_DIR/config.json" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -f "$FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" ]]; then
|
||||
echo "ERROR: weight-map index missing: $FRANK_LARGE_MODEL_DIR/model.safetensors.index.json" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
shards_found=$(find "$FRANK_LARGE_MODEL_DIR" -maxdepth 1 -name 'model-*-of-*.safetensors' | wc -l)
|
||||
if [[ "$shards_found" != "$FRANK_LARGE_MODEL_SHARDS" ]]; then
|
||||
echo "ERROR: payload shard count wrong: $FRANK_LARGE_MODEL_DIR (found $shards_found, want $FRANK_LARGE_MODEL_SHARDS)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-mimo.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$FRANK_LARGE_SPEC" ]]; then
|
||||
: "${FRANK_LARGE_SPEC_DIR:?FRANK_LARGE_SPEC_DIR not set (host dir holding the draft weights)}"
|
||||
# dflash ships ONE model.safetensors; the Qwen3.5-9B MTP head ships as
|
||||
# shards behind model.safetensors.index.json. Either shape is a real
|
||||
# draft; config.json must be present in both, and a directory with
|
||||
# neither weight layout is not a draft at all.
|
||||
if [[ ! -f "$FRANK_LARGE_SPEC_DIR/config.json" ]] ||
|
||||
{ [[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors" ]] &&
|
||||
[[ ! -f "$FRANK_LARGE_SPEC_DIR/model.safetensors.index.json" ]]; }; then
|
||||
echo "ERROR: spec draft absent or partial: $FRANK_LARGE_SPEC_DIR (FRANK_LARGE_SPEC=$FRANK_LARGE_SPEC)" >&2
|
||||
echo "frank-large: re-run scripts/download-frank-large-dflash.sh (or the Qwen3.5-9B fetch for mtp)" >&2
|
||||
exit 1
|
||||
fi
|
||||
# The models tree is mounted read-only at /models; the engine must be
|
||||
# given the CONTAINER path to the draft, not the host path.
|
||||
FRANK_LARGE_SPEC_CONTAINER_DIR="/models/${FRANK_LARGE_SPEC_DIR##*/}"
|
||||
fi
|
||||
|
||||
# The engine needs the RENDER nodes too, not just /dev/kfd: mounting the
|
||||
# whole /dev/dri directory (never individual renderD nodes) is the
|
||||
# verified-working shape (Dankserver 2026-09-29: individual render nodes
|
||||
# enumerate the device but cannot create a context).
|
||||
for dev in /dev/kfd /dev/dri; do
|
||||
if [[ ! -e "$dev" ]]; then
|
||||
echo "ERROR: $dev missing; gfx1151 cannot serve without it" >&2
|
||||
for f in "$MODEL_FILE" "$DRAFTER_FILE"; do
|
||||
if [[ ! -f "$f" ]]; then
|
||||
echo "missing asset $f (run the matching scripts/download-frank-large*.sh)" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
SPEC_ARGS=()
|
||||
if [[ -n "$FRANK_LARGE_SPEC" ]]; then
|
||||
SPEC_ARGS=(--speculative-config "{\"method\":\"$FRANK_LARGE_SPEC\",\"model\":\"$FRANK_LARGE_SPEC_CONTAINER_DIR\",\"num_speculative_tokens\":$FRANK_LARGE_SPEC_TOKENS}")
|
||||
fi
|
||||
# --- env -------------------------------------------------------------------
|
||||
# gufo links against the host ROCm through its RPATH (/opt/rocm ->
|
||||
# /etc/alternatives/rocm), so no LD_LIBRARY_PATH is needed. The device is
|
||||
# already gfx1151 — `gufo diagnose` reports it natively — so no
|
||||
# HSA_OVERRIDE_GFX_VERSION either; the override existed for the prebuilt
|
||||
# llama.cpp runtime and can only misreport the arch here.
|
||||
export HIP_VISIBLE_DEVICES=0
|
||||
|
||||
QUANT_ARGS=()
|
||||
if [[ -n "$FRANK_LARGE_QUANT" ]]; then
|
||||
# Online quantization: vLLM quantizes the BF16 checkpoint at load time
|
||||
# (weights >= 6-bit only; KV is untouched — the context stays bf16).
|
||||
QUANT_ARGS=(--quantization "$FRANK_LARGE_QUANT")
|
||||
if [[ -n "$FRANK_LARGE_QUANT_CONFIG" ]]; then
|
||||
QUANT_ARGS+=(--quantization-config "$FRANK_LARGE_QUANT_CONFIG")
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# Remove stale container identities BEFORE starting: a crashed unit can
|
||||
# leave the name registered forever after, and the bench-window identity
|
||||
# (vllm-mimo) holds the GPU and port when a window ends without teardown.
|
||||
for name in $FRANK_LARGE_LEGACY_CONTAINERS "$FRANK_LARGE_CONTAINER"; do
|
||||
if podman ps -a --format '{{.Names}}' | grep -qx "$name"; then
|
||||
echo "removing stale container: $name"
|
||||
podman rm -f "$name" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
|
||||
# --device /dev/mem exists for the fleet's other GPU containers; the GTT
|
||||
# aperture (114 GiB) needs no size knob. Models mount read-only wholesale so
|
||||
# spec drafts resolve from disk. THE PATH HANDED TO vllm IS THE CONTAINER
|
||||
# PATH: the engine runs inside this container, and a host path that is not
|
||||
# mounted at the same location dies as "Repo id must be in the form
|
||||
# 'namespace/repo_name'" (2026-10-01 first boot, 51 crash-loop restarts).
|
||||
MODEL_CONTAINER_DIR="/models/${FRANK_LARGE_MODEL_DIR##*/}"
|
||||
podman run -d \
|
||||
--name "$FRANK_LARGE_CONTAINER" \
|
||||
--device /dev/kfd --device /dev/dri --device /dev/mem \
|
||||
--security-opt seccomp=unconfined --group-add keep-groups \
|
||||
--ulimit memlock=-1:-1 \
|
||||
-v "$FRANK_LARGE_MODELS_DIR:/models:ro" \
|
||||
-v /home/dank/models/hf-cache:/root/.cache/huggingface \
|
||||
-e HSA_ENABLE_SDMA=0 \
|
||||
-p "$FRANK_LARGE_PUBLISH" \
|
||||
--entrypoint vllm \
|
||||
"$FRANK_LARGE_IMAGE" \
|
||||
serve "$MODEL_CONTAINER_DIR" \
|
||||
--served-model-name "$FRANK_LARGE_SERVED_NAME" \
|
||||
--host 0.0.0.0 --port "${FRANK_LARGE_PUBLISH##*:}" \
|
||||
--dtype bfloat16 \
|
||||
--max-model-len "$FRANK_LARGE_MAXLEN" \
|
||||
--max-num-seqs "$FRANK_LARGE_SEQS" \
|
||||
--gpu-memory-utilization "$FRANK_LARGE_UTIL" \
|
||||
--reasoning-parser "$FRANK_LARGE_REASONING_PARSER" \
|
||||
--enable-auto-tool-choice \
|
||||
--tool-call-parser "$FRANK_LARGE_TOOL_PARSER" \
|
||||
--language-model-only \
|
||||
--trust-remote-code \
|
||||
"${QUANT_ARGS[@]}" \
|
||||
"${SPEC_ARGS[@]}"
|
||||
|
||||
# Block on the container so a container death returns non-zero from ExecStart
|
||||
# and the unit's Restart=always fires. systemd stop goes through the unit's
|
||||
# ExecStop (podman stop -t); podman wait unblocks when the container is gone.
|
||||
exec podman wait "$FRANK_LARGE_CONTAINER"
|
||||
# --- the serve command ---------------------------------------------------
|
||||
# Server options precede the subcommand, model options follow it.
|
||||
exec "$GUFO_BIN" serve \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--sessions "$SESSIONS" \
|
||||
llm \
|
||||
--model "$MODEL_FILE" \
|
||||
--served-model-name "$ALIAS" \
|
||||
--context "$CONTEXT_SIZE" \
|
||||
--speculative dflash2 \
|
||||
--dflash-model "$DRAFTER_FILE" \
|
||||
--draft-tokens "$DRAFT_TOKENS" \
|
||||
--draft-policy "$DRAFT_POLICY" \
|
||||
--temperature 0 \
|
||||
--think auto \
|
||||
--preserve-thinking on
|
||||
Reference in new issue
Block a user