Frank
de85654b76
- CONTEXT_SIZE=1048576 / PARALLEL_SLOTS=4: four conversations, each the full native 262144 window (per-slot window unchanged; slots divide the pool, they never multiply it). - CACHE_TYPE_V=q8_0 / CACHE_TYPE_K=f16: main KV drops 24,576 -> 18,816 B/token (-23.4%); the shared MTP drafter's KV ignores the V type (measured: 2,048 MiB draft pool = the f16 law at 1,048,576 cells). - OT_OVERRIDES: blocks 0-3 on ROCm0 and 45-47 on ROCm1 stay GPU-resident (2.42/2.49 GiB per block, measured); the rest stay CPU-computed. Measured free under load: ~4.9 GB card0 / ~3.4 GB card1; a 5th block on card0 lands at 2.27 GB, under the 3 GB checkpoint floor, so blocks are atomic and 4 is the safe fill. - serve script: --cache-type-k/v env-wired tunables; ONE joined --override-tensor flag (b11030 deprecates repeated flags, and the loader is first-match-wins, so the specific patterns must precede the broad exps=CPU sweep). - seams: the window test asserts per-slot = 262144 at 4 slots; new KV-type test; OT exact-value assertion pins the ordering law.
141 lines
5.5 KiB
Bash
Executable File
141 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 as frank-primary on
|
|
# Dankserver's two Radeon AI PRO R9700s (gfx1201), using the unsloth
|
|
# b11030-mix prebuilt (ROCm gfx120X, bundled runtime).
|
|
#
|
|
# Shape: one stream, the model's full 262144 context, all non-overridden
|
|
# tensors on the GPUs (--n-gpu-layers 999 across both cards), with the
|
|
# routed experts and the 51B-param n-gram embedding table computed in host
|
|
# RAM via --override-tensor (the x4-pair law: streaming over PCIe loses,
|
|
# computing in RAM does not care). mmap stays ON (the PLE table must stay
|
|
# page-cache-backed; --no-mmap forces it into anonymous memory and the
|
|
# cgroup kill is silent).
|
|
#
|
|
# Spec chain: draft-mtp (the shared Q8_0 head, loaded via -md) chained with
|
|
# the draftless ngram-mod matcher. The shared head borrows the target's
|
|
# embedding tensors, which is why per_layer_token_embd.weight must be
|
|
# CPU-resident (see frank-primary.env). depth 2.
|
|
#
|
|
# Vision: the BF16 projector, loaded explicitly via --mmproj (this build
|
|
# never autoloads projectors — the frank-medium 2026-09-17 trap: a server
|
|
# started without it comes up healthy and text-only, and answers image
|
|
# requests with 500).
|
|
#
|
|
# Template: the GGUF's embedded chat template via --jinja. The Qwen3.8
|
|
# family template gates reasoning_effort to {xhigh, medium, low} — the
|
|
# harness folds its levels onto that set in models.yml
|
|
# (reasoningEffortMap). If the embedded template ever falls back to
|
|
# peg-native and bloats prompts (~73,600 tokens for a ~250-token request),
|
|
# switch to --chat-template-file scripts/qwen-sharp-v22.4.0.jinja the way
|
|
# frank-large does.
|
|
#
|
|
# Tunables come from frank-primary.env unless FRANK_PRIMARY_ENV_FILE
|
|
# overrides it; anything not set there falls through to the defaults below.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
|
: "${MODEL_ROOT:=/home/dank/models/Qwen3.8-Flash-Next-GGUF}"
|
|
: "${BIND_HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=262144}"
|
|
: "${PARALLEL_SLOTS:=1}"
|
|
: "${SPEC_DRAFT_N_MAX:=2}"
|
|
: "${NGL:=999}"
|
|
: "${OT_OVERRIDES:=exps=CPU,per_layer_token_embd.weight=CPU}"
|
|
: "${TENSOR_SPLIT:=1,1}"
|
|
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
|
: "${BATCH_SIZE:=8192}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${VERBOSITY:=4}"
|
|
|
|
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
|
MODEL_PARTS=(
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00002-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00003-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00004-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00005-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00006-of-00006.gguf"
|
|
)
|
|
DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
|
MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf"
|
|
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "runtime not found at $SERVER_BIN" >&2
|
|
echo " fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2
|
|
exit 1
|
|
fi
|
|
# Every part is checked: part 1 is the -m target and parts 2-6 are followed
|
|
# by the split-file loader, so a missing tail part fails at LOAD time with
|
|
# a confusing partial-mmap error unless it is caught here.
|
|
for f in "${MODEL_PARTS[@]}" "$DRAFTER_FILE" "$MMPROJ_FILE"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "missing asset: $f" >&2
|
|
echo " run scripts/download-frank-primary.sh on this host" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# --- env (the unsloth build's bundled ROCm runtime) -----------------------
|
|
# Both R9700s (gfx1201, HIP devices 0 and 1); the host's Raphael iGPU is
|
|
# device 2 and must stay hidden from the enumeration.
|
|
export HIP_VISIBLE_DEVICES=0,1
|
|
export LD_LIBRARY_PATH="$RUNTIME_DIR:${LD_LIBRARY_PATH:-}"
|
|
|
|
# --- the serve command ---------------------------------------------------
|
|
ARGS=(
|
|
--host "$BIND_HOST"
|
|
--port "$PORT"
|
|
--model "${MODEL_PARTS[0]}"
|
|
--alias frank-primary
|
|
--ctx-size "$CONTEXT_SIZE"
|
|
--parallel "$PARALLEL_SLOTS"
|
|
-lv "$VERBOSITY"
|
|
--metrics
|
|
--n-gpu-layers "$NGL"
|
|
--tensor-split "$TENSOR_SPLIT"
|
|
--jinja
|
|
--reasoning auto
|
|
--reasoning-format deepseek
|
|
--reasoning-budget -1
|
|
--cache-type-k "$CACHE_TYPE_K"
|
|
--cache-type-v "$CACHE_TYPE_V"
|
|
--batch-size "$BATCH_SIZE"
|
|
--ubatch-size "$UBATCH_SIZE"
|
|
--cache-ram "$CACHE_RAM"
|
|
# Vision: explicit, never autoloaded by this build
|
|
--mmproj "$MMPROJ_FILE"
|
|
# Spec chain: the shared MTP head + the draftless ngram matcher
|
|
--spec-type draft-mtp,ngram-mod
|
|
--model-draft "$DRAFTER_FILE"
|
|
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
|
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
|
# Greedy server default: deterministic e2e, and the harness overrides
|
|
# sampling per-request (models.yml compat.extraBody carries Qwen's
|
|
# recommended temp 1.0 / top_p 0.95 / top_k 20).
|
|
--temp 0
|
|
--top-p 1.0
|
|
--top-k 0
|
|
--min-p 0.0
|
|
)
|
|
|
|
# Comma-separated <pattern>=<backend> pairs, ONE flag: b11030 logs
|
|
# DEPRECATED for repeated --override-tensor and wants the joined list.
|
|
# The loader applies the FIRST matching pattern per tensor, so the
|
|
# specific block patterns must precede the broad exps=CPU sweep in
|
|
# frank-primary.env.
|
|
ARGS+=(--override-tensor "$OT_OVERRIDES")
|
|
|
|
exec "$SERVER_BIN" "${ARGS[@]}"
|