Frank
a9b94a79eb
Discovery, not guesswork: dump the GGUF v3 headers of the fetched parts (scripts/dump-gguf.py, added — a dependency-free reader that never touches tensor data, so a 50 GiB part scans in milliseconds) before the first load. What the dump says (parts 2-3, 2026-09-26): - The 51B-param PLE n-gram table is ONE tensor: per_layer_token_embd. weight, dims [160, 320001536], Q8_0, 54.4 GiB — exactly the whole of part 3. It is the same tensor the shared MTP head borrows, so the =CPU override is required for placement AND for the draft load (GPU-resident borrow: "invalid vector subscript"). - The model is qwen4exp, 512 experts, top-10 routing (qwen4exp.ple.ngram_size=3, heads_per_ngram=8). - Part 1 is metadata-only (ntensors=0); parts 2-6 carry tensors. The placeholder pattern ngram_embeddings matched nothing — dropped from OT_OVERRIDES (env, serve-script default, seam, AGENTS.md, runbook, LLM index) before it could masquerade as a working override at load time. Deployed set: exps=CPU,per_layer_token_embd.weight=CPU. Suite: 346 green.
135 lines
5.2 KiB
Bash
Executable File
135 lines
5.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve unsloth/Qwen3.8-Flash-Next-GGUF at Q8_0 as frank-primary on
|
|
# Dankserver's two Radeon AI PRO R9700s (gfx1201), using the unsloth
|
|
# b11030-mix prebuilt (ROCm gfx120X, bundled runtime).
|
|
#
|
|
# Shape: one stream, the model's full 262144 context, all non-overridden
|
|
# tensors on the GPUs (--n-gpu-layers 999 across both cards), with the
|
|
# routed experts and the 51B-param n-gram embedding table computed in host
|
|
# RAM via --override-tensor (the x4-pair law: streaming over PCIe loses,
|
|
# computing in RAM does not care). mmap stays ON (the PLE table must stay
|
|
# page-cache-backed; --no-mmap forces it into anonymous memory and the
|
|
# cgroup kill is silent).
|
|
#
|
|
# Spec chain: draft-mtp (the shared Q8_0 head, loaded via -md) chained with
|
|
# the draftless ngram-mod matcher. The shared head borrows the target's
|
|
# embedding tensors, which is why per_layer_token_embd.weight must be
|
|
# CPU-resident (see frank-primary.env). depth 2.
|
|
#
|
|
# Vision: the BF16 projector, loaded explicitly via --mmproj (this build
|
|
# never autoloads projectors — the frank-medium 2026-09-17 trap: a server
|
|
# started without it comes up healthy and text-only, and answers image
|
|
# requests with 500).
|
|
#
|
|
# Template: the GGUF's embedded chat template via --jinja. The Qwen3.8
|
|
# family template gates reasoning_effort to {xhigh, medium, low} — the
|
|
# harness folds its levels onto that set in models.yml
|
|
# (reasoningEffortMap). If the embedded template ever falls back to
|
|
# peg-native and bloats prompts (~73,600 tokens for a ~250-token request),
|
|
# switch to --chat-template-file scripts/qwen-sharp-v22.4.0.jinja the way
|
|
# frank-large does.
|
|
#
|
|
# Tunables come from frank-primary.env unless FRANK_PRIMARY_ENV_FILE
|
|
# overrides it; anything not set there falls through to the defaults below.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_PRIMARY_ENV_FILE:-$SCRIPT_DIR/frank-primary.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) ------------------
|
|
: "${RUNTIME_DIR:=/home/dank/GitPackages/unsloth-llama-b11030}"
|
|
: "${MODEL_ROOT:=/home/dank/models/Qwen3.8-Flash-Next-GGUF}"
|
|
: "${BIND_HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CONTEXT_SIZE:=262144}"
|
|
: "${PARALLEL_SLOTS:=1}"
|
|
: "${SPEC_DRAFT_N_MAX:=2}"
|
|
: "${NGL:=999}"
|
|
: "${OT_OVERRIDES:=exps=CPU,per_layer_token_embd.weight=CPU}"
|
|
: "${TENSOR_SPLIT:=1,1}"
|
|
: "${SPEC_DRAFT_DEVICE:=ROCm0}"
|
|
: "${BATCH_SIZE:=8192}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
: "${CACHE_RAM:=16384}"
|
|
|
|
SERVER_BIN="$RUNTIME_DIR/llama-server"
|
|
MODEL_PARTS=(
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00001-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00002-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00003-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00004-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00005-of-00006.gguf"
|
|
"$MODEL_ROOT/Q8_0/Qwen3.8-Flash-Next-Q8_0-00006-of-00006.gguf"
|
|
)
|
|
DRAFTER_FILE="$MODEL_ROOT/MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"
|
|
MMPROJ_FILE="$MODEL_ROOT/mmproj-BF16.gguf"
|
|
|
|
if [[ ! -x "$SERVER_BIN" ]]; then
|
|
echo "runtime not found at $SERVER_BIN" >&2
|
|
echo " fetch the unsloth b11030-mix gfx120X prebuilt (see the runbook)" >&2
|
|
exit 1
|
|
fi
|
|
# Every part is checked: part 1 is the -m target and parts 2-6 are followed
|
|
# by the split-file loader, so a missing tail part fails at LOAD time with
|
|
# a confusing partial-mmap error unless it is caught here.
|
|
for f in "${MODEL_PARTS[@]}" "$DRAFTER_FILE" "$MMPROJ_FILE"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "missing asset: $f" >&2
|
|
echo " run scripts/download-frank-primary.sh on this host" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
# --- env (the unsloth build's bundled ROCm runtime) -----------------------
|
|
# Both R9700s (gfx1201, HIP devices 0 and 1); the host's Raphael iGPU is
|
|
# device 2 and must stay hidden from the enumeration.
|
|
export HIP_VISIBLE_DEVICES=0,1
|
|
export LD_LIBRARY_PATH="$RUNTIME_DIR:${LD_LIBRARY_PATH:-}"
|
|
|
|
# --- the serve command ---------------------------------------------------
|
|
ARGS=(
|
|
--host "$BIND_HOST"
|
|
--port "$PORT"
|
|
--model "${MODEL_PARTS[0]}"
|
|
--alias frank-primary
|
|
--ctx-size "$CONTEXT_SIZE"
|
|
--parallel "$PARALLEL_SLOTS"
|
|
--n-gpu-layers "$NGL"
|
|
--tensor-split "$TENSOR_SPLIT"
|
|
--jinja
|
|
--reasoning auto
|
|
--reasoning-format deepseek
|
|
--reasoning-budget -1
|
|
--flash-attn on
|
|
--batch-size "$BATCH_SIZE"
|
|
--ubatch-size "$UBATCH_SIZE"
|
|
--cache-ram "$CACHE_RAM"
|
|
# Vision: explicit, never autoloaded by this build
|
|
--mmproj "$MMPROJ_FILE"
|
|
# Spec chain: the shared MTP head + the draftless ngram matcher
|
|
--spec-type draft-mtp,ngram-mod
|
|
--model-draft "$DRAFTER_FILE"
|
|
--spec-draft-n-max "$SPEC_DRAFT_N_MAX"
|
|
--spec-draft-device "$SPEC_DRAFT_DEVICE"
|
|
# Greedy server default: deterministic e2e, and the harness overrides
|
|
# sampling per-request (models.yml compat.extraBody carries Qwen's
|
|
# recommended temp 1.0 / top_p 0.95 / top_k 20).
|
|
--temp 0
|
|
--top-p 1.0
|
|
--top-k 0
|
|
--min-p 0.0
|
|
)
|
|
|
|
# Comma-separated <pattern>=<backend> pairs -> repeated --override-tensor.
|
|
IFS=',' read -ra _ots <<< "$OT_OVERRIDES"
|
|
for ot in "${_ots[@]}"; do
|
|
ARGS+=(--override-tensor "$ot")
|
|
done
|
|
|
|
exec "$SERVER_BIN" "${ARGS[@]}"
|