Frank
8551bb006b
frank-small: llama.cpp CUDA (master, sm_86) on MiMo Q6_K GGUF, 2 x 262144, asymmetric KV (f16 K + q8_0 V, flash-attn pinned for the V quant)
75 lines
2.9 KiB
Bash
Executable File
75 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on
|
|
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
|
#
|
|
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
|
|
# safetensors to the llama.cpp GGUF path, because llama.cpp is the only
|
|
# runtime on this host that can serve the MiMo qwen35 hybrid with
|
|
# ASYMMETRIC KV: --cache-type-k f16 (unquantized keys) + --cache-type-v
|
|
# q8_0 (quantized values), which is Frank's explicit KV contract for this
|
|
# endpoint. SGLang's --kv-cache-dtype only moves K and V together.
|
|
#
|
|
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
|
|
# DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
|
|
# the DeltaNet state is per-slot fixed size. GGUF path is TEXT-ONLY: no
|
|
# mmproj, the in-checkpoint vision tower does not exist here (models.yml
|
|
# declares frank-small text-only since this swap).
|
|
#
|
|
# Sampling: llama.cpp takes it per request (the fleet harness sends its
|
|
# recipe from models.yml); greedy is the harness default.
|
|
#
|
|
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
|
: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
|
|
: "${ALIAS:=frank-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7771}"
|
|
# ctx-size is the POOL llama.cpp divides across slots: 524288 / 2 = TWO
|
|
# full native 262144 windows.
|
|
: "${CTX_SIZE:=524288}"
|
|
: "${PARALLEL:=2}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=q8_0}"
|
|
: "${NGL:=99}"
|
|
|
|
# Fail loudly rather than half-loading: a missing file is a download that
|
|
# has not finished, and Restart=on-failure would turn it into a crash loop
|
|
# that reads like a runtime fault.
|
|
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
|
|
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
|
done
|
|
|
|
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
|
# open after boot; make sure it exists before the server needs it.
|
|
nvidia-modprobe -u -c=0 2>/dev/null || true
|
|
|
|
# q8_0 on V REQUIRES flash attention (the quantized-value kernels only exist
|
|
# on the FA path); an FA-less build silently falls back and the KV contract
|
|
# in frank-small.env becomes a lie, so pin the flag pair here.
|
|
exec "$LLAMA_BIN" \
|
|
--model "$MODEL_FILE" \
|
|
--alias "$ALIAS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" \
|
|
--ctx-size "$CTX_SIZE" \
|
|
--parallel "$PARALLEL" \
|
|
--n-gpu-layers "$NGL" \
|
|
--flash-attn on \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
--jinja \
|
|
--reasoning auto \
|
|
--reasoning-format deepseek
|