Frank
6ec0b1a85d
The freed third slot funded exact V; the remaining ~3.5 GiB of honest nvidia-smi free (3,571, not the 4,046 total-minus-used I quoted - the driver holds ~450 MiB of reserve) funds the +875 MiB of Q6_K weights. Projected landing ~2.7 GiB free, inside the 2-3 GiB floor; measure on boot, step the pool one 12,288-token increment if it lands under 2 GiB. - payload dir MiMo-Q6K/ made self-contained (mmproj hardlinked from MiMo-Q5KM/, link-count-2 verified) - env/serve/deployer/bats repointed; download script now fetches and verifies Q6_K (7,793,710,624 B, ef96d05a...) with Q5_K_M documented as the rollback; payload sha verified live on the host before deploy - seam green 14/14 against the stub engine
97 lines
4.1 KiB
Bash
Executable File
97 lines
4.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve bartowski/MiMo-V2.6-Distill-Qwen-9B-GGUF (Q6_K) as frank-small on
|
|
# Dankclaw's RTX 3090 via llama.cpp (master build, CUDA sm_86).
|
|
#
|
|
# 2026-10-01 night (Frank): swapped from SGLang + INT8-W8A16-AutoRound
|
|
# safetensors to the llama.cpp GGUF path, because llama.cpp is the only
|
|
# runtime on this host that moves K and V independently — the asymmetric
|
|
# f16-K + q8_0-V contract that endpoint was built for. 2026-10-03 (Frank):
|
|
# the pair is now f16/f16 — two wide slots freed the VRAM, so the values are
|
|
# unquantized too. The per-tensor flags stay explicit either way.
|
|
#
|
|
# Architecture: qwen3_5 hybrid — 32 layers, 8 full-attention + 24 Gated
|
|
# DeltaNet, native 262144 window. KV lives only in the 8 attention layers;
|
|
# the DeltaNet state is per-slot fixed size. Vision comes from the repo's
|
|
# separate projector (the Q6_K GGUF carries no `v.*` tensors at all), so
|
|
# --mmproj is what gives this endpoint eyes; MMPROJ_FILE="" is the documented
|
|
# text-only opt-out.
|
|
#
|
|
# Sampling: llama.cpp takes it per request (the fleet harness sends its
|
|
# recipe from models.yml); greedy is the harness default.
|
|
#
|
|
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/llama-master/build-cuda/bin/llama-server}"
|
|
: "${MODEL_FILE:=/home/dank/models/MiMo-Q6K/MiMo-V2.6-Distill-Qwen-9B-Q6_K.gguf}"
|
|
: "${ALIAS:=frank-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7771}"
|
|
# ctx-size is the POOL llama.cpp divides across slots: 401408 / 2 gives
|
|
# 200,704 per slot — two independent conversations, each one wide enough to
|
|
# hold a whole frank-primary-sized prompt in a single shot (the native 262,144
|
|
# no longer fits twice on a 24 GiB card once the vision tower is loaded;
|
|
# frank-small.env carries the three measurements that set this number).
|
|
: "${CTX_SIZE:=401408}"
|
|
: "${PARALLEL:=2}"
|
|
# Compute-buffer geometry. Before 2026-10-03 this endpoint never passed either
|
|
# flag, so it ran on llama.cpp's own defaults (2048/512); they are tunables now
|
|
# because the 1024 ubatch is what this shape was booted with.
|
|
: "${BATCH_SIZE:=2048}"
|
|
: "${UBATCH_SIZE:=1024}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${NGL:=99}"
|
|
# Vision projector. No colon on purpose: an explicitly empty value means
|
|
# "text-only, on purpose" and must NOT be replaced by the default path.
|
|
: "${MMPROJ_FILE=/home/dank/models/MiMo-Q6K/mmproj-MiMo-V2.6-Distill-Qwen-9B-f16.gguf}"
|
|
|
|
# Fail loudly rather than half-loading: a missing file is a download that
|
|
# has not finished, and Restart=always would turn it into a crash loop that
|
|
# reads like a runtime fault.
|
|
for path in "$LLAMA_BIN" "$MODEL_FILE"; do
|
|
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
|
done
|
|
EXTRA_ARGS=()
|
|
if [[ -n "$MMPROJ_FILE" ]]; then
|
|
[[ -e "$MMPROJ_FILE" ]] || { echo "frank-small: missing $MMPROJ_FILE" >&2; exit 1; }
|
|
EXTRA_ARGS+=(--mmproj "$MMPROJ_FILE")
|
|
fi
|
|
|
|
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
|
# open after boot; make sure it exists before the server needs it.
|
|
nvidia-modprobe -u -c=0 2>/dev/null || true
|
|
|
|
# Flash attention stays pinned on: it is the fast path for f16 KV, and it is
|
|
# the only path that could serve quantized V at all — so if CACHE_TYPE_V is
|
|
# ever set back to q8_0 this flag must not be lost, or the build falls back
|
|
# silently and the KV contract in frank-small.env becomes a lie.
|
|
ARGS=(
|
|
--model "$MODEL_FILE"
|
|
--alias "$ALIAS"
|
|
--host "$BIND_ADDR"
|
|
--port "$LISTEN_PORT"
|
|
--ctx-size "$CTX_SIZE"
|
|
--parallel "$PARALLEL"
|
|
--batch-size "$BATCH_SIZE"
|
|
--ubatch-size "$UBATCH_SIZE"
|
|
--n-gpu-layers "$NGL"
|
|
--flash-attn on
|
|
--cache-type-k "$CACHE_TYPE_K"
|
|
--cache-type-v "$CACHE_TYPE_V"
|
|
--jinja
|
|
--reasoning auto
|
|
--reasoning-format deepseek
|
|
)
|
|
exec "$LLAMA_BIN" "${ARGS[@]}" ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}
|