Frank
dbd764a493
CTX_SIZE=1048576 / PARALLEL=4: every slot gets the model's full trained window. Measured fit arithmetic on the 3090: 1 full-context slot idles at 11,996 MiB, 4 at 22,346 -> 3,450 MiB per slot, ~8,546 MiB fixed. A 5th slot projects 25.8 GiB past the 24,576 MiB card, and the KV pool allocates upfront, so 4 is the ceiling. Explicit CTX_SIZE bypasses --fit on purpose: fit maximises the TOTAL pool and would divide a smaller window across slots at parallel>1. The slot-grid seam now pins both the full-window division and the budget inequality so a future capacity bump fails pre-deploy, not in the boot allocation. Audio verified while measuring: the mmproj's audio encoder transcribes input_audio parts verbatim (JFK WAV sample). Header comments updated.
133 lines
5.0 KiB
Bash
Executable File
133 lines
5.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
|
|
# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
|
|
#
|
|
# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
|
|
# weights plus the BF16 vision projector and q8_0 KV inside the card's
|
|
# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
|
|
# belongs to the halogen engine that serves frank-large.
|
|
#
|
|
# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
|
|
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
|
|
# passed via --model-draft with --spec-type draft-mtp: the draft context
|
|
# loads the assistant model. The same --spec-type value chains the
|
|
# draftless ngram-mod matcher, which attaches unconditionally; two
|
|
# context-backed drafters cannot share one process, so this chain is the
|
|
# only shape that carries both.
|
|
#
|
|
# Context: CTX_SIZE is set explicitly to PARALLEL x 262144 so every slot
|
|
# gets the model's FULL window; --fit (the CTX_SIZE-unset branch below)
|
|
# maximises the TOTAL pool instead, which would divide a smaller window
|
|
# across slots. The 4-slot layout is measured to fit the card (see the
|
|
# SIZING note in frank-small.env); more slots would fail the KV allocation.
|
|
#
|
|
# Vision + audio: --mmproj loads the BF16 projector, which also carries
|
|
# an audio encoder — input_audio content parts transcribe verbatim
|
|
# (verified 2026-09-17). The fleet's image path lives here, which is why
|
|
# halogen no longer sets HALOGEN_VISION_TOWER.
|
|
#
|
|
# Sampling: greedy by default (the fleet contract). Clients that send their own
|
|
# temperature get their own values, and the harness overrides per model from
|
|
# models.yml. --jinja is pinned even though this build defaults it on: the
|
|
# model's own template supplies reasoning_effort, preserve_thinking and the
|
|
# tool-call grammar, and a future build that flips that default must not
|
|
# silently take tools away.
|
|
#
|
|
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
|
|
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
|
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
|
|
: "${DRAFT_FILE:=}"
|
|
: "${ALIAS:=frank-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7771}"
|
|
: "${CTX_SIZE:=}"
|
|
: "${PARALLEL:=1}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${BATCH_SIZE:=2048}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
: "${NUM_THREADS:=8}"
|
|
: "${THREAD_BATCH:=8}"
|
|
: "${NGL:=999}"
|
|
: "${FLASH_ATTN:=on}"
|
|
: "${SPEC_TYPE:=draft-mtp}"
|
|
: "${TEMPERATURE:=0.0}"
|
|
: "${REASONING_FORMAT:=deepseek}"
|
|
: "${REASONING_BUDGET:=-1}"
|
|
|
|
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
|
# is a download that has not finished, and the unit's restart policy would
|
|
# otherwise turn that into a crash loop that reads like a runtime fault.
|
|
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
|
|
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
|
done
|
|
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
|
[[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
|
|
fi
|
|
|
|
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
|
# open after boot; make sure it exists before the server needs it.
|
|
nvidia-modprobe -u -c=0 2>/dev/null || true
|
|
|
|
MMPROJ_ARGS=()
|
|
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
|
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
|
fi
|
|
|
|
SPEC_ARGS=()
|
|
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
|
SPEC_ARGS=(--spec-type "$SPEC_TYPE")
|
|
fi
|
|
|
|
DRAFT_ARGS=()
|
|
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
|
DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
|
|
fi
|
|
|
|
# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
|
|
# whatever the weights, projector, draft context and compute buffers leave
|
|
# on the card; the engine prints the chosen value at startup.
|
|
if [[ -n "${CTX_SIZE:-}" ]]; then
|
|
CTX_ARGS=(--ctx-size "$CTX_SIZE")
|
|
else
|
|
CTX_ARGS=(--fit on)
|
|
fi
|
|
|
|
exec "$LLAMA_BIN" \
|
|
--model "$MODEL_DIR/$MODEL_FILE" \
|
|
"${MMPROJ_ARGS[@]}" \
|
|
"${DRAFT_ARGS[@]}" \
|
|
--alias "$ALIAS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" \
|
|
--gpu-layers "$NGL" \
|
|
--flash-attn "$FLASH_ATTN" \
|
|
"${CTX_ARGS[@]}" \
|
|
--parallel "$PARALLEL" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--threads "$NUM_THREADS" \
|
|
--threads-batch "$THREAD_BATCH" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
"${SPEC_ARGS[@]}" \
|
|
--jinja \
|
|
--metrics \
|
|
--temp "$TEMPERATURE" \
|
|
--reasoning-format "$REASONING_FORMAT" \
|
|
--reasoning-budget "$REASONING_BUDGET"
|