Frank
4b7de59dc8
The Qwen3.8-27B IQ3_S file carried its own MTP head; Gemma4's ships as a separate assistant file (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers) that loads via --model-draft and shares the target's last layers' KV. The draftless ngram-mod matcher chains behind it in the same --spec-type value — two context-backed drafters cannot share one process, so this chain is the only shape that carries both. KV is q8_0 (16 KiB/token/layer vs 32 at f16) and CTX_SIZE is unset so --fit sizes the pool against what the 7.4 GiB of payloads leave on the 24 GiB card. build-cuda was rebuilt with -DGGML_CUDA_FA_ALL_QUANTS=ON, which quantized KV on CUDA requires. The three seams that pinned the Qwen contract (MTP-in-file, the Qwen mmproj name, the f16 KV arithmetic) now pin the Gemma4 contract instead, and the deploy handler's presence check points at the new payload.
130 lines
4.8 KiB
Bash
Executable File
130 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
|
|
# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
|
|
#
|
|
# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
|
|
# weights plus the BF16 vision projector and q8_0 KV inside the card's
|
|
# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
|
|
# belongs to the halogen engine that serves frank-large.
|
|
#
|
|
# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
|
|
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
|
|
# passed via --model-draft with --spec-type draft-mtp: the draft context
|
|
# loads the assistant model. The same --spec-type value chains the
|
|
# draftless ngram-mod matcher, which attaches unconditionally; two
|
|
# context-backed drafters cannot share one process, so this chain is the
|
|
# only shape that carries both.
|
|
#
|
|
# Context: CTX_SIZE is left unset and --fit sizes the KV pool (q8_0 K and
|
|
# V) against what the weights, projector, draft context and compute
|
|
# buffers leave on the card; the chosen value lands in the startup log.
|
|
#
|
|
# Vision: --mmproj loads the BF16 projector (it also carries an audio
|
|
# encoder; vision is the fleet contract). The fleet's image path lives
|
|
# here, which is why halogen no longer sets HALOGEN_VISION_TOWER.
|
|
#
|
|
# Sampling: greedy by default (the fleet contract). Clients that send their own
|
|
# temperature get their own values, and the harness overrides per model from
|
|
# models.yml. --jinja is pinned even though this build defaults it on: the
|
|
# model's own template supplies reasoning_effort, preserve_thinking and the
|
|
# tool-call grammar, and a future build that flips that default must not
|
|
# silently take tools away.
|
|
#
|
|
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
|
|
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
|
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
|
|
: "${DRAFT_FILE:=}"
|
|
: "${ALIAS:=frank-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7771}"
|
|
: "${CTX_SIZE:=}"
|
|
: "${PARALLEL:=1}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${BATCH_SIZE:=2048}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
: "${NUM_THREADS:=8}"
|
|
: "${THREAD_BATCH:=8}"
|
|
: "${NGL:=999}"
|
|
: "${FLASH_ATTN:=on}"
|
|
: "${SPEC_TYPE:=draft-mtp}"
|
|
: "${TEMPERATURE:=0.0}"
|
|
: "${REASONING_FORMAT:=deepseek}"
|
|
: "${REASONING_BUDGET:=-1}"
|
|
|
|
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
|
# is a download that has not finished, and the unit's restart policy would
|
|
# otherwise turn that into a crash loop that reads like a runtime fault.
|
|
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
|
|
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
|
done
|
|
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
|
[[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
|
|
fi
|
|
|
|
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
|
# open after boot; make sure it exists before the server needs it.
|
|
nvidia-modprobe -u -c=0 2>/dev/null || true
|
|
|
|
MMPROJ_ARGS=()
|
|
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
|
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
|
fi
|
|
|
|
SPEC_ARGS=()
|
|
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
|
SPEC_ARGS=(--spec-type "$SPEC_TYPE")
|
|
fi
|
|
|
|
DRAFT_ARGS=()
|
|
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
|
DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
|
|
fi
|
|
|
|
# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
|
|
# whatever the weights, projector, draft context and compute buffers leave
|
|
# on the card; the engine prints the chosen value at startup.
|
|
if [[ -n "${CTX_SIZE:-}" ]]; then
|
|
CTX_ARGS=(--ctx-size "$CTX_SIZE")
|
|
else
|
|
CTX_ARGS=(--fit on)
|
|
fi
|
|
|
|
exec "$LLAMA_BIN" \
|
|
--model "$MODEL_DIR/$MODEL_FILE" \
|
|
"${MMPROJ_ARGS[@]}" \
|
|
"${DRAFT_ARGS[@]}" \
|
|
--alias "$ALIAS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" \
|
|
--gpu-layers "$NGL" \
|
|
--flash-attn "$FLASH_ATTN" \
|
|
"${CTX_ARGS[@]}" \
|
|
--parallel "$PARALLEL" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--threads "$NUM_THREADS" \
|
|
--threads-batch "$THREAD_BATCH" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
"${SPEC_ARGS[@]}" \
|
|
--jinja \
|
|
--metrics \
|
|
--temp "$TEMPERATURE" \
|
|
--reasoning-format "$REASONING_FORMAT" \
|
|
--reasoning-budget "$REASONING_BUDGET"
|