Frank
4d13333da7
Both were implicit build defaults (3 and 50); the throughput sweep needs to move them without editing the exec line each time. Defaults preserve current behavior exactly. Additive only.
139 lines
5.3 KiB
Bash
Executable File
139 lines
5.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve HauhauCS/Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced (Q4_K_M) as
|
|
# frank-small on Dankclaw's RTX 3090 via mainline llama.cpp.
|
|
#
|
|
# Topology: the whole model lives in VRAM (-ngl 999) -- 6.87 GiB of Q4_K_M
|
|
# weights plus the BF16 vision projector and q8_0 KV inside the card's
|
|
# 24 GiB. The host CPU only tokenizes and samples here; this box's CPU
|
|
# belongs to the halogen engine that serves frank-large.
|
|
#
|
|
# MTP: this model's Multi-Token Prediction head ships as a SEPARATE file
|
|
# (mtp-gemma-4-12B-it.gguf, arch gemma4-assistant, 4 nextn layers). It is
|
|
# passed via --model-draft with --spec-type draft-mtp: the draft context
|
|
# loads the assistant model. The same --spec-type value chains the
|
|
# draftless ngram-mod matcher, which attaches unconditionally; two
|
|
# context-backed drafters cannot share one process, so this chain is the
|
|
# only shape that carries both.
|
|
#
|
|
# Context: CTX_SIZE is set explicitly to PARALLEL x 262144 so every slot
|
|
# gets the model's FULL window; --fit (the CTX_SIZE-unset branch below)
|
|
# maximises the TOTAL pool instead, which would divide a smaller window
|
|
# across slots. The 4-slot layout is measured to fit the card (see the
|
|
# SIZING note in frank-small.env); more slots would fail the KV allocation.
|
|
#
|
|
# Vision + audio: --mmproj loads the BF16 projector, which also carries
|
|
# an audio encoder — input_audio content parts transcribe verbatim
|
|
# (verified 2026-09-17). The fleet's image path lives here, which is why
|
|
# halogen no longer sets HALOGEN_VISION_TOWER.
|
|
#
|
|
# Sampling: greedy by default (the fleet contract). Clients that send their own
|
|
# temperature get their own values, and the harness overrides per model from
|
|
# models.yml. --jinja is pinned even though this build defaults it on: the
|
|
# model's own template supplies reasoning_effort, preserve_thinking and the
|
|
# tool-call grammar, and a future build that flips that default must not
|
|
# silently take tools away.
|
|
#
|
|
# Tunables come from frank-small.env unless FRANK_SMALL_ENV_FILE overrides it.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -------------------
|
|
: "${LLAMA_BIN:=/home/dank/GitPackages/llama.cpp-mainline/build-cuda/bin/llama-server}"
|
|
: "${MODEL_DIR:=/home/dank/models/Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp}"
|
|
: "${MODEL_FILE:=Qwen3.8-27B-GSQ-RCO-IQ3_S-mtp.gguf}"
|
|
: "${MMPROJ_PATH:=$MODEL_DIR/mmproj-Gemma4-12B-QAT-Uncensored-HauhauCS-Balanced-BF16.gguf}"
|
|
: "${DRAFT_FILE:=}"
|
|
: "${ALIAS:=frank-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7771}"
|
|
: "${CTX_SIZE:=}"
|
|
: "${PARALLEL:=1}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${BATCH_SIZE:=2048}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
# Decode-shape knobs: the MTP drafter's per-round proposal length (build
|
|
# default 3; the Gemma4 sidecar carries 4 nextn layers) and the engine's
|
|
# busy-wait level (build default 50).
|
|
: "${SPEC_DRAFT_N_MAX:=3}"
|
|
: "${POLL_LEVEL:=50}"
|
|
: "${NUM_THREADS:=8}"
|
|
: "${THREAD_BATCH:=8}"
|
|
: "${NGL:=999}"
|
|
: "${FLASH_ATTN:=on}"
|
|
: "${SPEC_TYPE:=draft-mtp}"
|
|
: "${TEMPERATURE:=0.0}"
|
|
: "${REASONING_FORMAT:=deepseek}"
|
|
: "${REASONING_BUDGET:=-1}"
|
|
|
|
# Fail loudly rather than half-loading a model into VRAM: a missing file here
|
|
# is a download that has not finished, and the unit's restart policy would
|
|
# otherwise turn that into a crash loop that reads like a runtime fault.
|
|
for path in "$LLAMA_BIN" "$MODEL_DIR/$MODEL_FILE" "$MMPROJ_PATH"; do
|
|
[[ -e "$path" ]] || { echo "frank-small: missing $path" >&2; exit 1; }
|
|
done
|
|
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
|
[[ -e "$MODEL_DIR/$DRAFT_FILE" ]] || { echo "frank-small: missing $MODEL_DIR/$DRAFT_FILE" >&2; exit 1; }
|
|
fi
|
|
|
|
# The UVM device node is created lazily by nvidia-modprobe on the first CUDA
|
|
# open after boot; make sure it exists before the server needs it.
|
|
nvidia-modprobe -u -c=0 2>/dev/null || true
|
|
|
|
MMPROJ_ARGS=()
|
|
if [[ -n "${MMPROJ_PATH:-}" && -f "$MMPROJ_PATH" ]]; then
|
|
MMPROJ_ARGS=(--mmproj "$MMPROJ_PATH")
|
|
fi
|
|
|
|
SPEC_ARGS=()
|
|
if [[ -n "${SPEC_TYPE:-}" && "$SPEC_TYPE" != "none" ]]; then
|
|
SPEC_ARGS=(--spec-type "$SPEC_TYPE" --spec-draft-n-max "$SPEC_DRAFT_N_MAX")
|
|
fi
|
|
|
|
DRAFT_ARGS=()
|
|
if [[ -n "${DRAFT_FILE:-}" ]]; then
|
|
DRAFT_ARGS=(--model-draft "$MODEL_DIR/$DRAFT_FILE")
|
|
fi
|
|
|
|
# CTX_SIZE set -> pinned context. Unset -> --fit sizes the KV pool against
|
|
# whatever the weights, projector, draft context and compute buffers leave
|
|
# on the card; the engine prints the chosen value at startup.
|
|
if [[ -n "${CTX_SIZE:-}" ]]; then
|
|
CTX_ARGS=(--ctx-size "$CTX_SIZE")
|
|
else
|
|
CTX_ARGS=(--fit on)
|
|
fi
|
|
|
|
exec "$LLAMA_BIN" \
|
|
--model "$MODEL_DIR/$MODEL_FILE" \
|
|
"${MMPROJ_ARGS[@]}" \
|
|
"${DRAFT_ARGS[@]}" \
|
|
--alias "$ALIAS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" \
|
|
--gpu-layers "$NGL" \
|
|
--flash-attn "$FLASH_ATTN" \
|
|
"${CTX_ARGS[@]}" \
|
|
--parallel "$PARALLEL" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--threads "$NUM_THREADS" \
|
|
--threads-batch "$THREAD_BATCH" \
|
|
--poll "$POLL_LEVEL" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
"${SPEC_ARGS[@]}" \
|
|
--jinja \
|
|
--metrics \
|
|
--temp "$TEMPERATURE" \
|
|
--reasoning-format "$REASONING_FORMAT" \
|
|
--reasoning-budget "$REASONING_BUDGET"
|