Frank
04a8fb7806
Reconfigure to 3 parallel 160000-ctx slots (was 1 slot). KV cache is now asymmetric: K at q8_0, V at q5_0, both with Hadamard. The 3 slots triple the KV, so expert blocks 27-34 are pinned back to CPU (freed ~8.7 GiB) leaving output-adjacent 35-40 on CUDA. Q8_0 model, split-mode none, spec disabled (ik limits spec to 1 slot). AGENTS.md: drop the scratch-port rule (test/bench processes cannot share the single GPU with the running service anyway). Env, serve script, bats, docs updated.
120 lines
4.4 KiB
Bash
Executable File
120 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q8_0) as
|
|
# frank-default-small via ik_llama.cpp on Dankserver (RTX 3090, MoE
|
|
# experts on CPU).
|
|
#
|
|
# Runtime: ik_llama.cpp llama-server (ikawrakow ik_llama.cpp). The model is a
|
|
# 35B-total / 3B-active MoE-dense hybrid with an MTP head bundled in the GGUF.
|
|
# Three parallel slots (PARALLEL=3), each with 160000 ctx. Speculative decoding
|
|
# is disabled (ik_llama limits spec to a single slot).
|
|
#
|
|
# KV cache: K q8_0 + V q5_0 with Hadamard transforms (requires a build
|
|
# with GGML_CUDA_FA_ALL_QUANTS=ON + GGML_IQK_FA_ALL_QUANTS=ON; see the env
|
|
# file). q8_KV crashes at init on this arch (ggml.c:5427); q8_0/q5_0 work.#
|
|
# Split mode is none: Dankserver has a single RTX 3090 and ik_llama rejects
|
|
# split-mode graph with fewer than 2 devices (hard load error).
|
|
|
|
#
|
|
# Expert offload: the expert (FFN exps) tensors selected by the layer-bounded
|
|
# --override-tensor pattern are pinned to CPU. Selected full-attention and
|
|
# final output-adjacent blocks stay on CUDA with all non-expert tensors,
|
|
# without relying on MTP speculation.
|
|
#
|
|
# Speculative decoding: disabled (none). MTP is also intentionally disabled.
|
|
# A low-parameter MoE (3B active) does not need speculation here.#
|
|
# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
|
|
# overrides that path.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -----------------
|
|
: "${MODEL_DIR:=/home/dank/models}"
|
|
: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q8_0.gguf}"
|
|
: "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}"
|
|
: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
|
|
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
|
# Render the Qwen-Sharp template with the true Jinja engine. ik_llama only
|
|
# applies --chat-template-file as real Jinja when --jinja is passed;
|
|
# otherwise it falls back to built-in non-Jinja handlers and silently drops
|
|
# the custom template's thinking/image/tool formatting.
|
|
: "${USE_JINJA:=1}"
|
|
: "${ALIAS:=frank-default-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7770}"
|
|
: "${CTX_SIZE:=160000}"
|
|
: "${PARALLEL:=3}"
|
|
: "${SPEC_TYPE:=none}"
|
|
: "${CACHE_TYPE_K:=q8_0}"
|
|
: "${CACHE_TYPE_V:=q5_0}"
|
|
: "${K_CACHE_HADAMARD:=0}"
|
|
: "${V_CACHE_HADAMARD:=0}"
|
|
: "${SPLIT_MODE:=none}"
|
|
: "${VERBOSITY:=4}"
|
|
# Expert offload: --override-tensor regex pinned to the CPU buft. Same pattern
|
|
# the internal --cpu-moe / --n-cpu-moe use; explicit here so we only move the
|
|
# MoE expert weights and leave every other tensor on the GPU.
|
|
: "${OVERRIDE_EXPERT:=blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps}"
|
|
: "${OVERRIDE_BUFT:=CPU}"
|
|
: "${N_GPU_LAYERS:=999}"
|
|
: "${BATCH_SIZE:=4096}"
|
|
: "${UBATCH_SIZE:=1024}"
|
|
: "${CTX_CHECKPOINTS:=16}"
|
|
: "${NO_MMAP:=1}"
|
|
: "${TEMPERATURE:=1.0}"
|
|
: "${TOP_K:=20}"
|
|
: "${TOP_P:=0.95}"
|
|
: "${MIN_P:=0.0}"
|
|
: "${THREADS:=7}"
|
|
: "${THREADS_HTTP:=8}"
|
|
|
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
|
MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
|
|
|
|
for f in "$MODEL" "$MMPROJ" "$CHAT_TEMPLATE_FILE"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "missing required file: $f" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ ! -x "$LLAMA_SERVER" ]]; then
|
|
echo "missing llama-server binary: $LLAMA_SERVER" >&2
|
|
exit 1
|
|
fi
|
|
|
|
exec "$LLAMA_SERVER" \
|
|
--model "$MODEL" \
|
|
--mmproj "$MMPROJ" \
|
|
--chat-template-file "$CHAT_TEMPLATE_FILE" \
|
|
$([ "$USE_JINJA" = "1" ] && printf -- "--jinja") \
|
|
--spec-type "$SPEC_TYPE" \
|
|
--override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
$([ "$K_CACHE_HADAMARD" = "1" ] && printf -- "--k-cache-hadamard") \
|
|
$([ "$V_CACHE_HADAMARD" = "1" ] && printf -- "--v-cache-hadamard") \
|
|
--split-mode "$SPLIT_MODE" \
|
|
--verbosity "$VERBOSITY" \
|
|
--ctx-size "$CTX_SIZE" \
|
|
--parallel "$PARALLEL" \
|
|
--gpu-layers "$N_GPU_LAYERS" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
|
$([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
|
|
--temp "$TEMPERATURE" \
|
|
--top-k "$TOP_K" \
|
|
--top-p "$TOP_P" \
|
|
--min-p "$MIN_P" \
|
|
--threads "$THREADS" \
|
|
--threads-http "$THREADS_HTTP" \
|
|
--alias "$ALIAS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" |