Frank
4144f0f0e8
ik_llama limits speculative decoding to a single server slot (-np 1) and hard-exits at load if a spec stage is configured with --parallel > 1: "Speculative decoding is currently limited to a single server slot (-np 1)." This service keeps PARALLEL=3 (one slot per user), so set SPEC_TYPE=none to disable speculation. It is a low-parameter MoE (3B active of 35B), so throughput is fine without it. MTP stays disabled. KB and tests updated.
118 lines
4.2 KiB
Bash
Executable File
118 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve dealignai/Ornith-1.5-35B-A3B-UNCENSORED (CRACK, Q6_K) as
|
|
# frank-default-small via ik_llama.cpp on Dankserver (RTX 3090, MoE
|
|
# experts on CPU).
|
|
#
|
|
# Runtime: ik_llama.cpp llama-server (ikawrakow ik_llama.cpp). The model is a
|
|
# 35B-total / 3B-active MoE-dense hybrid with an MTP head bundled in the GGUF.
|
|
# Speculative decoding is disabled: ik_llama limits any spec stage to a single
|
|
# server slot (-np 1), and this service keeps 3 parallel slots.
|
|
#
|
|
# KV cache: q8_0 with Hadamard transforms on K and V (requires a build with
|
|
# GGML_CUDA_FA_ALL_QUANTS=ON + GGML_IQK_FA_ALL_QUANTS=ON; see the env file).
|
|
# The ik_llama fast q8_KV type crashes at init on this arch (ggml.c:5427);
|
|
# q8_0 is used instead.
|
|
#
|
|
# Split mode is none: Dankserver has a single RTX 3090 and ik_llama rejects
|
|
# split-mode graph with fewer than 2 devices (hard load error).
|
|
|
|
#
|
|
# Expert offload: the expert (FFN exps) tensors selected by the layer-bounded
|
|
# --override-tensor pattern are pinned to CPU. Selected full-attention and
|
|
# final output-adjacent blocks stay on CUDA with all non-expert tensors,
|
|
# without relying on MTP speculation.
|
|
#
|
|
# Speculative decoding: disabled (none). ik_llama hard-exits if a spec stage
|
|
# is configured with --parallel > 1. MTP is also intentionally disabled.
|
|
# A low-parameter MoE (3B active) does not need speculation here.
|
|
#
|
|
# Tunables come from frank-default-small.env unless FRANK_SMALL_ENV_FILE
|
|
# overrides that path.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -----------------
|
|
: "${MODEL_DIR:=/home/dank/models}"
|
|
: "${MODEL_FILE:=Ornith-1.5-35B-A3B-CRACK-Q6_K.gguf}"
|
|
: "${MMPROJ_FILE:=mmproj-Ornith-1.5-35B-A3B-f16.gguf}"
|
|
: "${LLAMA_SERVER:=/home/dank/llama.cpp/build/bin/llama-server}"
|
|
: "${CHAT_TEMPLATE_FILE:=$SCRIPT_DIR/chat-template.jinja}"
|
|
: "${ALIAS:=frank-default-small}"
|
|
: "${BIND_ADDR:=0.0.0.0}"
|
|
: "${LISTEN_PORT:=7770}"
|
|
: "${CTX_SIZE:=491520}"
|
|
: "${PARALLEL:=3}"
|
|
: "${SPEC_TYPE:=none}"
|
|
: "${CACHE_TYPE_K:=f16}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${K_CACHE_HADAMARD:=0}"
|
|
: "${V_CACHE_HADAMARD:=0}"
|
|
: "${SPLIT_MODE:=none}"
|
|
: "${VERBOSITY:=4}"
|
|
# Expert offload: --override-tensor regex pinned to the CPU buft. Same pattern
|
|
# the internal --cpu-moe / --n-cpu-moe use; explicit here so we only move the
|
|
# MoE expert weights and leave every other tensor on the GPU.
|
|
: "${OVERRIDE_EXPERT:=blk\.(0|1|2|4|5|6|8|9|10|12|13|14|16|17|18|20|21|22|24|25|26|28|29|30|32|33|34)\.ffn_(up|down|gate|gate_up)_(ch|)exps}"
|
|
: "${OVERRIDE_BUFT:=CPU}"
|
|
: "${N_GPU_LAYERS:=999}"
|
|
: "${BATCH_SIZE:=4096}"
|
|
: "${UBATCH_SIZE:=1024}"
|
|
: "${CTX_CHECKPOINTS:=16}"
|
|
: "${NO_MMAP:=1}"
|
|
: "${TEMPERATURE:=1.0}"
|
|
: "${TOP_K:=20}"
|
|
: "${TOP_P:=0.95}"
|
|
: "${MIN_P:=0.0}"
|
|
: "${THREADS:=7}"
|
|
: "${THREADS_HTTP:=8}"
|
|
|
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
|
MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
|
|
|
|
for f in "$MODEL" "$MMPROJ" "$CHAT_TEMPLATE_FILE"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "missing required file: $f" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ ! -x "$LLAMA_SERVER" ]]; then
|
|
echo "missing llama-server binary: $LLAMA_SERVER" >&2
|
|
exit 1
|
|
fi
|
|
|
|
exec "$LLAMA_SERVER" \
|
|
--model "$MODEL" \
|
|
--mmproj "$MMPROJ" \
|
|
--chat-template-file "$CHAT_TEMPLATE_FILE" \
|
|
--spec-type "$SPEC_TYPE" \
|
|
--override-tensor "$OVERRIDE_EXPERT=$OVERRIDE_BUFT" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
$([ "$K_CACHE_HADAMARD" = "1" ] && printf -- "--k-cache-hadamard") \
|
|
$([ "$V_CACHE_HADAMARD" = "1" ] && printf -- "--v-cache-hadamard") \
|
|
--split-mode "$SPLIT_MODE" \
|
|
--verbosity "$VERBOSITY" \
|
|
--ctx-size "$CTX_SIZE" \
|
|
--parallel "$PARALLEL" \
|
|
--gpu-layers "$N_GPU_LAYERS" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
|
$([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
|
|
--temp "$TEMPERATURE" \
|
|
--top-k "$TOP_K" \
|
|
--top-p "$TOP_P" \
|
|
--min-p "$MIN_P" \
|
|
--threads "$THREADS" \
|
|
--threads-http "$THREADS_HTTP" \
|
|
--alias "$ALIAS" \
|
|
--host "$BIND_ADDR" \
|
|
--port "$LISTEN_PORT" |