Frank
9971f83e8b
Lower each stream to 200000 ctx (pool 600000) to free KV VRAM for the larger batch. Add --no-mmap so the CPU-resident MoE experts load eagerly into RAM (avoids decode page faults). Keep KV on GPU. Experts already fully on CPU.
75 lines
2.4 KiB
Bash
Executable File
75 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Serve HauhauCS Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive:Q6_K_P
|
|
# with a DFlash draft model on dankserver (RTX 3090, MoE experts on CPU).
|
|
# Runtime: upstream llama.cpp llama-server (b10108+).
|
|
#
|
|
# Tunables come from frank-default-small.env in this directory unless the
|
|
# FRANK_SMALL_ENV_FILE environment variable overrides that path.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="${FRANK_SMALL_ENV_FILE:-$SCRIPT_DIR/frank-default-small.env}"
|
|
if [[ -f "$ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$ENV_FILE"
|
|
set +a
|
|
fi
|
|
|
|
# --- defaults (used when the env file does not set them) -----------------
|
|
: "${MODEL_DIR:=/home/dank/models}"
|
|
: "${MODEL_FILE:=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf}"
|
|
: "${DRAFT_FILE:=Qwen3.6-35B-A3B-DFlash-Q8_0.gguf}"
|
|
: "${MMPROJ_FILE:=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf}"
|
|
: "${ALIAS:=frank-default-small}"
|
|
: "${HOST:=0.0.0.0}"
|
|
: "${PORT:=7770}"
|
|
: "${CTX_SIZE:=262144}"
|
|
: "${PARALLEL:=3}"
|
|
: "${SPEC_TYPE:=draft-dflash}"
|
|
: "${SPEC_N_MAX:=7}"
|
|
: "${CACHE_TYPE_K:=q8_0}"
|
|
: "${CACHE_TYPE_V:=f16}"
|
|
: "${N_CPU_MOE:=40}" # all 40 layers: MoE expert weights stay on CPU
|
|
: "${N_GPU_LAYERS:=all}" # every non-expert tensor goes to GPU
|
|
: "${BATCH_SIZE:=2048}"
|
|
: "${UBATCH_SIZE:=512}"
|
|
: "${CTX_CHECKPOINTS:=0}"
|
|
: "${NO_MMAP:=0}"
|
|
: "${THREADS:=8}" # 7700X physical cores
|
|
: "${THREADS_HTTP:=8}"
|
|
|
|
LLAMA_SERVER="${LLAMA_SERVER:-/home/dank/llama.cpp/build/bin/llama-server}"
|
|
MODEL="$MODEL_DIR/$MODEL_FILE"
|
|
DRAFT="$MODEL_DIR/$DRAFT_FILE"
|
|
MMPROJ="$MODEL_DIR/$MMPROJ_FILE"
|
|
|
|
for f in "$MODEL" "$DRAFT" "$MMPROJ"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "missing required file: $f" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
exec "$LLAMA_SERVER" \
|
|
--model "$MODEL" \
|
|
--model-draft "$DRAFT" \
|
|
--mmproj "$MMPROJ" \
|
|
--jinja \
|
|
--spec-type "$SPEC_TYPE" \
|
|
--spec-draft-n-max "$SPEC_N_MAX" \
|
|
--cache-type-k "$CACHE_TYPE_K" \
|
|
--cache-type-v "$CACHE_TYPE_V" \
|
|
--ctx-size "$CTX_SIZE" \
|
|
--parallel "$PARALLEL" \
|
|
--n-cpu-moe "$N_CPU_MOE" \
|
|
--gpu-layers "$N_GPU_LAYERS" \
|
|
--batch-size "$BATCH_SIZE" \
|
|
--ubatch-size "$UBATCH_SIZE" \
|
|
--ctx-checkpoints "$CTX_CHECKPOINTS" \
|
|
$([ "$NO_MMAP" = "1" ] && printf -- "--no-mmap") \
|
|
--threads "$THREADS" \
|
|
--threads-http "$THREADS_HTTP" \
|
|
--alias "$ALIAS" \
|
|
--host "$HOST" \
|
|
--port "$PORT" |