Frank
aa410f4439
The LLM fleet serve scripts, systemd units, env tunables, and bench harnesses join the deploy scripts. The vllmScripts deploy.sh is removed here; its functionality is folded into the unified scripts/deploy.sh in a follow-up commit. bench-spec-stack.sh gains the executable bit (required by the scripts_syntax test seam).
62 lines
1.8 KiB
Bash
62 lines
1.8 KiB
Bash
# Tunables for frank-default-small (HauhauCS Qwen3.6-35B-A3B Q6_K_P,
|
|
# ngram-mod spec, Dankserver, RTX 3090).
|
|
#
|
|
# Runtime: upstream llama.cpp llama-server (b10121). RTX 3090 24 GB;
|
|
# Ryzen 7 7700X; 62 GB RAM. MoE experts offloaded to CPU (--n-cpu-moe 36).
|
|
# Restored 2026-08-18 (was the pre-2026-08-17-redo frank-default-small).
|
|
#
|
|
# WARNING: do not name any variable TEMP, HOST, or GPU_MAX_HEAP_SIZE. The
|
|
# ROCm HIP runtime reads those bare env vars (see the 2026-08-13 TEMP
|
|
# incident). This box is NVIDIA/CUDA, but the rule keeps the env portable.
|
|
|
|
# Model files (already on disk; Q6_K_P GGUF + f16 mmproj vision encoder).
|
|
MODEL_DIR=/home/dank/models
|
|
MODEL_FILE=Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q6_K_P.gguf
|
|
MMPROJ_FILE=mmproj-Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-f16.gguf
|
|
# Upstream llama.cpp binary (b10121).
|
|
LLAMA_SERVER=/home/dank/llama.cpp/build/bin/llama-server
|
|
|
|
ALIAS=frank-default-small
|
|
BIND_ADDR=0.0.0.0
|
|
LISTEN_PORT=7770
|
|
|
|
# 3 parallel slots x 262144 ctx each = ~786432 total context pool (max per-slot).
|
|
# Dual-f16 KV at 786432 cells leaves ~15 GiB free on the 3090.
|
|
CTX_SIZE=786432
|
|
PARALLEL=3
|
|
|
|
# Spec: ngram-mod (n_min=48, n_max=64, n_match=24).
|
|
SPEC_TYPE=ngram-mod
|
|
NGRAM_MOD_N_MIN=48
|
|
NGRAM_MOD_N_MAX=64
|
|
NGRAM_MOD_N_MATCH=24
|
|
|
|
# Expert offload: 38 of 40 layers keep MoE weights on CPU; 2 on GPU.
|
|
N_CPU_MOE=38
|
|
N_GPU_LAYERS=all
|
|
|
|
# KV cache on GPU, both K and V at f16.
|
|
CACHE_TYPE_K=f16
|
|
CACHE_TYPE_V=f16
|
|
|
|
VERBOSITY=4
|
|
# Store SWA/context checkpoints in RAM to hold peak GPU memory down.
|
|
CTX_CHECKPOINTS=16
|
|
|
|
# Big prefill batches (ubatch 2048, batch 4096).
|
|
BATCH_SIZE=4096
|
|
UBATCH_SIZE=2048
|
|
|
|
# Eager RAM load (MoE experts on CPU; ~31 GB RAM).
|
|
NO_MMAP=1
|
|
|
|
# Sampling (HauhauCS Qwen3.6-35B-A3B model card, thinking-general).
|
|
TEMPERATURE=1.0
|
|
TOP_K=20
|
|
TOP_P=0.95
|
|
MIN_P=0.0
|
|
|
|
# Compute threads lowered to 7 so other processes keep a core.
|
|
THREADS=7
|
|
THREADS_HTTP=8
|